v1.0.0 #1
23
README.md
Normal file
23
README.md
Normal file
@@ -0,0 +1,23 @@
|
||||
# blocklist
|
||||
|
||||
Пакет Go для получения IP адресов от различных сервисов.
|
||||
|
||||
## Установка
|
||||
```sh
|
||||
go get git.kor-elf.net/kor-elf-shield/blocklist
|
||||
```
|
||||
|
||||
## Примеры использования
|
||||
- [Пример получения списка IP адресов от Spamhaus](/examples/spamhaus.go)
|
||||
- [Пример получения списка IP адресов от DShield](/examples/dshield.go)
|
||||
- [Пример получения списка IP адресов от HONEYPOT](/examples/honeypot.go)
|
||||
- [Пример получения списка IP адресов от Tor](/examples/tor.go)
|
||||
- [Пример получения списка IP адресов от CIARMY](/examples/ciarmy.go)
|
||||
- [Пример получения списка IP адресов от Daniel Gerzo (BruteforceBlocker)](/examples/bruteforceblocker.go)
|
||||
- [Пример получения списка IP адресов от Blocklist.de](/examples/blocklist.go)
|
||||
- [Пример получения списка IP адресов от GreenSnow](/examples/greensnow.go)
|
||||
- [Пример получения списка IP адресов от StopForumSpam](/examples/stopforumspam.go)
|
||||
|
||||
## Лицензия
|
||||
|
||||
[MIT](https://git.kor-elf.net/kor-elf-shield/blocklist/src/branch/main/LICENSE)
|
||||
248
blocklist.go
Normal file
248
blocklist.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package blocklist
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
const (
|
||||
// contextTimeout defines the maximum duration for context operations before timing out.
|
||||
contextTimeout = 15 * time.Second
|
||||
|
||||
// requestTimeout defines the maximum duration for request operations before timing out.
|
||||
requestTimeout = 20 * time.Second
|
||||
|
||||
// maxDownloadSize defines the maximum allowed size of the downloaded file in bytes.
|
||||
maxDownloadSize int64 = 20 << 20 // 20 MiB
|
||||
|
||||
// maxArchiveFileSize defines the maximum allowed size of the extracted file from ZIP in bytes.
|
||||
maxArchiveFileSize uint64 = 50 << 20 // 50 MiB
|
||||
)
|
||||
|
||||
// Config defines the configuration for the blocklist.
|
||||
type Config struct {
|
||||
|
||||
// Limit specifies the maximum number of items to process or validate.
|
||||
Limit uint
|
||||
|
||||
// Validator specifies the IP validator to use.
|
||||
Validator parser.IPValidator
|
||||
|
||||
// ContextTimeout defines the maximum duration for context operations before timing out.
|
||||
ContextTimeout time.Duration
|
||||
|
||||
// RequestTimeout defines the maximum duration for request operations before timing out.
|
||||
RequestTimeout time.Duration
|
||||
}
|
||||
|
||||
type ConfigZip struct {
|
||||
// Config is the configuration for the blocklist.
|
||||
Config Config
|
||||
|
||||
// MaxDownloadSize defines the maximum allowed size of the downloaded file in bytes.
|
||||
MaxDownloadSize int64
|
||||
|
||||
// MaxArchiveFileSize defines the maximum allowed size of the extracted file from ZIP in bytes.
|
||||
MaxArchiveFileSize uint64
|
||||
}
|
||||
|
||||
// NewConfig creates a new Config with default values.
|
||||
// limit is the maximum number of items to process or validate. 0 means no limit.
|
||||
func NewConfig(limit uint) Config {
|
||||
return Config{
|
||||
Limit: limit,
|
||||
Validator: &parser.DefaultIPValidator{},
|
||||
ContextTimeout: contextTimeout,
|
||||
RequestTimeout: requestTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
// NewConfigWithValidator creates a new Config with the specified validator.
|
||||
// limit is the maximum number of items to process or validate. 0 means no limit.
|
||||
// validator is the IP validator to use.
|
||||
func NewConfigWithValidator(limit uint, validator parser.IPValidator) Config {
|
||||
return Config{
|
||||
Limit: limit,
|
||||
Validator: validator,
|
||||
ContextTimeout: contextTimeout,
|
||||
RequestTimeout: requestTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
func NewConfigZip(c Config) ConfigZip {
|
||||
return ConfigZip{
|
||||
Config: c,
|
||||
MaxDownloadSize: maxDownloadSize,
|
||||
MaxArchiveFileSize: maxArchiveFileSize,
|
||||
}
|
||||
}
|
||||
|
||||
// Get fetches data from the given URL, parses the response using the provided parser, and applies the given configuration.
|
||||
// It returns the parsed IPs and any errors that occurred during the process.
|
||||
func Get(fileUrl string, parser parser.Parser, c Config) (parser.IPs, error) {
|
||||
parsedURL, err := url.Parse(fileUrl)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid url: %w", err)
|
||||
}
|
||||
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), c.ContextTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: c.RequestTimeout,
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = res.Body.Close()
|
||||
}()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
return parser.Parse(res.Body, c.Validator, c.Limit)
|
||||
}
|
||||
|
||||
// GetZip fetches data from the given URL, parses the response using the provided parser, and applies the given configuration.
|
||||
// It returns the parsed IPs and any errors that occurred during the process.
|
||||
func GetZip(fileUrl string, parser parser.Parser, c ConfigZip) (parser.IPs, error) {
|
||||
parsedURL, err := url.Parse(fileUrl)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid url: %w", err)
|
||||
}
|
||||
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), c.Config.ContextTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: c.Config.RequestTimeout,
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = res.Body.Close()
|
||||
}()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
if c.MaxDownloadSize > 0 && res.ContentLength > c.MaxDownloadSize {
|
||||
return nil, fmt.Errorf("downloaded file is too large: content-length %d exceeds limit %d", res.ContentLength, c.MaxDownloadSize)
|
||||
}
|
||||
|
||||
reader := res.Body
|
||||
if c.MaxDownloadSize > 0 {
|
||||
reader = io.NopCloser(io.LimitReader(res.Body, c.MaxDownloadSize+1))
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
if c.MaxDownloadSize > 0 && int64(len(body)) > c.MaxDownloadSize {
|
||||
return nil, fmt.Errorf("downloaded file exceeds limit %d bytes", c.MaxDownloadSize)
|
||||
}
|
||||
|
||||
if !isZip(body) {
|
||||
return nil, fmt.Errorf("invalid zip archive")
|
||||
}
|
||||
|
||||
return parseZip(body, parser, c)
|
||||
}
|
||||
|
||||
func isZip(body []byte) bool {
|
||||
return len(body) >= 4 &&
|
||||
body[0] == 'P' &&
|
||||
body[1] == 'K' &&
|
||||
body[2] == 0x03 &&
|
||||
body[3] == 0x04
|
||||
}
|
||||
|
||||
func parseZip(body []byte, p parser.Parser, c ConfigZip) (parser.IPs, error) {
|
||||
reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open zip archive: %w", err)
|
||||
}
|
||||
|
||||
file := findArchiveFile(reader.File)
|
||||
if file == nil {
|
||||
return nil, fmt.Errorf("zip archive does not contain a supported file")
|
||||
}
|
||||
|
||||
if c.MaxArchiveFileSize > 0 && file.UncompressedSize64 > c.MaxArchiveFileSize {
|
||||
return nil, fmt.Errorf("file %q in zip is too large: %d exceeds limit %d", file.Name, file.UncompressedSize64, c.MaxArchiveFileSize)
|
||||
}
|
||||
|
||||
rc, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open file %q from zip: %w", file.Name, err)
|
||||
}
|
||||
defer func() {
|
||||
_ = rc.Close()
|
||||
}()
|
||||
|
||||
var zipReader io.Reader = rc
|
||||
if c.MaxArchiveFileSize > 0 {
|
||||
zipReader = io.LimitReader(rc, int64(c.MaxArchiveFileSize)+1)
|
||||
}
|
||||
|
||||
return p.Parse(zipReader, c.Config.Validator, c.Config.Limit)
|
||||
}
|
||||
|
||||
func findArchiveFile(files []*zip.File) *zip.File {
|
||||
var fallback *zip.File
|
||||
|
||||
for _, file := range files {
|
||||
if file.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
if fallback == nil {
|
||||
fallback = file
|
||||
}
|
||||
|
||||
name := strings.ToLower(file.Name)
|
||||
if strings.HasSuffix(name, ".txt") ||
|
||||
strings.HasSuffix(name, ".json") ||
|
||||
strings.HasSuffix(name, ".xml") ||
|
||||
strings.HasSuffix(name, ".rss") {
|
||||
return file
|
||||
}
|
||||
}
|
||||
|
||||
return fallback
|
||||
}
|
||||
50
examples/blocklist.go
Normal file
50
examples/blocklist.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://www.blocklist.de/en/export.html
|
||||
*/
|
||||
|
||||
func main() {
|
||||
// Getting a list of IP addresses that were entered in the last hour
|
||||
// time=seconds
|
||||
url := "https://api.blocklist.de/getlast.php?time=3600"
|
||||
extract := parser.NewDefaultTextExtract(0, "\t")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
|
||||
/*
|
||||
// This second list retrieves all the IP addresses added in the last 48 hours and is usually a
|
||||
// very large list (over 10000 entries), so be sure that you have the resources available to use it
|
||||
url := "http://lists.blocklist.de/lists/all.txt"
|
||||
extract := parser.NewDefaultTextExtract(0, "\t")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
*/
|
||||
}
|
||||
29
examples/bruteforceblocker.go
Normal file
29
examples/bruteforceblocker.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://danger.rulez.sk/index.php/bruteforceblocker/
|
||||
*/
|
||||
|
||||
func main() {
|
||||
url := "https://danger.rulez.sk/projects/bruteforceblocker/blist.php"
|
||||
extract := parser.NewDefaultTextExtract(0, "\t")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
29
examples/ciarmy.go
Normal file
29
examples/ciarmy.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://www.ciarmy.com/#list
|
||||
*/
|
||||
|
||||
func main() {
|
||||
url := "https://www.ciarmy.com/list/ci-badguys.txt"
|
||||
extract := parser.NewDefaultTextExtract(0, " ")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
47
examples/dshield.go
Normal file
47
examples/dshield.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://dshield.org/
|
||||
*/
|
||||
|
||||
func main() {
|
||||
url := "https://www.dshield.org/block.txt"
|
||||
extract := parser.NewCIDRTextExtract(0, 2, "\t")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
|
||||
/*
|
||||
// You can also get a range of IP addresses from this service (from to)
|
||||
url := "https://www.dshield.org/block.txt"
|
||||
extract := parser.NewIntervalTextExtract(0, 1, "\t")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{})
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
*/
|
||||
}
|
||||
29
examples/greensnow.go
Normal file
29
examples/greensnow.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://greensnow.co/
|
||||
*/
|
||||
|
||||
func main() {
|
||||
url := "https://blocklist.greensnow.co/greensnow.txt"
|
||||
extract := parser.NewDefaultTextExtract(0, " ")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
53
examples/honeypot.go
Normal file
53
examples/honeypot.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://www.projecthoneypot.org/list_of_ips.php
|
||||
*/
|
||||
|
||||
type rssItem struct {
|
||||
Title string `xml:"title"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
url := "https://www.projecthoneypot.org/list_of_ips.php?t=d&rss=1"
|
||||
pars, err := parser.NewRss(func(decoder *xml.Decoder, start xml.StartElement) (string, error) {
|
||||
if start.Name.Local != "item" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
var rssItem rssItem
|
||||
if err := decoder.DecodeElement(&rssItem, &start); err != nil {
|
||||
return "", fmt.Errorf("decode rss item: %w", err)
|
||||
}
|
||||
|
||||
if rssItem.Title == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
fields := strings.Split(rssItem.Title, "|")
|
||||
if len(fields) != 2 {
|
||||
return "", nil
|
||||
}
|
||||
return strings.TrimSpace(fields[0]), nil
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
40
examples/spamhaus.go
Normal file
40
examples/spamhaus.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
||||
*/
|
||||
|
||||
type lineJson struct {
|
||||
IP string `json:"cidr"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
url := "https://www.spamhaus.org/drop/drop_v4.json"
|
||||
//url := "https://www.spamhaus.org/drop/drop_v6.json"
|
||||
pars, err := parser.NewJsonLines(func(item json.RawMessage) (string, error) {
|
||||
var line lineJson
|
||||
if err := json.Unmarshal(item, &line); err != nil {
|
||||
return "", fmt.Errorf("unmarshal json item: %w", err)
|
||||
}
|
||||
return line.IP, nil
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
31
examples/stopforumspam.go
Normal file
31
examples/stopforumspam.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://www.stopforumspam.com/downloads
|
||||
*/
|
||||
|
||||
func main() {
|
||||
url := "https://www.stopforumspam.com/downloads/listed_ip_1.zip"
|
||||
//url := "https://www.stopforumspam.com/downloads/listed_ip_1_ipv6.zip"
|
||||
extract := parser.NewDefaultTextExtract(0, " ")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
configZip := blocklist.NewConfigZip(config)
|
||||
ips, err := blocklist.GetZip(url, pars, configZip)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
29
examples/tor.go
Normal file
29
examples/tor.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist"
|
||||
"git.kor-elf.net/kor-elf-shield/blocklist/parser"
|
||||
)
|
||||
|
||||
/**
|
||||
* An example of how to get a list of IP addresses from a service https://check.torproject.org/torbulkexitlist
|
||||
*/
|
||||
|
||||
func main() {
|
||||
url := "https://check.torproject.org/torbulkexitlist"
|
||||
extract := parser.NewDefaultTextExtract(0, " ")
|
||||
pars, err := parser.NewText(extract)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// limit 0 - no limit
|
||||
limit := uint(0)
|
||||
config := blocklist.NewConfig(limit)
|
||||
ips, err := blocklist.Get(url, pars, config)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(ips)
|
||||
}
|
||||
3
go.mod
Normal file
3
go.mod
Normal file
@@ -0,0 +1,3 @@
|
||||
module git.kor-elf.net/kor-elf-shield/blocklist
|
||||
|
||||
go 1.25
|
||||
65
parser/json_lines.go
Normal file
65
parser/json_lines.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// jsonLinesExtract defines the function signature for extracting IP addresses from a JSON Lines item.
|
||||
type jsonLinesExtract func(item json.RawMessage) (string, error)
|
||||
|
||||
// jsonLinesParser is a parser for JSON Lines data.
|
||||
type jsonLinesParser struct {
|
||||
// extract is the function that extracts an IP address from a JSON Lines item.
|
||||
extract jsonLinesExtract
|
||||
}
|
||||
|
||||
// NewJsonLines creates a new JSON Lines parser.
|
||||
func NewJsonLines(extract jsonLinesExtract) (Parser, error) {
|
||||
if extract == nil {
|
||||
return nil, fmt.Errorf("json lines extract is nil")
|
||||
}
|
||||
|
||||
return &jsonLinesParser{
|
||||
extract: extract,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Parse parses the JSON Lines data from the given reader.
|
||||
// It returns a slice of IP addresses and any errors that occurred during the process.
|
||||
func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) {
|
||||
decoder := json.NewDecoder(body)
|
||||
ips := make(IPs, 0)
|
||||
for {
|
||||
var item json.RawMessage
|
||||
if err := decoder.Decode(&item); err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, fmt.Errorf("decode json item: %w", err)
|
||||
}
|
||||
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
ip, err := p.extract(item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract ip: %w", err)
|
||||
}
|
||||
|
||||
ip = strings.TrimSpace(ip)
|
||||
if !validator.IsValid(ip) {
|
||||
continue
|
||||
}
|
||||
|
||||
ips = append(ips, ip)
|
||||
if limit > 0 && uint(len(ips)) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
91
parser/parser.go
Normal file
91
parser/parser.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Parser interface defines the contract for parsing IP addresses from a given reader.
|
||||
type Parser interface {
|
||||
// Parse reads the body and returns a slice of IP addresses.
|
||||
Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error)
|
||||
}
|
||||
|
||||
// IPValidator interface defines the contract for validating IP addresses.
|
||||
type IPValidator interface {
|
||||
// IsValid checks if the given IP address is valid.
|
||||
IsValid(ip string) bool
|
||||
}
|
||||
|
||||
// IPs is a slice of IP addresses.
|
||||
type IPs []string
|
||||
|
||||
// DefaultIPValidator implements IPValidator interface.
|
||||
// It validates IP addresses by parsing them using net.ParseIP and net.ParseCIDR.
|
||||
type DefaultIPValidator struct{}
|
||||
|
||||
// IsValid checks if the given IP address is valid.
|
||||
// It returns true if the IP address is not a loopback address.
|
||||
func (v *DefaultIPValidator) IsValid(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(value); ip != nil {
|
||||
if ip.IsLoopback() {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if ip, _, err := net.ParseCIDR(value); err == nil {
|
||||
if ip.IsLoopback() {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// IPRangeValidator implements IPValidator interface.
|
||||
// It validates IP ranges by parsing them using net.ParseIP and checking if the start and end IPs are in the same network.
|
||||
type IPRangeValidator struct{}
|
||||
|
||||
// IsValid checks if the given IP range is valid.
|
||||
// It returns true if the start and end IPs are in the same network.
|
||||
func (v *IPRangeValidator) IsValid(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
parts := strings.Split(value, "-")
|
||||
if len(parts) != 2 {
|
||||
return false
|
||||
}
|
||||
|
||||
start := net.ParseIP(strings.TrimSpace(parts[0]))
|
||||
end := net.ParseIP(strings.TrimSpace(parts[1]))
|
||||
if start == nil || end == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
start4 := start.To4()
|
||||
end4 := end.To4()
|
||||
|
||||
switch {
|
||||
case start4 != nil && end4 != nil:
|
||||
return bytes.Compare(start4, end4) <= 0
|
||||
case start4 == nil && end4 == nil:
|
||||
start16 := start.To16()
|
||||
end16 := end.To16()
|
||||
if start16 == nil || end16 == nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Compare(start16, end16) <= 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
68
parser/rss.go
Normal file
68
parser/rss.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rssExtract defines the function signature for extracting IP addresses from a RSS item.
|
||||
type rssExtract func(decoder *xml.Decoder, start xml.StartElement) (string, error)
|
||||
|
||||
// rssParser is a parser for RSS data.
|
||||
type rssParser struct {
|
||||
// extract is the function that extracts an IP address from a RSS item.
|
||||
extract rssExtract
|
||||
}
|
||||
|
||||
// NewRss creates a new RSS parser.
|
||||
func NewRss(extract rssExtract) (Parser, error) {
|
||||
if extract == nil {
|
||||
return nil, fmt.Errorf("rss extract is nil")
|
||||
}
|
||||
|
||||
return &rssParser{
|
||||
extract: extract,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Parse parses the RSS data from the given reader.
|
||||
// It returns a slice of IP addresses and any errors that occurred during the process.
|
||||
func (p *rssParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) {
|
||||
decoder := xml.NewDecoder(body)
|
||||
ips := make(IPs, 0)
|
||||
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("parse rss: %w", err)
|
||||
}
|
||||
|
||||
start, ok := token.(xml.StartElement)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
ip, err := p.extract(decoder, start)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extract rss ip: %w", err)
|
||||
}
|
||||
|
||||
ip = strings.TrimSpace(ip)
|
||||
if !validator.IsValid(ip) {
|
||||
continue
|
||||
}
|
||||
|
||||
ips = append(ips, ip)
|
||||
if limit > 0 && uint(len(ips)) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
160
parser/text.go
Normal file
160
parser/text.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TextExtract defines the function signature for extracting IP addresses from a text item.
|
||||
type TextExtract interface {
|
||||
// Extract extracts an IP address from the given text line.
|
||||
Extract(line string) (string, bool)
|
||||
}
|
||||
|
||||
// defaultTextExtract is a default implementation of TextExtract.
|
||||
type defaultTextExtract struct {
|
||||
// fieldIndexOne is the index of the field that contains the IP address.
|
||||
fieldIndexOne uint8
|
||||
|
||||
// separator is the separator used to split the fields.
|
||||
separator string
|
||||
}
|
||||
|
||||
// Extract extracts an IP address from the given text line.
|
||||
// It returns the IP address and a boolean indicating whether the IP address was found.
|
||||
func (d *defaultTextExtract) Extract(line string) (string, bool) {
|
||||
fields := strings.Split(line, d.separator)
|
||||
if len(fields) <= int(d.fieldIndexOne) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return fields[d.fieldIndexOne], true
|
||||
}
|
||||
|
||||
// intervalTextExtract is a TextExtract implementation that extracts an IP address range from a text line.
|
||||
type intervalTextExtract struct {
|
||||
// fieldIndexOne specifies the index of the first field to extract from the split string.
|
||||
// From this field, the IP address range will be extracted.
|
||||
fieldIndexOne uint8
|
||||
|
||||
// fieldIndexTwo specifies the index of the second field to extract from the split string.
|
||||
// This field will be used as the end of the IP address range.
|
||||
fieldIndexTwo uint8
|
||||
|
||||
// separator specifies the separator used to split the fields.
|
||||
separator string
|
||||
}
|
||||
|
||||
// Extract extracts an IP address range from the given text line.
|
||||
// It returns the IP address range and a boolean indicating whether the IP address range was found.
|
||||
func (d *intervalTextExtract) Extract(line string) (string, bool) {
|
||||
fields := strings.Split(line, d.separator)
|
||||
if len(fields) <= int(d.fieldIndexOne) || len(fields) <= int(d.fieldIndexTwo) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return fields[d.fieldIndexOne] + "-" + fields[d.fieldIndexTwo], true
|
||||
}
|
||||
|
||||
// cidrTextExtract is a TextExtract implementation that extracts an IP address range from a text line.
|
||||
type cidrTextExtract struct {
|
||||
// fieldIndexOne is the index of the field that contains the IP address.
|
||||
fieldIndexOne uint8
|
||||
|
||||
// fieldIndexTwo is the index of the field that contains the CIDR prefix length.
|
||||
fieldIndexTwo uint8
|
||||
|
||||
// separator is the separator used to split the fields.
|
||||
separator string
|
||||
}
|
||||
|
||||
// Extract extracts an IP address range from the given text line.
|
||||
// It returns the IP address range and a boolean indicating whether the IP address range was found.
|
||||
func (d *cidrTextExtract) Extract(line string) (string, bool) {
|
||||
fields := strings.Split(line, d.separator)
|
||||
if len(fields) <= int(d.fieldIndexOne) || len(fields) <= int(d.fieldIndexTwo) {
|
||||
return "", false
|
||||
}
|
||||
return fields[d.fieldIndexOne] + "/" + fields[d.fieldIndexTwo], true
|
||||
}
|
||||
|
||||
// NewDefaultTextExtract creates a new default TextExtract instance.
|
||||
func NewDefaultTextExtract(fieldIndexOne uint8, separator string) TextExtract {
|
||||
return &defaultTextExtract{
|
||||
fieldIndexOne: fieldIndexOne,
|
||||
separator: separator,
|
||||
}
|
||||
}
|
||||
|
||||
// NewIntervalTextExtract creates a new TextExtract instance that extracts an IP address range.
|
||||
func NewIntervalTextExtract(fieldIndexOne uint8, fieldIndexTwo uint8, separator string) TextExtract {
|
||||
return &intervalTextExtract{
|
||||
fieldIndexOne: fieldIndexOne,
|
||||
fieldIndexTwo: fieldIndexTwo,
|
||||
separator: separator,
|
||||
}
|
||||
}
|
||||
|
||||
// NewCIDRTextExtract creates a new TextExtract instance that extracts an IP address range.
|
||||
func NewCIDRTextExtract(fieldIndexOne uint8, fieldIndexTwo uint8, separator string) TextExtract {
|
||||
return &cidrTextExtract{
|
||||
fieldIndexOne: fieldIndexOne,
|
||||
fieldIndexTwo: fieldIndexTwo,
|
||||
separator: separator,
|
||||
}
|
||||
}
|
||||
|
||||
// textParser is a parser implementation that reads text data from an io.Reader and extracts IP addresses.
|
||||
type textParser struct {
|
||||
textExtract TextExtract
|
||||
}
|
||||
|
||||
// NewText creates a new TextParser instance.
|
||||
func NewText(textExtract TextExtract) (Parser, error) {
|
||||
if textExtract == nil || textExtract.Extract == nil {
|
||||
return nil, fmt.Errorf("text extract is nil")
|
||||
}
|
||||
|
||||
return &textParser{
|
||||
textExtract: textExtract,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Parse reads text data from the given io.Reader and extracts IP addresses.
|
||||
func (p *textParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) {
|
||||
scanner := bufio.NewScanner(body)
|
||||
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024)
|
||||
|
||||
var ips IPs
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
ip, isFound := p.textExtract.Extract(line)
|
||||
if !isFound {
|
||||
continue
|
||||
}
|
||||
|
||||
ip = strings.TrimSpace(ip)
|
||||
if !validator.IsValid(ip) {
|
||||
continue
|
||||
}
|
||||
|
||||
ips = append(ips, ip)
|
||||
if limit > 0 && uint(len(ips)) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
return ips, nil
|
||||
}
|
||||
Reference in New Issue
Block a user