From b72401fba658a0e3d0c2e837e06ea3906767d930 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:23:59 +0500 Subject: [PATCH 01/32] Initialize Go module for the project --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..58c6f9f --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module git.kor-elf.net/kor-elf-shield/blocklist + +go 1.25 -- 2.49.1 From 2409794159cd108307e2a87232aee5502ca7aad5 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:35:35 +0500 Subject: [PATCH 02/32] Add IP address parser and validators --- parser/parser.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 parser/parser.go diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..eb23709 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,91 @@ +package parser + +import ( + "bytes" + "io" + "net" + "strings" +) + +// Parser interface defines the contract for parsing IP addresses from a given reader. +type Parser interface { + // Parse reads the body and returns a slice of IP addresses. + Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) +} + +// IPValidator interface defines the contract for validating IP addresses. +type IPValidator interface { + // IsValid checks if the given IP address is valid. + IsValid(ip string) bool +} + +// IPs is a slice of IP addresses. +type IPs []string + +// DefaultIPValidator implements IPValidator interface. +// It validates IP addresses by parsing them using net.ParseIP and net.ParseCIDR. +type DefaultIPValidator struct{} + +// IsValid checks if the given IP address is valid. +// It returns true if the IP address is not a loopback address. +func (v *DefaultIPValidator) IsValid(value string) bool { + if value == "" { + return false + } + + if ip := net.ParseIP(value); ip != nil { + if ip.IsLoopback() { + return false + } + return true + } + + if ip, _, err := net.ParseCIDR(value); err == nil { + if ip.IsLoopback() { + return false + } + return true + } + + return false +} + +// IPRangeValidator implements IPValidator interface. +// It validates IP ranges by parsing them using net.ParseIP and checking if the start and end IPs are in the same network. +type IPRangeValidator struct{} + +// IsValid checks if the given IP range is valid. +// It returns true if the start and end IPs are in the same network. +func (v *IPRangeValidator) IsValid(value string) bool { + if value == "" { + return false + } + + parts := strings.Split(value, "-") + if len(parts) != 2 { + return false + } + + start := net.ParseIP(strings.TrimSpace(parts[0])) + end := net.ParseIP(strings.TrimSpace(parts[1])) + if start == nil || end == nil { + return false + } + + start4 := start.To4() + end4 := end.To4() + + switch { + case start4 != nil && end4 != nil: + return bytes.Compare(start4, end4) <= 0 + case start4 == nil && end4 == nil: + start16 := start.To16() + end16 := end.To16() + if start16 == nil || end16 == nil { + return false + } + return bytes.Compare(start16, end16) <= 0 + default: + return false + } +} -- 2.49.1 From cc0dc64d91568bd65da920fc415f2555ba3ca0f5 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:36:04 +0500 Subject: [PATCH 03/32] Add blocklist package with config and data fetching functions --- blocklist.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 blocklist.go diff --git a/blocklist.go b/blocklist.go new file mode 100644 index 0000000..06a2e7b --- /dev/null +++ b/blocklist.go @@ -0,0 +1,93 @@ +package blocklist + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +const ( + + // contextTimeout defines the maximum duration for context operations before timing out. + contextTimeout = 15 * time.Second + // requestTimeout defines the maximum duration for request operations before timing out. + requestTimeout = 20 * time.Second +) + +// Config defines the configuration for the blocklist. +type Config struct { + + // Limit specifies the maximum number of items to process or validate. + Limit uint + + // Validator specifies the IP validator to use. + Validator parser.IPValidator + + // ContextTimeout defines the maximum duration for context operations before timing out. + ContextTimeout time.Duration + + // RequestTimeout defines the maximum duration for request operations before timing out. + RequestTimeout time.Duration +} + +// NewConfig creates a new Config with default values. +func NewConfig(limit uint) Config { + return Config{ + Limit: limit, + Validator: &parser.DefaultIPValidator{}, + ContextTimeout: contextTimeout, + RequestTimeout: requestTimeout, + } +} + +// NewConfigWithValidator creates a new Config with the specified validator. +func NewConfigWithValidator(limit uint, validator parser.IPValidator) Config { + return Config{ + Limit: limit, + Validator: validator, + ContextTimeout: contextTimeout, + RequestTimeout: requestTimeout, + } +} + +// Get fetches data from the given URL, parses the response using the provided parser, and applies the given configuration. +// It returns the parsed IPs and any errors that occurred during the process. +func Get(fileUrl string, parser parser.Parser, c Config) (parser.IPs, error) { + parsedURL, err := url.Parse(fileUrl) + if err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme) + } + + ctx, cancel := context.WithTimeout(context.Background(), c.ContextTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + client := &http.Client{ + Timeout: c.RequestTimeout, + } + + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer func() { + _ = res.Body.Close() + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + + return parser.Parse(res.Body, c.Validator, c.Limit) +} -- 2.49.1 From 7f2376a9047f01958a691e34d9fbb51f1fa146ba Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:36:25 +0500 Subject: [PATCH 04/32] Add JSON lines parser implementation with IP extraction support --- parser/json_lines.go | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 parser/json_lines.go diff --git a/parser/json_lines.go b/parser/json_lines.go new file mode 100644 index 0000000..c67f192 --- /dev/null +++ b/parser/json_lines.go @@ -0,0 +1,59 @@ +package parser + +import ( + "encoding/json" + "fmt" + "io" + "strings" +) + +type jsonLinesExtract func(item json.RawMessage) (string, error) + +type jsonLinesParser struct { + extract jsonLinesExtract +} + +func NewJsonLines(extract jsonLinesExtract) (Parser, error) { + if extract == nil { + return nil, fmt.Errorf("json lines extract is nil") + } + + return &jsonLinesParser{ + extract: extract, + }, nil +} + +func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) { + decoder := json.NewDecoder(body) + ips := make(IPs, 0) + for { + var item json.RawMessage + if err := decoder.Decode(&item); err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("decode json item: %w", err) + } + + if item == nil { + continue + } + + ip, err := p.extract(item) + if err != nil { + return nil, fmt.Errorf("extract ip: %w", err) + } + + ip = strings.TrimSpace(ip) + if !validator.IsValid(ip) { + continue + } + + ips = append(ips, ip) + if limit > 0 && uint(len(ips)) >= limit { + break + } + } + + return ips, nil +} -- 2.49.1 From b47597310cb08794fa6bdf86bea8d221626b3fb8 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:37:17 +0500 Subject: [PATCH 05/32] Improve readability by adjusting blocklist constants spacing --- blocklist.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blocklist.go b/blocklist.go index 06a2e7b..84f7ec5 100644 --- a/blocklist.go +++ b/blocklist.go @@ -11,9 +11,9 @@ import ( ) const ( - // contextTimeout defines the maximum duration for context operations before timing out. contextTimeout = 15 * time.Second + // requestTimeout defines the maximum duration for request operations before timing out. requestTimeout = 20 * time.Second ) -- 2.49.1 From c1e0d071a6206203cfeb44ae03868ea5815d49d0 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:41:24 +0500 Subject: [PATCH 06/32] Add comments to JSON Lines parser for improved documentation --- parser/json_lines.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/parser/json_lines.go b/parser/json_lines.go index c67f192..2192a04 100644 --- a/parser/json_lines.go +++ b/parser/json_lines.go @@ -7,12 +7,14 @@ import ( "strings" ) +// jsonLinesExtract defines the function signature for extracting IP addresses from a JSON Lines item. type jsonLinesExtract func(item json.RawMessage) (string, error) type jsonLinesParser struct { extract jsonLinesExtract } +// NewJsonLines creates a new JSON Lines parser. func NewJsonLines(extract jsonLinesExtract) (Parser, error) { if extract == nil { return nil, fmt.Errorf("json lines extract is nil") @@ -23,6 +25,8 @@ func NewJsonLines(extract jsonLinesExtract) (Parser, error) { }, nil } +// Parse parses the JSON Lines data from the given reader. +// It returns a slice of IP addresses and any errors that occurred during the process. func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) { decoder := json.NewDecoder(body) ips := make(IPs, 0) -- 2.49.1 From 1f95752b451cdf4355972f242583d517eca6a761 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:47:27 +0500 Subject: [PATCH 07/32] Add comments to jsonLinesParser for improved documentation --- parser/json_lines.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parser/json_lines.go b/parser/json_lines.go index 2192a04..eca1f55 100644 --- a/parser/json_lines.go +++ b/parser/json_lines.go @@ -10,7 +10,9 @@ import ( // jsonLinesExtract defines the function signature for extracting IP addresses from a JSON Lines item. type jsonLinesExtract func(item json.RawMessage) (string, error) +// jsonLinesParser is a parser for JSON Lines data. type jsonLinesParser struct { + // extract is the function that extracts an IP address from a JSON Lines item. extract jsonLinesExtract } -- 2.49.1 From b649509fcbb49d7fbb3bf1dca18544eae4e6032c Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:54:24 +0500 Subject: [PATCH 08/32] Add text parser implementation for IP extraction from plain text files --- parser/text.go | 160 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 parser/text.go diff --git a/parser/text.go b/parser/text.go new file mode 100644 index 0000000..528eb5c --- /dev/null +++ b/parser/text.go @@ -0,0 +1,160 @@ +package parser + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +// TextExtract defines the function signature for extracting IP addresses from a text item. +type TextExtract interface { + // Extract extracts an IP address from the given text line. + Extract(line string) (string, bool) +} + +// defaultTextExtract is a default implementation of TextExtract. +type defaultTextExtract struct { + // fieldIndexOne is the index of the field that contains the IP address. + fieldIndexOne uint8 + + // separator is the separator used to split the fields. + separator string +} + +// Extract extracts an IP address from the given text line. +// It returns the IP address and a boolean indicating whether the IP address was found. +func (d *defaultTextExtract) Extract(line string) (string, bool) { + fields := strings.Split(line, d.separator) + if len(fields) <= int(d.fieldIndexOne) { + return "", false + } + + return fields[d.fieldIndexOne], true +} + +// intervalTextExtract is a TextExtract implementation that extracts an IP address range from a text line. +type intervalTextExtract struct { + // fieldIndexOne specifies the index of the first field to extract from the split string. + // From this field, the IP address range will be extracted. + fieldIndexOne uint8 + + // fieldIndexTwo specifies the index of the second field to extract from the split string. + // This field will be used as the end of the IP address range. + fieldIndexTwo uint8 + + // separator specifies the separator used to split the fields. + separator string +} + +// Extract extracts an IP address range from the given text line. +// It returns the IP address range and a boolean indicating whether the IP address range was found. +func (d *intervalTextExtract) Extract(line string) (string, bool) { + fields := strings.Split(line, d.separator) + if len(fields) <= int(d.fieldIndexOne) || len(fields) <= int(d.fieldIndexTwo) { + return "", false + } + + return fields[d.fieldIndexOne] + "-" + fields[d.fieldIndexTwo], true +} + +// cidrTextExtract is a TextExtract implementation that extracts an IP address range from a text line. +type cidrTextExtract struct { + // fieldIndexOne is the index of the field that contains the IP address. + fieldIndexOne uint8 + + // fieldIndexTwo is the index of the field that contains the CIDR prefix length. + fieldIndexTwo uint8 + + // separator is the separator used to split the fields. + separator string +} + +// Extract extracts an IP address range from the given text line. +// It returns the IP address range and a boolean indicating whether the IP address range was found. +func (d *cidrTextExtract) Extract(line string) (string, bool) { + fields := strings.Split(line, d.separator) + if len(fields) <= int(d.fieldIndexOne) || len(fields) <= int(d.fieldIndexTwo) { + return "", false + } + return fields[d.fieldIndexOne] + "/" + fields[d.fieldIndexTwo], true +} + +// NewDefaultTextExtract creates a new default TextExtract instance. +func NewDefaultTextExtract(fieldIndexOne uint8, separator string) TextExtract { + return &defaultTextExtract{ + fieldIndexOne: fieldIndexOne, + separator: separator, + } +} + +// NewIntervalTextExtract creates a new TextExtract instance that extracts an IP address range. +func NewIntervalTextExtract(fieldIndexOne uint8, fieldIndexTwo uint8, separator string) TextExtract { + return &intervalTextExtract{ + fieldIndexOne: fieldIndexOne, + fieldIndexTwo: fieldIndexTwo, + separator: separator, + } +} + +// NewCIDRTextExtract creates a new TextExtract instance that extracts an IP address range. +func NewCIDRTextExtract(fieldIndexOne uint8, fieldIndexTwo uint8, separator string) TextExtract { + return &cidrTextExtract{ + fieldIndexOne: fieldIndexOne, + fieldIndexTwo: fieldIndexTwo, + separator: separator, + } +} + +// textParser is a parser implementation that reads text data from an io.Reader and extracts IP addresses. +type textParser struct { + textExtract TextExtract +} + +// NewText creates a new TextParser instance. +func NewText(textExtract TextExtract) (Parser, error) { + if textExtract == nil || textExtract.Extract == nil { + return nil, fmt.Errorf("text extract is nil") + } + + return &textParser{ + textExtract: textExtract, + }, nil +} + +// Parse reads text data from the given io.Reader and extracts IP addresses. +func (p *textParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) { + scanner := bufio.NewScanner(body) + + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + + var ips IPs + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") { + continue + } + + ip, isFound := p.textExtract.Extract(line) + if !isFound { + continue + } + + ip = strings.TrimSpace(ip) + if !validator.IsValid(ip) { + continue + } + + ips = append(ips, ip) + if limit > 0 && uint(len(ips)) >= limit { + break + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + + return ips, nil +} -- 2.49.1 From ae2bcc6ddfff010e2ac5ab746573054c09f6e574 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sat, 14 Mar 2026 11:55:48 +0500 Subject: [PATCH 09/32] Add RSS parser implementation for IP extraction --- parser/rss.go | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 parser/rss.go diff --git a/parser/rss.go b/parser/rss.go new file mode 100644 index 0000000..c17c150 --- /dev/null +++ b/parser/rss.go @@ -0,0 +1,68 @@ +package parser + +import ( + "encoding/xml" + "fmt" + "io" + "strings" +) + +// rssExtract defines the function signature for extracting IP addresses from a RSS item. +type rssExtract func(decoder *xml.Decoder, start xml.StartElement) (string, error) + +// rssParser is a parser for RSS data. +type rssParser struct { + // extract is the function that extracts an IP address from a RSS item. + extract rssExtract +} + +// NewRss creates a new RSS parser. +func NewRss(extract rssExtract) (Parser, error) { + if extract == nil { + return nil, fmt.Errorf("rss extract is nil") + } + + return &rssParser{ + extract: extract, + }, nil +} + +// Parse parses the RSS data from the given reader. +// It returns a slice of IP addresses and any errors that occurred during the process. +func (p *rssParser) Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error) { + decoder := xml.NewDecoder(body) + ips := make(IPs, 0) + + for { + token, err := decoder.Token() + if err != nil { + if err == io.EOF { + break + } + + return nil, fmt.Errorf("parse rss: %w", err) + } + + start, ok := token.(xml.StartElement) + if !ok { + continue + } + + ip, err := p.extract(decoder, start) + if err != nil { + return nil, fmt.Errorf("extract rss ip: %w", err) + } + + ip = strings.TrimSpace(ip) + if !validator.IsValid(ip) { + continue + } + + ips = append(ips, ip) + if limit > 0 && uint(len(ips)) >= limit { + break + } + } + + return ips, nil +} -- 2.49.1 From 16f233197ddccb1e9f4837774a91abfed9231b1e Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 15:35:19 +0500 Subject: [PATCH 10/32] Add comment to NewConfig explaining the limit parameter --- blocklist.go | 1 + 1 file changed, 1 insertion(+) diff --git a/blocklist.go b/blocklist.go index 84f7ec5..cb06c37 100644 --- a/blocklist.go +++ b/blocklist.go @@ -35,6 +35,7 @@ type Config struct { } // NewConfig creates a new Config with default values. +// limit is the maximum number of items to process or validate. 0 means no limit. func NewConfig(limit uint) Config { return Config{ Limit: limit, -- 2.49.1 From 941e0541bfc54f36c6119611261df030856865e6 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 15:46:59 +0500 Subject: [PATCH 11/32] Add Spamhaus JSON example to demonstrate blocklist usage --- examples/spamhaus.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 examples/spamhaus.go diff --git a/examples/spamhaus.go b/examples/spamhaus.go new file mode 100644 index 0000000..76bab7d --- /dev/null +++ b/examples/spamhaus.go @@ -0,0 +1,40 @@ +package main + +import ( + "encoding/json" + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of bad IP addresses from the service https://www.spamhaus.org/blocklists/do-not-route-or-peer/ + */ + +type lineJson struct { + IP string `json:"cidr"` +} + +func main() { + url := "https://www.spamhaus.org/drop/drop_v4.json" + //url := "https://www.spamhaus.org/drop/drop_v6.json" + pars, err := parser.NewJsonLines(func(item json.RawMessage) (string, error) { + var line lineJson + if err := json.Unmarshal(item, &line); err != nil { + return "", fmt.Errorf("unmarshal json item: %w", err) + } + return line.IP, nil + }) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From 7702b5a431c8a2c9caa6a9b2c315ff882ab685d8 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 15:56:08 +0500 Subject: [PATCH 12/32] Add DShield example for fetching and parsing blocklist data --- examples/dshield.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 examples/dshield.go diff --git a/examples/dshield.go b/examples/dshield.go new file mode 100644 index 0000000..e507159 --- /dev/null +++ b/examples/dshield.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of bad IP addresses from the service https://dshield.org/ + */ + +func main() { + url := "https://www.dshield.org/block.txt" + extract := parser.NewCIDRTextExtract(0, 2, "\t") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + config := blocklist.NewConfig(0) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) + + /* + // You can also get a range of IP addresses from this service (from to) + url := "https://www.dshield.org/block.txt" + extract := parser.NewIntervalTextExtract(0, 1, "\t") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + config := blocklist.NewConfigWithValidator(0, &parser.IPRangeValidator{}) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) + */ +} -- 2.49.1 From 9cda5799e1814c64f78677add73f91520770b6ec Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:03:58 +0500 Subject: [PATCH 13/32] Clarify usage of limit parameter in DShield example --- examples/dshield.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/dshield.go b/examples/dshield.go index e507159..f0f12ff 100644 --- a/examples/dshield.go +++ b/examples/dshield.go @@ -18,7 +18,9 @@ func main() { if err != nil { panic(err) } - config := blocklist.NewConfig(0) + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) ips, err := blocklist.Get(url, pars, config) if err != nil { panic(err) @@ -33,7 +35,9 @@ func main() { if err != nil { panic(err) } - config := blocklist.NewConfigWithValidator(0, &parser.IPRangeValidator{}) + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{}) ips, err := blocklist.Get(url, pars, config) if err != nil { panic(err) -- 2.49.1 From a20d939e0dafc6c9593213a01ebbe862ce797531 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:05:18 +0500 Subject: [PATCH 14/32] Clarify documentation for NewConfigWithValidator limit and validator parameters --- blocklist.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/blocklist.go b/blocklist.go index cb06c37..7561e97 100644 --- a/blocklist.go +++ b/blocklist.go @@ -46,6 +46,8 @@ func NewConfig(limit uint) Config { } // NewConfigWithValidator creates a new Config with the specified validator. +// limit is the maximum number of items to process or validate. 0 means no limit. +// validator is the IP validator to use. func NewConfigWithValidator(limit uint, validator parser.IPValidator) Config { return Config{ Limit: limit, -- 2.49.1 From 68eb3a055d45eb0340bdaba3e4287b5e83f91bf1 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:05:35 +0500 Subject: [PATCH 15/32] Add Project Honeypot example for fetching and parsing blocklist data --- examples/honeypot.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 examples/honeypot.go diff --git a/examples/honeypot.go b/examples/honeypot.go new file mode 100644 index 0000000..9a43046 --- /dev/null +++ b/examples/honeypot.go @@ -0,0 +1,53 @@ +package main + +import ( + "encoding/xml" + "fmt" + "strings" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of bad IP addresses from the service https://www.projecthoneypot.org/list_of_ips.php + */ + +type rssItem struct { + Title string `xml:"title"` +} + +func main() { + url := "https://www.projecthoneypot.org/list_of_ips.php?t=d&rss=1" + pars, err := parser.NewRss(func(decoder *xml.Decoder, start xml.StartElement) (string, error) { + if start.Name.Local != "item" { + return "", nil + } + + var rssItem rssItem + if err := decoder.DecodeElement(&rssItem, &start); err != nil { + return "", fmt.Errorf("decode rss item: %w", err) + } + + if rssItem.Title == "" { + return "", nil + } + + fields := strings.Split(rssItem.Title, "|") + if len(fields) != 2 { + return "", nil + } + return strings.TrimSpace(fields[0]), nil + }) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From 17dbfac864c0189a312c8c4ce0675bc4caa288f6 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:41:28 +0500 Subject: [PATCH 16/32] Clarify example comments by generalizing "bad IP addresses" to "IP addresses" --- examples/dshield.go | 2 +- examples/honeypot.go | 2 +- examples/spamhaus.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/dshield.go b/examples/dshield.go index f0f12ff..f94d87a 100644 --- a/examples/dshield.go +++ b/examples/dshield.go @@ -8,7 +8,7 @@ import ( ) /** - * An example of how to get a list of bad IP addresses from the service https://dshield.org/ + * An example of how to get a list of IP addresses from a service https://dshield.org/ */ func main() { diff --git a/examples/honeypot.go b/examples/honeypot.go index 9a43046..631c1df 100644 --- a/examples/honeypot.go +++ b/examples/honeypot.go @@ -10,7 +10,7 @@ import ( ) /** - * An example of how to get a list of bad IP addresses from the service https://www.projecthoneypot.org/list_of_ips.php + * An example of how to get a list of IP addresses from a service https://www.projecthoneypot.org/list_of_ips.php */ type rssItem struct { diff --git a/examples/spamhaus.go b/examples/spamhaus.go index 76bab7d..7b96bc8 100644 --- a/examples/spamhaus.go +++ b/examples/spamhaus.go @@ -9,7 +9,7 @@ import ( ) /** - * An example of how to get a list of bad IP addresses from the service https://www.spamhaus.org/blocklists/do-not-route-or-peer/ + * An example of how to get a list of IP addresses from a service https://www.spamhaus.org/blocklists/do-not-route-or-peer/ */ type lineJson struct { -- 2.49.1 From 062cb25551e9732b5a2227ac31ce855d56f11fb9 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:43:14 +0500 Subject: [PATCH 17/32] Add Tor example for fetching and parsing IP blocklist data --- examples/tor.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 examples/tor.go diff --git a/examples/tor.go b/examples/tor.go new file mode 100644 index 0000000..41f0078 --- /dev/null +++ b/examples/tor.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of IP addresses from a service https://check.torproject.org/torbulkexitlist + */ + +func main() { + url := "https://check.torproject.org/torbulkexitlist" + extract := parser.NewDefaultTextExtract(0, " ") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From f295c402af725f6939ca04c19dab81e09cb394de Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:45:31 +0500 Subject: [PATCH 18/32] Add README with installation steps, usage examples, and license information --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..1b0c036 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# blocklist + +Пакет Go для получения списков плохих IP адресов от разных сервисов. + +## Установка +```sh +go get git.kor-elf.net/kor-elf-shield/blocklist +``` + +## Примеры использования +- [Пример получения списка IP адресов от Spamhaus](/examples/spamhaus.go) +- [Пример получения списка IP адресов от DShield](/examples/dshield.go) +- [Пример получения списка IP адресов от honeypot](/examples/honeypot.go) +- [Пример получения списка IP адресов от Tor](/examples/tor.go) + +## Лицензия + +[MIT](https://git.kor-elf.net/kor-elf-shield/blocklist/src/branch/main/LICENSE) \ No newline at end of file -- 2.49.1 From e4b0ed6668fd9ba07fcd0fafc4e9eb726172627c Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:47:49 +0500 Subject: [PATCH 19/32] Clarify README description by generalizing "bad IP addresses" to "IP addresses" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b0c036..483925e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # blocklist -Пакет Go для получения списков плохих IP адресов от разных сервисов. +Пакет Go для получения IP-адресов от различных сервисов. ## Установка ```sh -- 2.49.1 From 364e6576411173f4364c35f24688ca01fa265d32 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 16:48:58 +0500 Subject: [PATCH 20/32] =?UTF-8?q?Fix=20typo=20in=20README=20by=20removing?= =?UTF-8?q?=20unnecessary=20hyphen=20in=20"IP=20=D0=B0=D0=B4=D1=80=D0=B5?= =?UTF-8?q?=D1=81=D0=BE=D0=B2"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 483925e..b660842 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # blocklist -Пакет Go для получения IP-адресов от различных сервисов. +Пакет Go для получения IP адресов от различных сервисов. ## Установка ```sh -- 2.49.1 From 172c32237bcb8ab666c7b0f0568f15bddf8ae433 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 17:40:42 +0500 Subject: [PATCH 21/32] Normalize capitalization of "HONEYPOT" in README example link. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b660842..c3db897 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ go get git.kor-elf.net/kor-elf-shield/blocklist ## Примеры использования - [Пример получения списка IP адресов от Spamhaus](/examples/spamhaus.go) - [Пример получения списка IP адресов от DShield](/examples/dshield.go) -- [Пример получения списка IP адресов от honeypot](/examples/honeypot.go) +- [Пример получения списка IP адресов от HONEYPOT](/examples/honeypot.go) - [Пример получения списка IP адресов от Tor](/examples/tor.go) ## Лицензия -- 2.49.1 From dd4e5619c75fcf1a69df5f5bc695a904cbf276e2 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 17:47:14 +0500 Subject: [PATCH 22/32] Add CIArmy example for fetching and parsing IP blocklist data --- examples/ciarmy.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 examples/ciarmy.go diff --git a/examples/ciarmy.go b/examples/ciarmy.go new file mode 100644 index 0000000..db1c06b --- /dev/null +++ b/examples/ciarmy.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of IP addresses from a service https://www.ciarmy.com/#list + */ + +func main() { + url := "https://www.ciarmy.com/list/ci-badguys.txt" + extract := parser.NewDefaultTextExtract(0, " ") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From 5b8974101e9d2379fc381a4e45b8fa044113b68b Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 17:48:00 +0500 Subject: [PATCH 23/32] Add CIArmy example link to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c3db897..231f799 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ go get git.kor-elf.net/kor-elf-shield/blocklist - [Пример получения списка IP адресов от DShield](/examples/dshield.go) - [Пример получения списка IP адресов от HONEYPOT](/examples/honeypot.go) - [Пример получения списка IP адресов от Tor](/examples/tor.go) +- [Пример получения списка IP адресов от CIARMY](/examples/ciarmy.go) ## Лицензия -- 2.49.1 From aeb855c0b8d1157ec5168a51fc93e8c47dd2d5db Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 18:07:40 +0500 Subject: [PATCH 24/32] Add BruteforceBlocker example for fetching and parsing IP blocklist data --- examples/bruteforceblocker.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 examples/bruteforceblocker.go diff --git a/examples/bruteforceblocker.go b/examples/bruteforceblocker.go new file mode 100644 index 0000000..51a543b --- /dev/null +++ b/examples/bruteforceblocker.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of IP addresses from a service https://danger.rulez.sk/index.php/bruteforceblocker/ + */ + +func main() { + url := "https://danger.rulez.sk/projects/bruteforceblocker/blist.php" + extract := parser.NewDefaultTextExtract(0, "\t") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From e18859e78f9ed62e0edf3b4636717a5acffe5108 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 18:18:05 +0500 Subject: [PATCH 25/32] Add a link to Daniel Gerzo's BruteforceBlocker list usage example to your README file. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 231f799..4038f59 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ go get git.kor-elf.net/kor-elf-shield/blocklist - [Пример получения списка IP адресов от HONEYPOT](/examples/honeypot.go) - [Пример получения списка IP адресов от Tor](/examples/tor.go) - [Пример получения списка IP адресов от CIARMY](/examples/ciarmy.go) +- [Пример получения списка IP адресов от Daniel Gerzo (BruteforceBlocker)](/examples/bruteforceblocker.go) ## Лицензия -- 2.49.1 From 790073aa951f7fb157a911568398c5d61860cf0f Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 18:58:19 +0500 Subject: [PATCH 26/32] Add Blocklist.de example for fetching and parsing IP blocklist data --- examples/blocklist.go | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 examples/blocklist.go diff --git a/examples/blocklist.go b/examples/blocklist.go new file mode 100644 index 0000000..a49df63 --- /dev/null +++ b/examples/blocklist.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of IP addresses from a service https://www.blocklist.de/en/export.html + */ + +func main() { + // Getting a list of IP addresses that were entered in the last hour + // time=seconds + url := "https://api.blocklist.de/getlast.php?time=3600" + extract := parser.NewDefaultTextExtract(0, "\t") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) + + /* + // This second list retrieves all the IP addresses added in the last 48 hours and is usually a + // very large list (over 10000 entries), so be sure that you have the resources available to use it + url := "http://lists.blocklist.de/lists/all.txt" + extract := parser.NewDefaultTextExtract(0, "\t") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) + */ +} -- 2.49.1 From ee0e36b40dc9e0ebac6ca169e27a29fec10a40e9 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 18:59:25 +0500 Subject: [PATCH 27/32] Add Blocklist.de example link to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4038f59..4a829e7 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ go get git.kor-elf.net/kor-elf-shield/blocklist - [Пример получения списка IP адресов от Tor](/examples/tor.go) - [Пример получения списка IP адресов от CIARMY](/examples/ciarmy.go) - [Пример получения списка IP адресов от Daniel Gerzo (BruteforceBlocker)](/examples/bruteforceblocker.go) +- [Пример получения списка IP адресов от Blocklist.de](/examples/blocklist.go) ## Лицензия -- 2.49.1 From faedb345df278826330ea83be0eef7d8ce061cfb Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 19:05:16 +0500 Subject: [PATCH 28/32] Add Greensnow example for fetching and parsing IP blocklist data --- examples/greensnow.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 examples/greensnow.go diff --git a/examples/greensnow.go b/examples/greensnow.go new file mode 100644 index 0000000..100512f --- /dev/null +++ b/examples/greensnow.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of IP addresses from a service https://greensnow.co/ + */ + +func main() { + url := "https://blocklist.greensnow.co/greensnow.txt" + extract := parser.NewDefaultTextExtract(0, " ") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + ips, err := blocklist.Get(url, pars, config) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From e4fc07871bbb87a822f11b2400ff5ce140b4f7ec Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 19:06:26 +0500 Subject: [PATCH 29/32] Add GreenSnow example link to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4a829e7..b82f1e9 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ go get git.kor-elf.net/kor-elf-shield/blocklist - [Пример получения списка IP адресов от CIARMY](/examples/ciarmy.go) - [Пример получения списка IP адресов от Daniel Gerzo (BruteforceBlocker)](/examples/bruteforceblocker.go) - [Пример получения списка IP адресов от Blocklist.de](/examples/blocklist.go) +- [Пример получения списка IP адресов от GreenSnow](/examples/greensnow.go) ## Лицензия -- 2.49.1 From 1dc7a8b22c0f3aad7825a38f0022598a3da2f807 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 20:27:31 +0500 Subject: [PATCH 30/32] Add ZIP archive handling support for fetching and parsing blocklist data --- blocklist.go | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/blocklist.go b/blocklist.go index 7561e97..712029c 100644 --- a/blocklist.go +++ b/blocklist.go @@ -1,10 +1,14 @@ package blocklist import ( + "archive/zip" + "bytes" "context" "fmt" + "io" "net/http" "net/url" + "strings" "time" "git.kor-elf.net/kor-elf-shield/blocklist/parser" @@ -16,6 +20,12 @@ const ( // requestTimeout defines the maximum duration for request operations before timing out. requestTimeout = 20 * time.Second + + // maxDownloadSize defines the maximum allowed size of the downloaded file in bytes. + maxDownloadSize int64 = 20 << 20 // 20 MiB + + // maxArchiveFileSize defines the maximum allowed size of the extracted file from ZIP in bytes. + maxArchiveFileSize uint64 = 50 << 20 // 50 MiB ) // Config defines the configuration for the blocklist. @@ -34,6 +44,17 @@ type Config struct { RequestTimeout time.Duration } +type ConfigZip struct { + // Config is the configuration for the blocklist. + Config Config + + // MaxDownloadSize defines the maximum allowed size of the downloaded file in bytes. + MaxDownloadSize int64 + + // MaxArchiveFileSize defines the maximum allowed size of the extracted file from ZIP in bytes. + MaxArchiveFileSize uint64 +} + // NewConfig creates a new Config with default values. // limit is the maximum number of items to process or validate. 0 means no limit. func NewConfig(limit uint) Config { @@ -57,6 +78,14 @@ func NewConfigWithValidator(limit uint, validator parser.IPValidator) Config { } } +func NewConfigZip(c Config) ConfigZip { + return ConfigZip{ + Config: c, + MaxDownloadSize: maxDownloadSize, + MaxArchiveFileSize: maxArchiveFileSize, + } +} + // Get fetches data from the given URL, parses the response using the provided parser, and applies the given configuration. // It returns the parsed IPs and any errors that occurred during the process. func Get(fileUrl string, parser parser.Parser, c Config) (parser.IPs, error) { @@ -94,3 +123,126 @@ func Get(fileUrl string, parser parser.Parser, c Config) (parser.IPs, error) { return parser.Parse(res.Body, c.Validator, c.Limit) } + +// GetZip fetches data from the given URL, parses the response using the provided parser, and applies the given configuration. +// It returns the parsed IPs and any errors that occurred during the process. +func GetZip(fileUrl string, parser parser.Parser, c ConfigZip) (parser.IPs, error) { + parsedURL, err := url.Parse(fileUrl) + if err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme) + } + + ctx, cancel := context.WithTimeout(context.Background(), c.Config.ContextTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + client := &http.Client{ + Timeout: c.Config.RequestTimeout, + } + + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer func() { + _ = res.Body.Close() + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + + if c.MaxDownloadSize > 0 && res.ContentLength > c.MaxDownloadSize { + return nil, fmt.Errorf("downloaded file is too large: content-length %d exceeds limit %d", res.ContentLength, c.MaxDownloadSize) + } + + reader := res.Body + if c.MaxDownloadSize > 0 { + reader = io.NopCloser(io.LimitReader(res.Body, c.MaxDownloadSize+1)) + } + + body, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + if c.MaxDownloadSize > 0 && int64(len(body)) > c.MaxDownloadSize { + return nil, fmt.Errorf("downloaded file exceeds limit %d bytes", c.MaxDownloadSize) + } + + if !isZip(body) { + return nil, fmt.Errorf("invalid zip archive") + } + + return parseZip(body, parser, c) +} + +func isZip(body []byte) bool { + return len(body) >= 4 && + body[0] == 'P' && + body[1] == 'K' && + body[2] == 0x03 && + body[3] == 0x04 +} + +func parseZip(body []byte, p parser.Parser, c ConfigZip) (parser.IPs, error) { + reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return nil, fmt.Errorf("open zip archive: %w", err) + } + + file := findArchiveFile(reader.File) + if file == nil { + return nil, fmt.Errorf("zip archive does not contain a supported file") + } + + if c.MaxArchiveFileSize > 0 && file.UncompressedSize64 > c.MaxArchiveFileSize { + return nil, fmt.Errorf("file %q in zip is too large: %d exceeds limit %d", file.Name, file.UncompressedSize64, c.MaxArchiveFileSize) + } + + rc, err := file.Open() + if err != nil { + return nil, fmt.Errorf("open file %q from zip: %w", file.Name, err) + } + defer func() { + _ = rc.Close() + }() + + var zipReader io.Reader = rc + if c.MaxArchiveFileSize > 0 { + zipReader = io.LimitReader(rc, int64(c.MaxArchiveFileSize)+1) + } + + return p.Parse(zipReader, c.Config.Validator, c.Config.Limit) +} + +func findArchiveFile(files []*zip.File) *zip.File { + var fallback *zip.File + + for _, file := range files { + if file.FileInfo().IsDir() { + continue + } + + if fallback == nil { + fallback = file + } + + name := strings.ToLower(file.Name) + if strings.HasSuffix(name, ".txt") || + strings.HasSuffix(name, ".json") || + strings.HasSuffix(name, ".xml") || + strings.HasSuffix(name, ".rss") { + return file + } + } + + return fallback +} -- 2.49.1 From 17115d97c62d7b3099da827f84ca4dfc6e7d111e Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 20:27:58 +0500 Subject: [PATCH 31/32] Add StopForumSpam example for fetching and parsing IP blocklist data --- examples/stopforumspam.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 examples/stopforumspam.go diff --git a/examples/stopforumspam.go b/examples/stopforumspam.go new file mode 100644 index 0000000..f5118e7 --- /dev/null +++ b/examples/stopforumspam.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + + "git.kor-elf.net/kor-elf-shield/blocklist" + "git.kor-elf.net/kor-elf-shield/blocklist/parser" +) + +/** + * An example of how to get a list of IP addresses from a service https://www.stopforumspam.com/downloads + */ + +func main() { + url := "https://www.stopforumspam.com/downloads/listed_ip_1.zip" + //url := "https://www.stopforumspam.com/downloads/listed_ip_1_ipv6.zip" + extract := parser.NewDefaultTextExtract(0, " ") + pars, err := parser.NewText(extract) + if err != nil { + panic(err) + } + // limit 0 - no limit + limit := uint(0) + config := blocklist.NewConfig(limit) + configZip := blocklist.NewConfigZip(config) + ips, err := blocklist.GetZip(url, pars, configZip) + if err != nil { + panic(err) + } + fmt.Println(ips) +} -- 2.49.1 From cfbf4ce50495ef05cb72fe615da969ded4248652 Mon Sep 17 00:00:00 2001 From: Leonid Nikitin Date: Sun, 15 Mar 2026 20:28:36 +0500 Subject: [PATCH 32/32] Add StopForumSpam example link to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b82f1e9..4cca62a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ go get git.kor-elf.net/kor-elf-shield/blocklist - [Пример получения списка IP адресов от Daniel Gerzo (BruteforceBlocker)](/examples/bruteforceblocker.go) - [Пример получения списка IP адресов от Blocklist.de](/examples/blocklist.go) - [Пример получения списка IP адресов от GreenSnow](/examples/greensnow.go) +- [Пример получения списка IP адресов от StopForumSpam](/examples/stopforumspam.go) ## Лицензия -- 2.49.1