Merge pull request 'v1.0.0' (#1) from develop into main

Reviewed-on: #1

blocklist.go

@@ -66,17 +66,6 @@ func NewConfig(limit uint) Config {
 	}
 }
 
-func NewConfigWithExclusionChecker(limit uint, exclusionChecker parser.ExclusionChecker) Config {
-	return Config{
-		Limit: limit,
-		Validator: &parser.DefaultIPValidator{
-			ExclusionChecker: exclusionChecker,
-		},
-		ContextTimeout: contextTimeout,
-		RequestTimeout: requestTimeout,
-	}
-}
-
 // NewConfigWithValidator creates a new Config with the specified validator.
 // limit is the maximum number of items to process or validate. 0 means no limit.
 // validator is the IP validator to use.
@@ -100,12 +89,29 @@ func NewConfigZip(c Config) ConfigZip {
 // Get fetches data from the given URL, parses the response using the provided parser, and applies the given configuration.
 // It returns the parsed IPs and any errors that occurred during the process.
 func Get(fileUrl string, parser parser.Parser, c Config) (parser.IPs, error) {
+	parsedURL, err := url.Parse(fileUrl)
+	if err != nil {
+		return nil, fmt.Errorf("invalid url: %w", err)
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme)
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), c.ContextTimeout)
 	defer cancel()
 
-	res, err := fetch(fileUrl, ctx, c.RequestTimeout)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+
+	client := &http.Client{
+		Timeout: c.RequestTimeout,
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("request failed: %w", err)
 	}
 	defer func() {
 		_ = res.Body.Close()
@@ -118,36 +124,32 @@ func Get(fileUrl string, parser parser.Parser, c Config) (parser.IPs, error) {
 	return parser.Parse(res.Body, c.Validator, c.Limit)
 }
 
-// GetSeparatedIPs fetches data from the given URL, parses the response using the provided parser, and applies the given configuration.
-// It returns the parsed IPs and any errors that occurred during the process.
-func GetSeparatedIPs(fileUrl string, parser parser.Parser, c Config) (ipV4 parser.IPs, ipV6 parser.IPs, err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), c.ContextTimeout)
-	defer cancel()
-
-	res, err := fetch(fileUrl, ctx, c.RequestTimeout)
-	if err != nil {
-		return nil, nil, err
-	}
-	defer func() {
-		_ = res.Body.Close()
-	}()
-
-	if res.StatusCode != http.StatusOK {
-		return nil, nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
-	}
-
-	return parser.ParseIPsByVersion(res.Body, c.Validator, c.Limit)
-}
-
 // GetZip fetches data from the given URL, parses the response using the provided parser, and applies the given configuration.
 // It returns the parsed IPs and any errors that occurred during the process.
 func GetZip(fileUrl string, parser parser.Parser, c ConfigZip) (parser.IPs, error) {
+	parsedURL, err := url.Parse(fileUrl)
+	if err != nil {
+		return nil, fmt.Errorf("invalid url: %w", err)
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme)
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), c.Config.ContextTimeout)
 	defer cancel()
 
-	res, err := fetch(fileUrl, ctx, c.Config.RequestTimeout)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+
+	client := &http.Client{
+		Timeout: c.Config.RequestTimeout,
+	}
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("request failed: %w", err)
 	}
 	defer func() {
 		_ = res.Body.Close()
@@ -182,49 +184,6 @@ func GetZip(fileUrl string, parser parser.Parser, c ConfigZip) (parser.IPs, erro
 	return parseZip(body, parser, c)
 }
 
-// GetZipSeparatedIPs fetches data from the given URL, parses the response using the provided parser, and applies the given configuration.
-// It returns the parsed IPs and any errors that occurred during the process.
-func GetZipSeparatedIPs(fileUrl string, parser parser.Parser, c ConfigZip) (ipV4 parser.IPs, ipV6 parser.IPs, err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), c.Config.ContextTimeout)
-	defer cancel()
-
-	res, err := fetch(fileUrl, ctx, c.Config.RequestTimeout)
-	if err != nil {
-		return nil, nil, err
-	}
-	defer func() {
-		_ = res.Body.Close()
-	}()
-
-	if res.StatusCode != http.StatusOK {
-		return nil, nil, fmt.Errorf("unexpected status code: %d", res.StatusCode)
-	}
-
-	if c.MaxDownloadSize > 0 && res.ContentLength > c.MaxDownloadSize {
-		return nil, nil, fmt.Errorf("downloaded file is too large: content-length %d exceeds limit %d", res.ContentLength, c.MaxDownloadSize)
-	}
-
-	reader := res.Body
-	if c.MaxDownloadSize > 0 {
-		reader = io.NopCloser(io.LimitReader(res.Body, c.MaxDownloadSize+1))
-	}
-
-	body, err := io.ReadAll(reader)
-	if err != nil {
-		return nil, nil, fmt.Errorf("read response body: %w", err)
-	}
-
-	if c.MaxDownloadSize > 0 && int64(len(body)) > c.MaxDownloadSize {
-		return nil, nil, fmt.Errorf("downloaded file exceeds limit %d bytes", c.MaxDownloadSize)
-	}
-
-	if !isZip(body) {
-		return nil, nil, fmt.Errorf("invalid zip archive")
-	}
-
-	return parseZipSeparatedIPs(body, parser, c)
-}
-
 func isZip(body []byte) bool {
 	return len(body) >= 4 &&
 		body[0] == 'P' &&
@@ -264,37 +223,6 @@ func parseZip(body []byte, p parser.Parser, c ConfigZip) (parser.IPs, error) {
 	return p.Parse(zipReader, c.Config.Validator, c.Config.Limit)
 }
 
-func parseZipSeparatedIPs(body []byte, p parser.Parser, c ConfigZip) (ipV4 parser.IPs, ipV6 parser.IPs, err error) {
-	reader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
-	if err != nil {
-		return nil, nil, fmt.Errorf("open zip archive: %w", err)
-	}
-
-	file := findArchiveFile(reader.File)
-	if file == nil {
-		return nil, nil, fmt.Errorf("zip archive does not contain a supported file")
-	}
-
-	if c.MaxArchiveFileSize > 0 && file.UncompressedSize64 > c.MaxArchiveFileSize {
-		return nil, nil, fmt.Errorf("file %q in zip is too large: %d exceeds limit %d", file.Name, file.UncompressedSize64, c.MaxArchiveFileSize)
-	}
-
-	rc, err := file.Open()
-	if err != nil {
-		return nil, nil, fmt.Errorf("open file %q from zip: %w", file.Name, err)
-	}
-	defer func() {
-		_ = rc.Close()
-	}()
-
-	var zipReader io.Reader = rc
-	if c.MaxArchiveFileSize > 0 {
-		zipReader = io.LimitReader(rc, int64(c.MaxArchiveFileSize)+1)
-	}
-
-	return p.ParseIPsByVersion(zipReader, c.Config.Validator, c.Config.Limit)
-}
-
 func findArchiveFile(files []*zip.File) *zip.File {
 	var fallback *zip.File
 
@@ -318,28 +246,3 @@ func findArchiveFile(files []*zip.File) *zip.File {
 
 	return fallback
 }
-
-func fetch(fileUrl string, ctx context.Context, requestTimeout time.Duration) (*http.Response, error) {
-	parsedURL, err := url.Parse(fileUrl)
-	if err != nil {
-		return nil, fmt.Errorf("invalid url: %w", err)
-	}
-	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
-		return nil, fmt.Errorf("invalid url scheme: %s", parsedURL.Scheme)
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
-	if err != nil {
-		return nil, fmt.Errorf("create request: %w", err)
-	}
-
-	client := &http.Client{
-		Timeout: requestTimeout,
-	}
-
-	res, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("request failed: %w", err)
-	}
-	return res, nil
-}

examples/blocklist.go

@@ -23,36 +23,12 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
 
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
-
 	/*
 		// This second list retrieves all the IP addresses added in the last 48 hours and is usually a
 		// very large list (over 10000 entries), so be sure that you have the resources available to use it
@@ -65,34 +41,10 @@ func main() {
 		// limit 0 - no limit
 		limit := uint(0)
 		config := blocklist.NewConfig(limit)
-
-		// If you need to exclude an IP address or subnet
-		//excludeIPs := []string{
-		//	"172.18.0.2",
-		//	//"172.18.0.0/24",
-		//	//"172.18.0.0-172.18.0.255",
-		//}
-		//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-		//if err != nil {
-		//	panic(err)
-		//}
-		//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-		// Get IPv4 and IPv6 addresses in one list
 		ips, err := blocklist.Get(url, pars, config)
 		if err != nil {
 			panic(err)
 		}
 		fmt.Println(ips)
-
-		// Get IPv4 and IPv6 addresses in two lists
-		ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-		if err != nil {
-			panic(err)
-		}
-		fmt.Println("IPv4")
-		fmt.Println(ipsV4)
-		fmt.Println("IPv6")
-		fmt.Println(ipsV6)
 	*/
 }

examples/bruteforceblocker.go

@@ -21,33 +21,9 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

examples/ciarmy.go

@@ -21,33 +21,9 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

examples/dshield.go

@@ -21,36 +21,12 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
 
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
-
 	/*
 		// You can also get a range of IP addresses from this service (from to)
 		url := "https://www.dshield.org/block.txt"
@@ -62,36 +38,10 @@ func main() {
 		// limit 0 - no limit
 		limit := uint(0)
 		config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{})
-
-		// If you need to exclude an IP address or subnet
-		//excludeIPs := []string{
-		//	"172.18.0.2",
-		//	//"172.18.0.0/24",
-		//	//"172.18.0.0-172.18.0.255",
-		//}
-		//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-		//if err != nil {
-		//	panic(err)
-		//}
-		//config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{
-		//	ExclusionChecker: exclusionChecker,
-		//})
-
-		// Get IPv4 and IPv6 addresses in one list
 		ips, err := blocklist.Get(url, pars, config)
 		if err != nil {
 			panic(err)
 		}
 		fmt.Println(ips)
-
-		// Get IPv4 and IPv6 addresses in two lists
-		ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-		if err != nil {
-			panic(err)
-		}
-		fmt.Println("IPv4")
-		fmt.Println(ipsV4)
-		fmt.Println("IPv6")
-		fmt.Println(ipsV6)
 	*/
 }

examples/greensnow.go

@@ -21,33 +21,9 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

examples/honeypot.go

@@ -45,33 +45,9 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

examples/spamhaus.go

@@ -32,33 +32,9 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

examples/stopforumspam.go

@@ -22,35 +22,10 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
 	configZip := blocklist.NewConfigZip(config)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.GetZip(url, pars, configZip)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetZipSeparatedIPs(url, pars, configZip)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

examples/tor.go

@@ -21,33 +21,9 @@ func main() {
 	// limit 0 - no limit
 	limit := uint(0)
 	config := blocklist.NewConfig(limit)
-
-	// If you need to exclude an IP address or subnet
-	//excludeIPs := []string{
-	//	"172.18.0.2",
-	//	//"172.18.0.0/24",
-	//	//"172.18.0.0-172.18.0.255",
-	//}
-	//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
-
-	// Get IPv4 and IPv6 addresses in one list
 	ips, err := blocklist.Get(url, pars, config)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(ips)
-
-	// Get IPv4 and IPv6 addresses in two lists
-	ipsV4, ipsV6, err := blocklist.GetSeparatedIPs(url, pars, config)
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println("IPv4")
-	fmt.Println(ipsV4)
-	fmt.Println("IPv6")
-	fmt.Println(ipsV6)
 }

parser/exclusion_checker.go

@@ -1,294 +0,0 @@
-package parser
-
-import (
-	"encoding/binary"
-	"fmt"
-	"math/big"
-	"net/netip"
-	"strings"
-)
-
-// ExclusionChecker is an interface for checking if an IP is excluded.
-type ExclusionChecker interface {
-	IsExcluded(ip string) (excluded bool, ips []string, err error)
-}
-
-type ipRange struct {
-	start netip.Addr
-	end   netip.Addr
-	isV4  bool
-}
-
-type exclusionChecker struct {
-	ipRanges []ipRange
-}
-
-// NewExclusionChecker creates a new exclusion checker.
-func NewExclusionChecker(ips []string) (ExclusionChecker, error) {
-	ipRanges := make([]ipRange, 0, len(ips))
-
-	for _, ip := range ips {
-		ipRange, err := parseSpec(ip)
-		if err != nil {
-			return nil, fmt.Errorf("invalid exclusion IP: %s", ip)
-		}
-
-		ipRanges = append(ipRanges, ipRange)
-	}
-
-	return &exclusionChecker{
-		ipRanges: ipRanges,
-	}, nil
-}
-
-// IsExcluded checks if the given IP is excluded.
-func (e *exclusionChecker) IsExcluded(ip string) (excluded bool, ips []string, err error) {
-	inIPRange, err := parseSpec(ip)
-	if err != nil {
-		return false, nil, err
-	}
-
-	pending := []ipRange{inIPRange}
-	matchedAny := false
-
-	for _, baseRange := range e.ipRanges {
-		// Different IP families do not overlap.
-		if baseRange.isV4 != inIPRange.isV4 {
-			continue
-		}
-
-		nextPending := make([]ipRange, 0, len(pending))
-		for _, candidate := range pending {
-			matched, remains := subtractRange(candidate, baseRange)
-			if matched {
-				matchedAny = true
-			}
-			nextPending = append(nextPending, remains...)
-		}
-
-		pending = nextPending
-		if len(pending) == 0 {
-			// Completely excluded the range.
-			return true, []string{}, nil
-		}
-	}
-
-	if !matchedAny {
-		return false, nil, nil
-	}
-
-	out := make([]string, 0, len(pending))
-	for _, r := range pending {
-		out = append(out, formatRange(r))
-	}
-	return true, out, nil
-}
-
-func parseSpec(s string) (ipRange, error) {
-	s = strings.TrimSpace(s)
-	if s == "" {
-		return ipRange{}, fmt.Errorf("empty value")
-	}
-
-	// range: a-b
-	if strings.Contains(s, "-") {
-		parts := strings.Split(s, "-")
-		if len(parts) != 2 {
-			return ipRange{}, fmt.Errorf("invalid range format")
-		}
-		a, err := parseAddr(strings.TrimSpace(parts[0]))
-		if err != nil {
-			return ipRange{}, err
-		}
-		b, err := parseAddr(strings.TrimSpace(parts[1]))
-		if err != nil {
-			return ipRange{}, err
-		}
-		if a.Is4() != b.Is4() {
-			return ipRange{}, fmt.Errorf("mixed ip families in range")
-		}
-		if compareAddr(a, b) > 0 {
-			return ipRange{}, fmt.Errorf("range start > end")
-		}
-		return ipRange{start: a, end: b, isV4: a.Is4()}, nil
-	}
-
-	// cidr
-	if strings.Contains(s, "/") {
-		p, err := netip.ParsePrefix(s)
-		if err != nil {
-			return ipRange{}, fmt.Errorf("invalid cidr: %w", err)
-		}
-		p = p.Masked()
-		start := p.Addr()
-
-		var end netip.Addr
-		if start.Is4() {
-			u := ip4ToU32(start)
-			hostBits := uint32(32 - p.Bits())
-			var mask uint32
-			if hostBits == 32 {
-				mask = ^uint32(0)
-			} else {
-				mask = (uint32(1) << hostBits) - 1
-			}
-			end = u32ToIP4(u | mask)
-			return ipRange{start: start, end: end, isV4: true}, nil
-		}
-
-		u := ip16ToBig(start)
-		hostBits := 128 - p.Bits()
-		ones := new(big.Int).Lsh(big.NewInt(1), uint(hostBits))
-		ones.Sub(ones, big.NewInt(1))
-		u.Or(u, ones)
-		end = bigToIP16(u)
-		return ipRange{start: start, end: end, isV4: false}, nil
-	}
-
-	// single ip
-	a, err := parseAddr(s)
-	if err != nil {
-		return ipRange{}, err
-	}
-	return ipRange{start: a, end: a, isV4: a.Is4()}, nil
-}
-
-func parseAddr(s string) (netip.Addr, error) {
-	a, err := netip.ParseAddr(s)
-	if err != nil {
-		return netip.Addr{}, fmt.Errorf("invalid ip: %w", err)
-	}
-	return a.Unmap(), nil // important for ::ffff:1.2.3.4
-}
-
-func compareAddr(a, b netip.Addr) int {
-	if a.Is4() && b.Is4() {
-		ua := ip4ToU32(a)
-		ub := ip4ToU32(b)
-		switch {
-		case ua < ub:
-			return -1
-		case ua > ub:
-			return 1
-		default:
-			return 0
-		}
-	}
-	aa := a.As16()
-	bb := b.As16()
-	for i := 0; i < 16; i++ {
-		if aa[i] < bb[i] {
-			return -1
-		}
-		if aa[i] > bb[i] {
-			return 1
-		}
-	}
-	return 0
-}
-
-func ip4ToU32(a netip.Addr) uint32 {
-	v := a.As4()
-	return binary.BigEndian.Uint32(v[:])
-}
-
-func u32ToIP4(u uint32) netip.Addr {
-	var b [4]byte
-	binary.BigEndian.PutUint32(b[:], u)
-	return netip.AddrFrom4(b)
-}
-
-func ip16ToBig(a netip.Addr) *big.Int {
-	v := a.As16()
-	return new(big.Int).SetBytes(v[:])
-}
-
-func bigToIP16(x *big.Int) netip.Addr {
-	var b [16]byte
-	raw := x.Bytes()
-	copy(b[16-len(raw):], raw)
-	return netip.AddrFrom16(b)
-}
-
-func subtractRange(incoming, base ipRange) (bool, []ipRange) {
-	// No intersection
-	if compareAddr(base.end, incoming.start) < 0 || compareAddr(base.start, incoming.end) > 0 {
-		return false, []ipRange{incoming}
-	}
-
-	// Full incoming coverage
-	if compareAddr(base.start, incoming.start) <= 0 && compareAddr(base.end, incoming.end) >= 0 {
-		return true, nil
-	}
-
-	remains := make([]ipRange, 0, 2)
-
-	// Left side
-	if compareAddr(base.start, incoming.start) > 0 {
-		leftEnd, ok := prevIP(base.start)
-		if ok && compareAddr(incoming.start, leftEnd) <= 0 {
-			remains = append(remains, ipRange{
-				start: incoming.start,
-				end:   leftEnd,
-				isV4:  incoming.isV4,
-			})
-		}
-	}
-
-	// Right side
-	if compareAddr(base.end, incoming.end) < 0 {
-		rightStart, ok := nextIP(base.end)
-		if ok && compareAddr(rightStart, incoming.end) <= 0 {
-			remains = append(remains, ipRange{
-				start: rightStart,
-				end:   incoming.end,
-				isV4:  incoming.isV4,
-			})
-		}
-	}
-
-	return true, remains
-}
-
-func nextIP(a netip.Addr) (netip.Addr, bool) {
-	if a.Is4() {
-		u := ip4ToU32(a)
-		if u == ^uint32(0) {
-			return netip.Addr{}, false
-		}
-		return u32ToIP4(u + 1), true
-	}
-
-	u := ip16ToBig(a)
-	max := new(big.Int).Lsh(big.NewInt(1), 128)
-	max.Sub(max, big.NewInt(1))
-	if u.Cmp(max) == 0 {
-		return netip.Addr{}, false
-	}
-	u.Add(u, big.NewInt(1))
-	return bigToIP16(u), true
-}
-
-func prevIP(a netip.Addr) (netip.Addr, bool) {
-	if a.Is4() {
-		u := ip4ToU32(a)
-		if u == 0 {
-			return netip.Addr{}, false
-		}
-		return u32ToIP4(u - 1), true
-	}
-
-	u := ip16ToBig(a)
-	if u.Sign() == 0 {
-		return netip.Addr{}, false
-	}
-	u.Sub(u, big.NewInt(1))
-	return bigToIP16(u), true
-}
-
-func formatRange(r ipRange) string {
-	if r.start == r.end {
-		return r.start.String()
-	}
-	return r.start.String() + "-" + r.end.String()
-}

parser/json_lines.go

@@ -55,10 +55,7 @@ func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uin
 			continue
 		}
 
-		ips, err = appendIfNotExcluded(ip, ips, validator)
+		ips = append(ips, ip)
-		if err != nil {
-			return nil, err
-		}
 		if limit > 0 && uint(len(ips)) >= limit {
 			break
 		}
@@ -66,56 +63,3 @@ func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uin
 
 	return ips, nil
 }
-
-// ParseIPsByVersion parses the JSON Lines data from the given reader
-// and returns a slice of IP addresses for each IP version.
-// It also returns any errors that occurred during the process.
-func (p *jsonLinesParser) ParseIPsByVersion(body io.Reader, validator IPValidator, limit uint) (ipV4 IPs, ipV6 IPs, err error) {
-	decoder := json.NewDecoder(body)
-	ipV4 = make(IPs, 0)
-	ipV6 = make(IPs, 0)
-	for {
-		var item json.RawMessage
-		if err := decoder.Decode(&item); err != nil {
-			if err == io.EOF {
-				break
-			}
-			return nil, nil, fmt.Errorf("decode json item: %w", err)
-		}
-
-		if item == nil {
-			continue
-		}
-
-		ip, err := p.extract(item)
-		if err != nil {
-			return nil, nil, fmt.Errorf("extract ip: %w", err)
-		}
-
-		ip = strings.TrimSpace(ip)
-		isValid, ipVersion := validator.IsValidAndReturnVersion(ip)
-		if !isValid {
-			continue
-		}
-
-		if ipVersion == IPVersion4 {
-			ipV4, err = appendIfNotExcluded(ip, ipV4, validator)
-			if err != nil {
-				return nil, nil, err
-			}
-		} else if ipVersion == IPVersion6 {
-			ipV6, err = appendIfNotExcluded(ip, ipV6, validator)
-			if err != nil {
-				return nil, nil, err
-			}
-		} else {
-			continue
-		}
-
-		if limit > 0 && uint(len(ipV4))+uint(len(ipV6)) >= limit {
-			break
-		}
-	}
-
-	return ipV4, ipV6, nil
-}

parser/parser.go

@@ -11,37 +11,20 @@ import (
 type Parser interface {
 	// Parse reads the body and returns a slice of IP addresses.
 	Parse(body io.Reader, validator IPValidator, limit uint) (IPs, error)
-
-	ParseIPsByVersion(body io.Reader, validator IPValidator, limit uint) (ipV4 IPs, ipV6 IPs, err error)
 }
 
 // IPValidator interface defines the contract for validating IP addresses.
 type IPValidator interface {
 	// IsValid checks if the given IP address is valid.
 	IsValid(ip string) bool
-
-	// IsValidAndReturnVersion validates the given IP address and returns whether it is valid and its version (IPv4 or IPv6).
-	IsValidAndReturnVersion(ip string) (bool, IPVersion)
-
-	// IsExcluded determines if the given IP address is excluded based on certain criteria and returns related IP addresses.
-	IsExcluded(ip string) (excluded bool, ips []string, err error)
 }
 
-type IPVersion int
-
-const (
-	IPVersion4 IPVersion = iota
-	IPVersion6
-)
-
 // IPs is a slice of IP addresses.
 type IPs []string
 
 // DefaultIPValidator implements IPValidator interface.
 // It validates IP addresses by parsing them using net.ParseIP and net.ParseCIDR.
-type DefaultIPValidator struct {
+type DefaultIPValidator struct{}
-	ExclusionChecker ExclusionChecker
-}
 
 // IsValid checks if the given IP address is valid.
 // It returns true if the IP address is not a loopback address.
@@ -67,60 +50,9 @@ func (v *DefaultIPValidator) IsValid(value string) bool {
 	return false
 }
 
-// IsValidAndReturnVersion checks if the given IP address is valid and returns the IP version.
-// It returns true if the IP address is not a loopback address and the IP version is either IPv4 or IPv6.
-func (v *DefaultIPValidator) IsValidAndReturnVersion(value string) (bool, IPVersion) {
-	if value == "" {
-		return false, IPVersion4
-	}
-
-	if ip := net.ParseIP(value); ip != nil {
-		if ip.IsLoopback() {
-			return false, IPVersion4
-		}
-
-		if ip.To4() != nil {
-			return true, IPVersion4
-		}
-
-		if ip.To16() != nil {
-			return true, IPVersion6
-		}
-
-		return false, IPVersion4
-	}
-
-	if ip, _, err := net.ParseCIDR(value); err == nil {
-		if ip.IsLoopback() {
-			return false, IPVersion4
-		}
-
-		if ip.To4() != nil {
-			return true, IPVersion4
-		}
-
-		if ip.To16() != nil {
-			return true, IPVersion6
-		}
-
-		return false, IPVersion4
-	}
-
-	return false, IPVersion4
-}
-
-func (v *DefaultIPValidator) IsExcluded(ip string) (excluded bool, ips []string, err error) {
-	if v.ExclusionChecker == nil {
-		return false, nil, nil
-	}
-	return v.ExclusionChecker.IsExcluded(ip)
-}
-
 // IPRangeValidator implements IPValidator interface.
 // It validates IP ranges by parsing them using net.ParseIP and checking if the start and end IPs are in the same network.
-type IPRangeValidator struct {
+type IPRangeValidator struct{}
-	ExclusionChecker ExclusionChecker
-}
 
 // IsValid checks if the given IP range is valid.
 // It returns true if the start and end IPs are in the same network.
@@ -157,50 +89,3 @@ func (v *IPRangeValidator) IsValid(value string) bool {
 		return false
 	}
 }
-
-func (v *IPRangeValidator) IsValidAndReturnVersion(value string) (bool, IPVersion) {
-	if value == "" {
-		return false, IPVersion4
-	}
-
-	parts := strings.Split(value, "-")
-	if len(parts) != 2 {
-		return false, IPVersion4
-	}
-
-	start := net.ParseIP(strings.TrimSpace(parts[0]))
-	end := net.ParseIP(strings.TrimSpace(parts[1]))
-	if start == nil || end == nil {
-		return false, IPVersion4
-	}
-
-	start4 := start.To4()
-	end4 := end.To4()
-
-	switch {
-	case start4 != nil && end4 != nil:
-		if bytes.Compare(start4, end4) <= 0 {
-			return true, IPVersion4
-		}
-		return false, IPVersion4
-	case start4 == nil && end4 == nil:
-		start16 := start.To16()
-		end16 := end.To16()
-		if start16 == nil || end16 == nil {
-			return false, IPVersion4
-		}
-		if bytes.Compare(start16, end16) <= 0 {
-			return true, IPVersion6
-		}
-		return false, IPVersion4
-	default:
-		return false, IPVersion4
-	}
-}
-
-func (v *IPRangeValidator) IsExcluded(ip string) (excluded bool, ips []string, err error) {
-	if v.ExclusionChecker == nil {
-		return false, nil, nil
-	}
-	return v.ExclusionChecker.IsExcluded(ip)
-}

parser/pkg.go

@@ -1,16 +0,0 @@
-package parser
-
-func appendIfNotExcluded(ip string, ips []string, validator IPValidator) ([]string, error) {
-	if exclude, listIP, err := validator.IsExcluded(ip); err != nil {
-		return ips, err
-	} else if exclude {
-		if listIP == nil || len(listIP) == 0 {
-			return ips, nil
-		}
-		ips = append(ips, listIP...)
-		return ips, nil
-	}
-
-	ips = append(ips, ip)
-	return ips, nil
-}

parser/rss.go

@@ -58,10 +58,7 @@ func (p *rssParser) Parse(body io.Reader, validator IPValidator, limit uint) (IP
 			continue
 		}
 
-		ips, err = appendIfNotExcluded(ip, ips, validator)
+		ips = append(ips, ip)
-		if err != nil {
-			return nil, err
-		}
 		if limit > 0 && uint(len(ips)) >= limit {
 			break
 		}
@@ -69,59 +66,3 @@ func (p *rssParser) Parse(body io.Reader, validator IPValidator, limit uint) (IP
 
 	return ips, nil
 }
-
-// ParseIPsByVersion parses the RSS data from the given reader
-// and returns a slice of IP addresses for each IP version.
-// It also returns any errors that occurred during the process.
-func (p *rssParser) ParseIPsByVersion(body io.Reader, validator IPValidator, limit uint) (ipV4 IPs, ipV6 IPs, err error) {
-	decoder := xml.NewDecoder(body)
-	ipV4 = make(IPs, 0)
-	ipV6 = make(IPs, 0)
-
-	for {
-		token, err := decoder.Token()
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-
-			return nil, nil, fmt.Errorf("parse rss: %w", err)
-		}
-
-		start, ok := token.(xml.StartElement)
-		if !ok {
-			continue
-		}
-
-		ip, err := p.extract(decoder, start)
-		if err != nil {
-			return nil, nil, fmt.Errorf("extract rss ip: %w", err)
-		}
-
-		ip = strings.TrimSpace(ip)
-		isValid, ipVersion := validator.IsValidAndReturnVersion(ip)
-		if !isValid {
-			continue
-		}
-
-		if ipVersion == IPVersion4 {
-			ipV4, err = appendIfNotExcluded(ip, ipV4, validator)
-			if err != nil {
-				return nil, nil, err
-			}
-		} else if ipVersion == IPVersion6 {
-			ipV6, err = appendIfNotExcluded(ip, ipV6, validator)
-			if err != nil {
-				return nil, nil, err
-			}
-		} else {
-			continue
-		}
-
-		if limit > 0 && uint(len(ipV4))+uint(len(ipV6)) >= limit {
-			break
-		}
-	}
-
-	return ipV4, ipV6, nil
-}

parser/text.go

@@ -146,11 +146,7 @@ func (p *textParser) Parse(body io.Reader, validator IPValidator, limit uint) (I
 			continue
 		}
 
-		var err error
+		ips = append(ips, ip)
-		ips, err = appendIfNotExcluded(ip, ips, validator)
-		if err != nil {
-			return nil, err
-		}
 		if limit > 0 && uint(len(ips)) >= limit {
 			break
 		}
@@ -162,55 +158,3 @@ func (p *textParser) Parse(body io.Reader, validator IPValidator, limit uint) (I
 
 	return ips, nil
 }
-
-func (p *textParser) ParseIPsByVersion(body io.Reader, validator IPValidator, limit uint) (ipV4 IPs, ipV6 IPs, err error) {
-	ipV4 = make(IPs, 0)
-	ipV6 = make(IPs, 0)
-
-	scanner := bufio.NewScanner(body)
-
-	buf := make([]byte, 0, 64*1024)
-	scanner.Buffer(buf, 1024*1024)
-
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "#") {
-			continue
-		}
-
-		ip, isFound := p.textExtract.Extract(line)
-		if !isFound {
-			continue
-		}
-
-		ip = strings.TrimSpace(ip)
-		isValid, ipVersion := validator.IsValidAndReturnVersion(ip)
-		if !isValid {
-			continue
-		}
-
-		if ipVersion == IPVersion4 {
-			ipV4, err = appendIfNotExcluded(ip, ipV4, validator)
-			if err != nil {
-				return nil, nil, err
-			}
-		} else if ipVersion == IPVersion6 {
-			ipV6, err = appendIfNotExcluded(ip, ipV6, validator)
-			if err != nil {
-				return nil, nil, err
-			}
-		} else {
-			continue
-		}
-
-		if limit > 0 && uint(len(ipV4))+uint(len(ipV6)) >= limit {
-			break
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		return nil, nil, fmt.Errorf("read response: %w", err)
-	}
-
-	return ipV4, ipV6, nil
-}