diff --git a/blocklist.go b/blocklist.go index 9a5df68..8992340 100644 --- a/blocklist.go +++ b/blocklist.go @@ -66,6 +66,17 @@ func NewConfig(limit uint) Config { } } +func NewConfigWithExclusionChecker(limit uint, exclusionChecker parser.ExclusionChecker) Config { + return Config{ + Limit: limit, + Validator: &parser.DefaultIPValidator{ + ExclusionChecker: exclusionChecker, + }, + ContextTimeout: contextTimeout, + RequestTimeout: requestTimeout, + } +} + // NewConfigWithValidator creates a new Config with the specified validator. // limit is the maximum number of items to process or validate. 0 means no limit. // validator is the IP validator to use. diff --git a/examples/blocklist.go b/examples/blocklist.go index bbd5a8d..ebc88ad 100644 --- a/examples/blocklist.go +++ b/examples/blocklist.go @@ -24,6 +24,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { @@ -54,6 +66,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/bruteforceblocker.go b/examples/bruteforceblocker.go index 734ca23..62db277 100644 --- a/examples/bruteforceblocker.go +++ b/examples/bruteforceblocker.go @@ -22,6 +22,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/ciarmy.go b/examples/ciarmy.go index 6885f18..6bc6805 100644 --- a/examples/ciarmy.go +++ b/examples/ciarmy.go @@ -22,6 +22,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/dshield.go b/examples/dshield.go index b208627..b0647c2 100644 --- a/examples/dshield.go +++ b/examples/dshield.go @@ -22,6 +22,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { @@ -51,6 +63,20 @@ func main() { limit := uint(0) config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{}) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{ + // ExclusionChecker: exclusionChecker, + //}) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/greensnow.go b/examples/greensnow.go index ebbd095..e7a4986 100644 --- a/examples/greensnow.go +++ b/examples/greensnow.go @@ -22,6 +22,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/honeypot.go b/examples/honeypot.go index dc4a79a..1f48b5e 100644 --- a/examples/honeypot.go +++ b/examples/honeypot.go @@ -46,6 +46,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/spamhaus.go b/examples/spamhaus.go index 8b41680..0e7b507 100644 --- a/examples/spamhaus.go +++ b/examples/spamhaus.go @@ -33,6 +33,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/examples/stopforumspam.go b/examples/stopforumspam.go index 1c3e81b..8341596 100644 --- a/examples/stopforumspam.go +++ b/examples/stopforumspam.go @@ -22,6 +22,19 @@ func main() { // limit 0 - no limit limit := uint(0) config := blocklist.NewConfig(limit) + + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + configZip := blocklist.NewConfigZip(config) // Get IPv4 and IPv6 addresses in one list diff --git a/examples/tor.go b/examples/tor.go index 37bedef..e6dee02 100644 --- a/examples/tor.go +++ b/examples/tor.go @@ -22,6 +22,18 @@ func main() { limit := uint(0) config := blocklist.NewConfig(limit) + // If you need to exclude an IP address or subnet + //excludeIPs := []string{ + // "172.18.0.2", + // //"172.18.0.0/24", + // //"172.18.0.0-172.18.0.255", + //} + //exclusionChecker, err := parser.NewExclusionChecker(excludeIPs) + //if err != nil { + // panic(err) + //} + //config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker) + // Get IPv4 and IPv6 addresses in one list ips, err := blocklist.Get(url, pars, config) if err != nil { diff --git a/parser/exclusion_checker.go b/parser/exclusion_checker.go new file mode 100644 index 0000000..73423d0 --- /dev/null +++ b/parser/exclusion_checker.go @@ -0,0 +1,294 @@ +package parser + +import ( + "encoding/binary" + "fmt" + "math/big" + "net/netip" + "strings" +) + +// ExclusionChecker is an interface for checking if an IP is excluded. +type ExclusionChecker interface { + IsExcluded(ip string) (excluded bool, ips []string, err error) +} + +type ipRange struct { + start netip.Addr + end netip.Addr + isV4 bool +} + +type exclusionChecker struct { + ipRanges []ipRange +} + +// NewExclusionChecker creates a new exclusion checker. +func NewExclusionChecker(ips []string) (ExclusionChecker, error) { + ipRanges := make([]ipRange, 0, len(ips)) + + for _, ip := range ips { + ipRange, err := parseSpec(ip) + if err != nil { + return nil, fmt.Errorf("invalid exclusion IP: %s", ip) + } + + ipRanges = append(ipRanges, ipRange) + } + + return &exclusionChecker{ + ipRanges: ipRanges, + }, nil +} + +// IsExcluded checks if the given IP is excluded. +func (e *exclusionChecker) IsExcluded(ip string) (excluded bool, ips []string, err error) { + inIPRange, err := parseSpec(ip) + if err != nil { + return false, nil, err + } + + pending := []ipRange{inIPRange} + matchedAny := false + + for _, baseRange := range e.ipRanges { + // Different IP families do not overlap. + if baseRange.isV4 != inIPRange.isV4 { + continue + } + + nextPending := make([]ipRange, 0, len(pending)) + for _, candidate := range pending { + matched, remains := subtractRange(candidate, baseRange) + if matched { + matchedAny = true + } + nextPending = append(nextPending, remains...) + } + + pending = nextPending + if len(pending) == 0 { + // Completely excluded the range. + return true, []string{}, nil + } + } + + if !matchedAny { + return false, nil, nil + } + + out := make([]string, 0, len(pending)) + for _, r := range pending { + out = append(out, formatRange(r)) + } + return true, out, nil +} + +func parseSpec(s string) (ipRange, error) { + s = strings.TrimSpace(s) + if s == "" { + return ipRange{}, fmt.Errorf("empty value") + } + + // range: a-b + if strings.Contains(s, "-") { + parts := strings.Split(s, "-") + if len(parts) != 2 { + return ipRange{}, fmt.Errorf("invalid range format") + } + a, err := parseAddr(strings.TrimSpace(parts[0])) + if err != nil { + return ipRange{}, err + } + b, err := parseAddr(strings.TrimSpace(parts[1])) + if err != nil { + return ipRange{}, err + } + if a.Is4() != b.Is4() { + return ipRange{}, fmt.Errorf("mixed ip families in range") + } + if compareAddr(a, b) > 0 { + return ipRange{}, fmt.Errorf("range start > end") + } + return ipRange{start: a, end: b, isV4: a.Is4()}, nil + } + + // cidr + if strings.Contains(s, "/") { + p, err := netip.ParsePrefix(s) + if err != nil { + return ipRange{}, fmt.Errorf("invalid cidr: %w", err) + } + p = p.Masked() + start := p.Addr() + + var end netip.Addr + if start.Is4() { + u := ip4ToU32(start) + hostBits := uint32(32 - p.Bits()) + var mask uint32 + if hostBits == 32 { + mask = ^uint32(0) + } else { + mask = (uint32(1) << hostBits) - 1 + } + end = u32ToIP4(u | mask) + return ipRange{start: start, end: end, isV4: true}, nil + } + + u := ip16ToBig(start) + hostBits := 128 - p.Bits() + ones := new(big.Int).Lsh(big.NewInt(1), uint(hostBits)) + ones.Sub(ones, big.NewInt(1)) + u.Or(u, ones) + end = bigToIP16(u) + return ipRange{start: start, end: end, isV4: false}, nil + } + + // single ip + a, err := parseAddr(s) + if err != nil { + return ipRange{}, err + } + return ipRange{start: a, end: a, isV4: a.Is4()}, nil +} + +func parseAddr(s string) (netip.Addr, error) { + a, err := netip.ParseAddr(s) + if err != nil { + return netip.Addr{}, fmt.Errorf("invalid ip: %w", err) + } + return a.Unmap(), nil // important for ::ffff:1.2.3.4 +} + +func compareAddr(a, b netip.Addr) int { + if a.Is4() && b.Is4() { + ua := ip4ToU32(a) + ub := ip4ToU32(b) + switch { + case ua < ub: + return -1 + case ua > ub: + return 1 + default: + return 0 + } + } + aa := a.As16() + bb := b.As16() + for i := 0; i < 16; i++ { + if aa[i] < bb[i] { + return -1 + } + if aa[i] > bb[i] { + return 1 + } + } + return 0 +} + +func ip4ToU32(a netip.Addr) uint32 { + v := a.As4() + return binary.BigEndian.Uint32(v[:]) +} + +func u32ToIP4(u uint32) netip.Addr { + var b [4]byte + binary.BigEndian.PutUint32(b[:], u) + return netip.AddrFrom4(b) +} + +func ip16ToBig(a netip.Addr) *big.Int { + v := a.As16() + return new(big.Int).SetBytes(v[:]) +} + +func bigToIP16(x *big.Int) netip.Addr { + var b [16]byte + raw := x.Bytes() + copy(b[16-len(raw):], raw) + return netip.AddrFrom16(b) +} + +func subtractRange(incoming, base ipRange) (bool, []ipRange) { + // No intersection + if compareAddr(base.end, incoming.start) < 0 || compareAddr(base.start, incoming.end) > 0 { + return false, []ipRange{incoming} + } + + // Full incoming coverage + if compareAddr(base.start, incoming.start) <= 0 && compareAddr(base.end, incoming.end) >= 0 { + return true, nil + } + + remains := make([]ipRange, 0, 2) + + // Left side + if compareAddr(base.start, incoming.start) > 0 { + leftEnd, ok := prevIP(base.start) + if ok && compareAddr(incoming.start, leftEnd) <= 0 { + remains = append(remains, ipRange{ + start: incoming.start, + end: leftEnd, + isV4: incoming.isV4, + }) + } + } + + // Right side + if compareAddr(base.end, incoming.end) < 0 { + rightStart, ok := nextIP(base.end) + if ok && compareAddr(rightStart, incoming.end) <= 0 { + remains = append(remains, ipRange{ + start: rightStart, + end: incoming.end, + isV4: incoming.isV4, + }) + } + } + + return true, remains +} + +func nextIP(a netip.Addr) (netip.Addr, bool) { + if a.Is4() { + u := ip4ToU32(a) + if u == ^uint32(0) { + return netip.Addr{}, false + } + return u32ToIP4(u + 1), true + } + + u := ip16ToBig(a) + max := new(big.Int).Lsh(big.NewInt(1), 128) + max.Sub(max, big.NewInt(1)) + if u.Cmp(max) == 0 { + return netip.Addr{}, false + } + u.Add(u, big.NewInt(1)) + return bigToIP16(u), true +} + +func prevIP(a netip.Addr) (netip.Addr, bool) { + if a.Is4() { + u := ip4ToU32(a) + if u == 0 { + return netip.Addr{}, false + } + return u32ToIP4(u - 1), true + } + + u := ip16ToBig(a) + if u.Sign() == 0 { + return netip.Addr{}, false + } + u.Sub(u, big.NewInt(1)) + return bigToIP16(u), true +} + +func formatRange(r ipRange) string { + if r.start == r.end { + return r.start.String() + } + return r.start.String() + "-" + r.end.String() +} diff --git a/parser/json_lines.go b/parser/json_lines.go index 442fc01..f259358 100644 --- a/parser/json_lines.go +++ b/parser/json_lines.go @@ -55,7 +55,10 @@ func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uin continue } - ips = append(ips, ip) + ips, err = appendIfNotExcluded(ip, ips, validator) + if err != nil { + return nil, err + } if limit > 0 && uint(len(ips)) >= limit { break } @@ -96,9 +99,15 @@ func (p *jsonLinesParser) ParseIPsByVersion(body io.Reader, validator IPValidato } if ipVersion == IPVersion4 { - ipV4 = append(ipV4, ip) + ipV4, err = appendIfNotExcluded(ip, ipV4, validator) + if err != nil { + return nil, nil, err + } } else if ipVersion == IPVersion6 { - ipV6 = append(ipV6, ip) + ipV6, err = appendIfNotExcluded(ip, ipV6, validator) + if err != nil { + return nil, nil, err + } } else { continue } diff --git a/parser/parser.go b/parser/parser.go index 5dc8947..9be7036 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -20,7 +20,11 @@ type IPValidator interface { // IsValid checks if the given IP address is valid. IsValid(ip string) bool + // IsValidAndReturnVersion validates the given IP address and returns whether it is valid and its version (IPv4 or IPv6). IsValidAndReturnVersion(ip string) (bool, IPVersion) + + // IsExcluded determines if the given IP address is excluded based on certain criteria and returns related IP addresses. + IsExcluded(ip string) (excluded bool, ips []string, err error) } type IPVersion int @@ -35,7 +39,9 @@ type IPs []string // DefaultIPValidator implements IPValidator interface. // It validates IP addresses by parsing them using net.ParseIP and net.ParseCIDR. -type DefaultIPValidator struct{} +type DefaultIPValidator struct { + ExclusionChecker ExclusionChecker +} // IsValid checks if the given IP address is valid. // It returns true if the IP address is not a loopback address. @@ -103,9 +109,18 @@ func (v *DefaultIPValidator) IsValidAndReturnVersion(value string) (bool, IPVers return false, IPVersion4 } +func (v *DefaultIPValidator) IsExcluded(ip string) (excluded bool, ips []string, err error) { + if v.ExclusionChecker == nil { + return false, nil, nil + } + return v.ExclusionChecker.IsExcluded(ip) +} + // IPRangeValidator implements IPValidator interface. // It validates IP ranges by parsing them using net.ParseIP and checking if the start and end IPs are in the same network. -type IPRangeValidator struct{} +type IPRangeValidator struct { + ExclusionChecker ExclusionChecker +} // IsValid checks if the given IP range is valid. // It returns true if the start and end IPs are in the same network. @@ -182,3 +197,10 @@ func (v *IPRangeValidator) IsValidAndReturnVersion(value string) (bool, IPVersio return false, IPVersion4 } } + +func (v *IPRangeValidator) IsExcluded(ip string) (excluded bool, ips []string, err error) { + if v.ExclusionChecker == nil { + return false, nil, nil + } + return v.ExclusionChecker.IsExcluded(ip) +} diff --git a/parser/pkg.go b/parser/pkg.go new file mode 100644 index 0000000..1075065 --- /dev/null +++ b/parser/pkg.go @@ -0,0 +1,16 @@ +package parser + +func appendIfNotExcluded(ip string, ips []string, validator IPValidator) ([]string, error) { + if exclude, listIP, err := validator.IsExcluded(ip); err != nil { + return ips, err + } else if exclude { + if listIP == nil || len(listIP) == 0 { + return ips, nil + } + ips = append(ips, listIP...) + return ips, nil + } + + ips = append(ips, ip) + return ips, nil +} diff --git a/parser/rss.go b/parser/rss.go index 02c125b..2c09e1c 100644 --- a/parser/rss.go +++ b/parser/rss.go @@ -58,7 +58,10 @@ func (p *rssParser) Parse(body io.Reader, validator IPValidator, limit uint) (IP continue } - ips = append(ips, ip) + ips, err = appendIfNotExcluded(ip, ips, validator) + if err != nil { + return nil, err + } if limit > 0 && uint(len(ips)) >= limit { break } @@ -102,9 +105,15 @@ func (p *rssParser) ParseIPsByVersion(body io.Reader, validator IPValidator, lim } if ipVersion == IPVersion4 { - ipV4 = append(ipV4, ip) + ipV4, err = appendIfNotExcluded(ip, ipV4, validator) + if err != nil { + return nil, nil, err + } } else if ipVersion == IPVersion6 { - ipV6 = append(ipV6, ip) + ipV6, err = appendIfNotExcluded(ip, ipV6, validator) + if err != nil { + return nil, nil, err + } } else { continue } diff --git a/parser/text.go b/parser/text.go index 80be696..4fe2873 100644 --- a/parser/text.go +++ b/parser/text.go @@ -146,7 +146,11 @@ func (p *textParser) Parse(body io.Reader, validator IPValidator, limit uint) (I continue } - ips = append(ips, ip) + var err error + ips, err = appendIfNotExcluded(ip, ips, validator) + if err != nil { + return nil, err + } if limit > 0 && uint(len(ips)) >= limit { break } @@ -186,9 +190,15 @@ func (p *textParser) ParseIPsByVersion(body io.Reader, validator IPValidator, li } if ipVersion == IPVersion4 { - ipV4 = append(ipV4, ip) + ipV4, err = appendIfNotExcluded(ip, ipV4, validator) + if err != nil { + return nil, nil, err + } } else if ipVersion == IPVersion6 { - ipV6 = append(ipV6, ip) + ipV6, err = appendIfNotExcluded(ip, ipV6, validator) + if err != nil { + return nil, nil, err + } } else { continue }