kor-elf-shield/blocklist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Merge pull request 'v1.2.0' (#3) from develop into main

Browse Source 
Reviewed-on: #3
This commit was merged in pull request #3.
This commit is contained in:
kor-elf
2026-08-12 16:36:11 +05:00
parent 7331ed2b2b e44264c21c
commit 16af4141f6
16 changed files with 517 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
blocklist.go
+11
View File
@@ -66,6 +66,17 @@ func NewConfig(limit uint) Config {
}
}
func NewConfigWithExclusionChecker(limit uint, exclusionChecker parser.ExclusionChecker) Config {
return Config{
Limit: limit,
Validator: &parser.DefaultIPValidator{
ExclusionChecker: exclusionChecker,
},
ContextTimeout: contextTimeout,
RequestTimeout: requestTimeout,
}
}
// NewConfigWithValidator creates a new Config with the specified validator.
// limit is the maximum number of items to process or validate. 0 means no limit.
// validator is the IP validator to use.
examples/blocklist.go
+24
View File
@@ -24,6 +24,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
@@ -54,6 +66,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/bruteforceblocker.go
+12
View File
@@ -22,6 +22,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/ciarmy.go
+12
View File
@@ -22,6 +22,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/dshield.go
+26
View File
@@ -22,6 +22,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
@@ -51,6 +63,20 @@ func main() {
limit := uint(0)
config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{})
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithValidator(limit, &parser.IPRangeValidator{
// ExclusionChecker: exclusionChecker,
//})
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/greensnow.go
+12
View File
@@ -22,6 +22,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/honeypot.go
+12
View File
@@ -46,6 +46,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/spamhaus.go
+12
View File
@@ -33,6 +33,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
examples/stopforumspam.go
+13
View File
@@ -22,6 +22,19 @@ func main() {
// limit 0 - no limit
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
configZip := blocklist.NewConfigZip(config)
// Get IPv4 and IPv6 addresses in one list
examples/tor.go
+12
View File
@@ -22,6 +22,18 @@ func main() {
limit := uint(0)
config := blocklist.NewConfig(limit)
// If you need to exclude an IP address or subnet
//excludeIPs := []string{
// "172.18.0.2",
// //"172.18.0.0/24",
// //"172.18.0.0-172.18.0.255",
//}
//exclusionChecker, err := parser.NewExclusionChecker(excludeIPs)
//if err != nil {
// panic(err)
//}
//config := blocklist.NewConfigWithExclusionChecker(limit, exclusionChecker)
// Get IPv4 and IPv6 addresses in one list
ips, err := blocklist.Get(url, pars, config)
if err != nil {
parser/exclusion_checker.go
+294
View File
@@ -0,0 +1,294 @@
package parser
import (
"encoding/binary"
"fmt"
"math/big"
"net/netip"
"strings"
)
// ExclusionChecker is an interface for checking if an IP is excluded.
type ExclusionChecker interface {
IsExcluded(ip string) (excluded bool, ips []string, err error)
}
type ipRange struct {
start netip.Addr
end netip.Addr
isV4 bool
}
type exclusionChecker struct {
ipRanges []ipRange
}
// NewExclusionChecker creates a new exclusion checker.
func NewExclusionChecker(ips []string) (ExclusionChecker, error) {
ipRanges := make([]ipRange, 0, len(ips))
for _, ip := range ips {
ipRange, err := parseSpec(ip)
if err != nil {
return nil, fmt.Errorf("invalid exclusion IP: %s", ip)
}
ipRanges = append(ipRanges, ipRange)
}
return &exclusionChecker{
ipRanges: ipRanges,
}, nil
}
// IsExcluded checks if the given IP is excluded.
func (e *exclusionChecker) IsExcluded(ip string) (excluded bool, ips []string, err error) {
inIPRange, err := parseSpec(ip)
if err != nil {
return false, nil, err
}
pending := []ipRange{inIPRange}
matchedAny := false
for _, baseRange := range e.ipRanges {
// Different IP families do not overlap.
if baseRange.isV4 != inIPRange.isV4 {
continue
}
nextPending := make([]ipRange, 0, len(pending))
for _, candidate := range pending {
matched, remains := subtractRange(candidate, baseRange)
if matched {
matchedAny = true
}
nextPending = append(nextPending, remains...)
}
pending = nextPending
if len(pending) == 0 {
// Completely excluded the range.
return true, []string{}, nil
}
}
if !matchedAny {
return false, nil, nil
}
out := make([]string, 0, len(pending))
for _, r := range pending {
out = append(out, formatRange(r))
}
return true, out, nil
}
func parseSpec(s string) (ipRange, error) {
s = strings.TrimSpace(s)
if s == "" {
return ipRange{}, fmt.Errorf("empty value")
}
// range: a-b
if strings.Contains(s, "-") {
parts := strings.Split(s, "-")
if len(parts) != 2 {
return ipRange{}, fmt.Errorf("invalid range format")
}
a, err := parseAddr(strings.TrimSpace(parts[0]))
if err != nil {
return ipRange{}, err
}
b, err := parseAddr(strings.TrimSpace(parts[1]))
if err != nil {
return ipRange{}, err
}
if a.Is4() != b.Is4() {
return ipRange{}, fmt.Errorf("mixed ip families in range")
}
if compareAddr(a, b) > 0 {
return ipRange{}, fmt.Errorf("range start > end")
}
return ipRange{start: a, end: b, isV4: a.Is4()}, nil
}
// cidr
if strings.Contains(s, "/") {
p, err := netip.ParsePrefix(s)
if err != nil {
return ipRange{}, fmt.Errorf("invalid cidr: %w", err)
}
p = p.Masked()
start := p.Addr()
var end netip.Addr
if start.Is4() {
u := ip4ToU32(start)
hostBits := uint32(32 - p.Bits())
var mask uint32
if hostBits == 32 {
mask = ^uint32(0)
} else {
mask = (uint32(1) << hostBits) - 1
}
end = u32ToIP4(u | mask)
return ipRange{start: start, end: end, isV4: true}, nil
}
u := ip16ToBig(start)
hostBits := 128 - p.Bits()
ones := new(big.Int).Lsh(big.NewInt(1), uint(hostBits))
ones.Sub(ones, big.NewInt(1))
u.Or(u, ones)
end = bigToIP16(u)
return ipRange{start: start, end: end, isV4: false}, nil
}
// single ip
a, err := parseAddr(s)
if err != nil {
return ipRange{}, err
}
return ipRange{start: a, end: a, isV4: a.Is4()}, nil
}
func parseAddr(s string) (netip.Addr, error) {
a, err := netip.ParseAddr(s)
if err != nil {
return netip.Addr{}, fmt.Errorf("invalid ip: %w", err)
}
return a.Unmap(), nil // important for ::ffff:1.2.3.4
}
func compareAddr(a, b netip.Addr) int {
if a.Is4() && b.Is4() {
ua := ip4ToU32(a)
ub := ip4ToU32(b)
switch {
case ua < ub:
return -1
case ua > ub:
return 1
default:
return 0
}
}
aa := a.As16()
bb := b.As16()
for i := 0; i < 16; i++ {
if aa[i] < bb[i] {
return -1
}
if aa[i] > bb[i] {
return 1
}
}
return 0
}
func ip4ToU32(a netip.Addr) uint32 {
v := a.As4()
return binary.BigEndian.Uint32(v[:])
}
func u32ToIP4(u uint32) netip.Addr {
var b [4]byte
binary.BigEndian.PutUint32(b[:], u)
return netip.AddrFrom4(b)
}
func ip16ToBig(a netip.Addr) *big.Int {
v := a.As16()
return new(big.Int).SetBytes(v[:])
}
func bigToIP16(x *big.Int) netip.Addr {
var b [16]byte
raw := x.Bytes()
copy(b[16-len(raw):], raw)
return netip.AddrFrom16(b)
}
func subtractRange(incoming, base ipRange) (bool, []ipRange) {
// No intersection
if compareAddr(base.end, incoming.start) < 0 || compareAddr(base.start, incoming.end) > 0 {
return false, []ipRange{incoming}
}
// Full incoming coverage
if compareAddr(base.start, incoming.start) <= 0 && compareAddr(base.end, incoming.end) >= 0 {
return true, nil
}
remains := make([]ipRange, 0, 2)
// Left side
if compareAddr(base.start, incoming.start) > 0 {
leftEnd, ok := prevIP(base.start)
if ok && compareAddr(incoming.start, leftEnd) <= 0 {
remains = append(remains, ipRange{
start: incoming.start,
end: leftEnd,
isV4: incoming.isV4,
})
}
}
// Right side
if compareAddr(base.end, incoming.end) < 0 {
rightStart, ok := nextIP(base.end)
if ok && compareAddr(rightStart, incoming.end) <= 0 {
remains = append(remains, ipRange{
start: rightStart,
end: incoming.end,
isV4: incoming.isV4,
})
}
}
return true, remains
}
func nextIP(a netip.Addr) (netip.Addr, bool) {
if a.Is4() {
u := ip4ToU32(a)
if u == ^uint32(0) {
return netip.Addr{}, false
}
return u32ToIP4(u + 1), true
}
u := ip16ToBig(a)
max := new(big.Int).Lsh(big.NewInt(1), 128)
max.Sub(max, big.NewInt(1))
if u.Cmp(max) == 0 {
return netip.Addr{}, false
}
u.Add(u, big.NewInt(1))
return bigToIP16(u), true
}
func prevIP(a netip.Addr) (netip.Addr, bool) {
if a.Is4() {
u := ip4ToU32(a)
if u == 0 {
return netip.Addr{}, false
}
return u32ToIP4(u - 1), true
}
u := ip16ToBig(a)
if u.Sign() == 0 {
return netip.Addr{}, false
}
u.Sub(u, big.NewInt(1))
return bigToIP16(u), true
}
func formatRange(r ipRange) string {
if r.start == r.end {
return r.start.String()
}
return r.start.String() + "-" + r.end.String()
}
parser/json_lines.go
+12 -3
View File
@@ -55,7 +55,10 @@ func (p *jsonLinesParser) Parse(body io.Reader, validator IPValidator, limit uin
continue
}
ips = append(ips, ip)
ips, err = appendIfNotExcluded(ip, ips, validator)
if err != nil {
return nil, err
}
if limit > 0 && uint(len(ips)) >= limit {
break
}
@@ -96,9 +99,15 @@ func (p *jsonLinesParser) ParseIPsByVersion(body io.Reader, validator IPValidato
}
if ipVersion == IPVersion4 {
ipV4 = append(ipV4, ip)
ipV4, err = appendIfNotExcluded(ip, ipV4, validator)
if err != nil {
return nil, nil, err
}
} else if ipVersion == IPVersion6 {
ipV6 = append(ipV6, ip)
ipV6, err = appendIfNotExcluded(ip, ipV6, validator)
if err != nil {
return nil, nil, err
}
} else {
continue
}
parser/parser.go
+24 -2
View File
@@ -20,7 +20,11 @@ type IPValidator interface {
// IsValid checks if the given IP address is valid.
IsValid(ip string) bool
// IsValidAndReturnVersion validates the given IP address and returns whether it is valid and its version (IPv4 or IPv6).
IsValidAndReturnVersion(ip string) (bool, IPVersion)
// IsExcluded determines if the given IP address is excluded based on certain criteria and returns related IP addresses.
IsExcluded(ip string) (excluded bool, ips []string, err error)
}
type IPVersion int
@@ -35,7 +39,9 @@ type IPs []string
// DefaultIPValidator implements IPValidator interface.
// It validates IP addresses by parsing them using net.ParseIP and net.ParseCIDR.
type DefaultIPValidator struct{}
type DefaultIPValidator struct {
ExclusionChecker ExclusionChecker
}
// IsValid checks if the given IP address is valid.
// It returns true if the IP address is not a loopback address.
@@ -103,9 +109,18 @@ func (v *DefaultIPValidator) IsValidAndReturnVersion(value string) (bool, IPVers
return false, IPVersion4
}
func (v *DefaultIPValidator) IsExcluded(ip string) (excluded bool, ips []string, err error) {
if v.ExclusionChecker == nil {
return false, nil, nil
}
return v.ExclusionChecker.IsExcluded(ip)
}
// IPRangeValidator implements IPValidator interface.
// It validates IP ranges by parsing them using net.ParseIP and checking if the start and end IPs are in the same network.
type IPRangeValidator struct{}
type IPRangeValidator struct {
ExclusionChecker ExclusionChecker
}
// IsValid checks if the given IP range is valid.
// It returns true if the start and end IPs are in the same network.
@@ -182,3 +197,10 @@ func (v *IPRangeValidator) IsValidAndReturnVersion(value string) (bool, IPVersio
return false, IPVersion4
}
}
func (v *IPRangeValidator) IsExcluded(ip string) (excluded bool, ips []string, err error) {
if v.ExclusionChecker == nil {
return false, nil, nil
}
return v.ExclusionChecker.IsExcluded(ip)
}
parser/pkg.go
+16
View File
@@ -0,0 +1,16 @@
package parser
func appendIfNotExcluded(ip string, ips []string, validator IPValidator) ([]string, error) {
if exclude, listIP, err := validator.IsExcluded(ip); err != nil {
return ips, err
} else if exclude {
if listIP == nil || len(listIP) == 0 {
return ips, nil
}
ips = append(ips, listIP...)
return ips, nil
}
ips = append(ips, ip)
return ips, nil
}
parser/rss.go
+12 -3
View File
@@ -58,7 +58,10 @@ func (p *rssParser) Parse(body io.Reader, validator IPValidator, limit uint) (IP
continue
}
ips = append(ips, ip)
ips, err = appendIfNotExcluded(ip, ips, validator)
if err != nil {
return nil, err
}
if limit > 0 && uint(len(ips)) >= limit {
break
}
@@ -102,9 +105,15 @@ func (p *rssParser) ParseIPsByVersion(body io.Reader, validator IPValidator, lim
}
if ipVersion == IPVersion4 {
ipV4 = append(ipV4, ip)
ipV4, err = appendIfNotExcluded(ip, ipV4, validator)
if err != nil {
return nil, nil, err
}
} else if ipVersion == IPVersion6 {
ipV6 = append(ipV6, ip)
ipV6, err = appendIfNotExcluded(ip, ipV6, validator)
if err != nil {
return nil, nil, err
}
} else {
continue
}
parser/text.go
+13 -3
View File
@@ -146,7 +146,11 @@ func (p *textParser) Parse(body io.Reader, validator IPValidator, limit uint) (I
continue
}
ips = append(ips, ip)
var err error
ips, err = appendIfNotExcluded(ip, ips, validator)
if err != nil {
return nil, err
}
if limit > 0 && uint(len(ips)) >= limit {
break
}
@@ -186,9 +190,15 @@ func (p *textParser) ParseIPsByVersion(body io.Reader, validator IPValidator, li
}
if ipVersion == IPVersion4 {
ipV4 = append(ipV4, ip)
ipV4, err = appendIfNotExcluded(ip, ipV4, validator)
if err != nil {
return nil, nil, err
}
} else if ipVersion == IPVersion6 {
ipV6 = append(ipV6, ip)
ipV6, err = appendIfNotExcluded(ip, ipV6, validator)
if err != nil {
return nil, nil, err
}
} else {
continue
}