kor-elf-shield/kor-elf-shield
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 8 Wiki Activity

Integrate advanced alert grouping functionality

Browse Source 
- Introduced `AlertGroup` structure for advanced rate-limiting and reset logic.
- Added support for nested rate-limit configuration with `RateLimit` structure.
- Implemented `alert_group.Group` service to facilitate alert group analysis and persistence.
- Integrated alert group logic into the analyzer configuration and runtime processing pipeline.
- Updated `LogAlertRule` to support group associations and validations.
- Enhanced repository structure with `AlertGroupRepository` for persistent alert group management.
This commit is contained in:
Leonid Nikitin
2026-02-16 22:26:33 +05:00
parent c6841d14f3
commit e85fd785cd
15 changed files with 324 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/cmd/daemon/start.go
View File

@@ -64,6 +64,7 @@ func runDaemon(ctx context.Context, _ *cli.Command) error {
defer func() {
_ = repositories.Close()
}()
config.Repositories = repositories
notificationsService, err := newNotificationsService(repositories.NotificationsQueue(), logger)
if err != nil {

5
internal/daemon/analyzer/analyzer.go
View File

@@ -7,6 +7,7 @@ import (
config2 "git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
analyzerLog "git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/log"
analysisServices "git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/log/analysis"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/db"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/notifications"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/log"
)
@@ -27,7 +28,7 @@ type analyzer struct {
logChan chan analysisServices.Entry
}
func New(config config2.Config, logger log.Logger, notify notifications.Notifications) Analyzer {
func New(config config2.Config, repositories db.Repositories, logger log.Logger, notify notifications.Notifications) Analyzer {
var journalMatches []string
journalMatchesUniq := map[string]struct{}{}
@@ -65,7 +66,7 @@ func New(config config2.Config, logger log.Logger, notify notifications.Notifica
systemdService := analyzerLog.NewSystemd(config.BinPath.Journalctl, journalMatches, logger)
filesService := analyzerLog.NewFileMonitoring(files, logger)
analysisService := analyzerLog.NewAnalysis(rulesIndex, logger, notify)
analysisService := analyzerLog.NewAnalysis(rulesIndex, repositories, logger, notify)
return &analyzer{
config: config,

29
internal/daemon/analyzer/config/alert_group.go Normal file
View File

@@ -0,0 +1,29 @@
package config
type RateLimit struct {
Count uint32
Period uint32
}
type AlertGroup struct {
Name string
Message string
RateLimits []RateLimit
RateLimitResetPeriod uint32
}
func (g *AlertGroup) RateLimit(level uint64) (rateLimit RateLimit, err error) {
lenRateLimits := len(g.RateLimits) - 1
if lenRateLimits == 0 {
return RateLimit{}, err
}
if level <= uint64(lenRateLimits) {
rateLimit = g.RateLimits[level]
} else {
rateLimit = g.RateLimits[lenRateLimits]
}
return rateLimit, nil
}

13
internal/daemon/analyzer/config/config.go
View File

@@ -5,7 +5,6 @@ import (
"regexp"
"strings"
"sync"
"time"
"unicode"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/setting/validate"
@@ -138,15 +137,3 @@ type PatternValue struct {
Name string
Value uint8
}
type RateLimit struct {
Count uint32
Period time.Duration
}
type AlertGroup struct {
Name string
Message string
RateLimits []RateLimit
RateLimitResetPeriod time.Duration
}

8
internal/daemon/analyzer/log/analysis.go
View File

@@ -2,6 +2,8 @@ package log
import (
analysisServices "git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/log/analysis"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/log/analysis/alert_group"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/db"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/notifications"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/log"
)
@@ -14,9 +16,11 @@ type analysis struct {
alertService analysisServices.Alert
}
func NewAnalysis(rulesIndex *analysisServices.RulesIndex, logger log.Logger, notify notifications.Notifications) Analysis {
func NewAnalysis(rulesIndex *analysisServices.RulesIndex, repositories db.Repositories, logger log.Logger, notify notifications.Notifications) Analysis {
alertGroupService := alert_group.NewGroup(repositories.AlertGroup(), logger)
return &analysis{
alertService: analysisServices.NewAlert(rulesIndex, logger, notify),
alertService: analysisServices.NewAlert(rulesIndex, alertGroupService, logger, notify),
}
}

29
internal/daemon/analyzer/log/analysis/alert.go
View File

@@ -5,6 +5,7 @@ import (
"time"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/log/analysis/alert_group"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/notifications"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/i18n"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/log"
@@ -15,9 +16,10 @@ type Alert interface {
}
type alert struct {
rulesIndex *RulesIndex
logger log.Logger
notify notifications.Notifications
rulesIndex *RulesIndex
alertGroupService alert_group.Group
logger log.Logger
notify notifications.Notifications
}
type alertAnalyzeRuleReturn struct {
@@ -32,11 +34,12 @@ type alertNotify struct {
fields []*regexField
}
func NewAlert(rulesIndex *RulesIndex, logger log.Logger, notify notifications.Notifications) Alert {
func NewAlert(rulesIndex *RulesIndex, alertGroupService alert_group.Group, logger log.Logger, notify notifications.Notifications) Alert {
return &alert{
rulesIndex: rulesIndex,
logger: logger,
notify: notify,
rulesIndex: rulesIndex,
alertGroupService: alertGroupService,
logger: logger,
notify: notify,
}
}
@@ -53,7 +56,19 @@ func (a *alert) Analyze(entry *Entry) {
groupName := ""
messages := []string{}
if rule.Group != nil {
alertGroup, err := a.alertGroupService.Analyze(rule.Group, entry.Time, entry.Message)
if err != nil {
a.logger.Error(fmt.Sprintf("Failed to analyze alert group: %s", err))
continue
}
if !alertGroup.Alerted {
continue
}
groupName = rule.Group.Name
for _, lastLog := range alertGroup.LastLogs {
messages = append(messages, lastLog)
}
} else {
messages = append(messages, entry.Message)
}

103
internal/daemon/analyzer/log/analysis/alert_group/group.go Normal file
View File

@@ -0,0 +1,103 @@
package alert_group
import (
"fmt"
"time"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/db/entity"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/db/repository"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/log"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/pkg/time_operation"
)
type Group interface {
Analyze(alertGroup *config.AlertGroup, eventTime time.Time, message string) (AnalysisResult, error)
}
type group struct {
alertGroupRepository repository.AlertGroupRepository
logger log.Logger
}
type AnalysisResult struct {
Alerted bool
LastLogs []string
}
func NewGroup(alertGroupRepository repository.AlertGroupRepository, logger log.Logger) Group {
return &group{
alertGroupRepository: alertGroupRepository,
logger: logger,
}
}
func (g *group) Analyze(alertGroup *config.AlertGroup, eventTime time.Time, message string) (AnalysisResult, error) {
analysisResult := AnalysisResult{
Alerted: false,
}
g.logger.Debug(fmt.Sprintf("Analyzing alert group %s", alertGroup.Name))
err := g.alertGroupRepository.Update(alertGroup.Name, func(entityAlertGroup *entity.AlertGroup) (*entity.AlertGroup, error) {
rateLimit, err := alertGroup.RateLimit(entityAlertGroup.CurrentLevelTriggerCount)
if err != nil {
return entityAlertGroup, err
}
if time_operation.IsRateLimited(entityAlertGroup.LastTriggeredAtUnix, eventTime, int64(rateLimit.Period)) {
g.logger.Debug(fmt.Sprintf("Alert group %s is rate limited", alertGroup.Name))
analysisResult, entityAlertGroup = g.analysisResult(rateLimit, eventTime, message, entityAlertGroup)
return entityAlertGroup, nil
}
entityAlertGroup.TriggerCount = 0
if time_operation.IsReset(entityAlertGroup.LastTriggeredAtUnix, eventTime, int64(alertGroup.RateLimitResetPeriod)) {
g.logger.Debug(fmt.Sprintf("Alert group %s is reset", alertGroup.Name))
entityAlertGroup.Reset()
rateLimit, err = alertGroup.RateLimit(0)
if err != nil {
return entityAlertGroup, err
}
}
g.logger.Debug(fmt.Sprintf("Alert not rate limited"))
analysisResult, entityAlertGroup = g.analysisResult(rateLimit, eventTime, message, entityAlertGroup)
return entityAlertGroup, nil
})
if err != nil {
return AnalysisResult{
Alerted: false,
}, err
}
return analysisResult, nil
}
func (g *group) analysisResult(rateLimit config.RateLimit, eventTime time.Time, message string, entityAlertGroup *entity.AlertGroup) (AnalysisResult, *entity.AlertGroup) {
analysisResult := AnalysisResult{
Alerted: false,
}
entityAlertGroup.LastTriggeredAtUnix = eventTime.Unix()
entityAlertGroup.TriggerCount++
entityAlertGroup.LastLogs = append(entityAlertGroup.LastLogs, fmt.Sprintf("event time: %s, message: %s", eventTime.Format(time.RFC3339), message))
g.logger.Debug(fmt.Sprintf("Alert triggered. Count: %d", entityAlertGroup.TriggerCount))
if entityAlertGroup.TriggerCount >= uint64(rateLimit.Count) {
g.logger.Debug(fmt.Sprintf("Alert reached rate limit"))
analysisResult.LastLogs = entityAlertGroup.LastLogs
analysisResult.Alerted = true
entityAlertGroup.CurrentLevelTriggerCount++
entityAlertGroup.TriggerCount = 0
entityAlertGroup.LastLogs = []string{}
} else {
g.logger.Debug(fmt.Sprintf("Alert not reached rate limit"))
}
return analysisResult, entityAlertGroup
}

1
internal/daemon/db/repository/repository.go
View File

@@ -8,6 +8,7 @@ import (
const (
notificationsQueueBucket = "notifications_queue"
alertGroupBucket = "alert_group"
)
func nextID(b *bbolt.Bucket) ([]byte, error) {

2
internal/daemon/options.go
View File

@@ -2,6 +2,7 @@ package daemon
import (
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/db"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/firewall"
)
@@ -12,4 +13,5 @@ type DaemonOptions struct {
PathNftables string
ConfigFirewall firewall.Config
ConfigAnalyzer config.Config
Repositories db.Repositories
}

2
internal/daemon/server.go
View File

@@ -29,7 +29,7 @@ func NewDaemon(opts DaemonOptions, logger log.Logger, notifications notification
firewall, err := firewall2.New(opts.PathNftables, logger, opts.ConfigFirewall, docker)
analyzerService := analyzer.New(opts.ConfigAnalyzer, logger, notifications)
analyzerService := analyzer.New(opts.ConfigAnalyzer, opts.Repositories, logger, notifications)
return &daemon{
pidFile: pidFile,

23
internal/pkg/time_operation/time.go Normal file
View File

@@ -0,0 +1,23 @@
package time_operation
import "time"
func IsRateLimited(lastTriggeredAtUnix int64, eventTime time.Time, rateLimit int64) bool {
if lastTriggeredAtUnix == 0 {
return true
}
return eventTime.Unix()-lastTriggeredAtUnix < rateLimit
}
func IsReset(lastTriggeredAtUnix int64, eventTime time.Time, resetPeriod int64) bool {
if resetPeriod == 0 || lastTriggeredAtUnix == 0 {
return false
}
if eventTime.Unix()-lastTriggeredAtUnix > resetPeriod {
return true
}
return false
}

37
internal/setting/analyzer/log_alert.go
View File

@@ -1,12 +1,20 @@
package analyzer
import (
"fmt"
"regexp"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
)
var (
reName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,254}$`)
)
type LogAlert struct {
Enabled bool `mapstructure:"enabled"`
Notify bool `mapstructure:"notify"`
Groups []LogAlertGroup
Rules []LogAlertRule
}
@@ -14,6 +22,7 @@ func defaultLogAlert() LogAlert {
return LogAlert{
Enabled: true,
Notify: true,
Groups: []LogAlertGroup{},
Rules: []LogAlertRule{},
}
}
@@ -29,12 +38,25 @@ func (l *LogAlert) ToSources() ([]*config.Source, error) {
return sources, nil
}
groups, err := l.groups()
if err != nil {
return nil, fmt.Errorf("groups: %w", err)
}
for _, rule := range l.Rules {
if !rule.Enabled {
continue
}
source, err := rule.ToSource(l.Notify)
var group *config.AlertGroup
if rule.Group != "" {
if _, ok := groups[rule.Group]; !ok {
return nil, fmt.Errorf("group %q not found", rule.Group)
}
group = groups[rule.Group]
}
source, err := rule.ToSource(l.Notify, group)
if err != nil {
return nil, err
}
@@ -43,3 +65,16 @@ func (l *LogAlert) ToSources() ([]*config.Source, error) {
return sources, nil
}
func (l *LogAlert) groups() (map[string]*config.AlertGroup, error) {
groups := make(map[string]*config.AlertGroup)
for _, group := range l.Groups {
g, err := group.ToGroup()
if err != nil {
return nil, err
}
groups[g.Name] = g
}
return groups, nil
}

57
internal/setting/analyzer/log_alert_group.go Normal file
View File

@@ -0,0 +1,57 @@
package analyzer
import (
"fmt"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
)
type LogAlertGroup struct {
Name string `mapstructure:"name"`
Message string `mapstructure:"message"`
RateLimitResetPeriod int `mapstructure:"rate_limit_reset_period"`
RateLimits []LogAlertGroupRateLimit `mapstructure:"rate_limits"`
}
func (g *LogAlertGroup) ToGroup() (*config.AlertGroup, error) {
if err := g.validate(); err != nil {
return nil, err
}
var rateLimits []config.RateLimit
for _, rateLimit := range g.RateLimits {
rLimit, err := rateLimit.ToRateLimit()
if err != nil {
return nil, err
}
rateLimits = append(rateLimits, rLimit)
}
return &config.AlertGroup{
Name: g.Name,
Message: g.Message,
RateLimits: rateLimits,
RateLimitResetPeriod: uint32(g.RateLimitResetPeriod),
}, nil
}
func (g *LogAlertGroup) validate() error {
if g.Name == "" {
return fmt.Errorf("alert group name is empty")
}
if !reName.MatchString(g.Name) {
return fmt.Errorf("alert group invalid name: %s", g.Name)
}
if g.RateLimitResetPeriod < 0 {
return fmt.Errorf("alert group rate limit reset period must be positive")
}
if len(g.RateLimits) == 0 {
return fmt.Errorf("alert group rate limits is empty")
}
return nil
}

33
internal/setting/analyzer/log_alert_group_rate_limit.go Normal file
View File

@@ -0,0 +1,33 @@
package analyzer
import (
"fmt"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
)
type LogAlertGroupRateLimit struct {
Count int `mapstructure:"count"`
Period int `mapstructure:"period"`
}
func (l *LogAlertGroupRateLimit) ToRateLimit() (config.RateLimit, error) {
if err := l.validate(); err != nil {
return config.RateLimit{}, err
}
return config.RateLimit{
Count: uint32(l.Count),
Period: uint32(l.Period),
}, nil
}
func (l *LogAlertGroupRateLimit) validate() error {
if l.Count <= 0 {
return fmt.Errorf("count must be greater than 0")
}
if l.Period <= 0 {
return fmt.Errorf("period must be greater than 0")
}
return nil
}

15
internal/setting/analyzer/log_alert_rule.go
View File

@@ -2,25 +2,21 @@ package analyzer
import (
"fmt"
"regexp"
"git.kor-elf.net/kor-elf-shield/kor-elf-shield/internal/daemon/analyzer/config"
)
var (
reName = regexp.MustCompile(`^[A-Za-z0-9-_]{0,255}$`)
)
type LogAlertRule struct {
Enabled bool `mapstructure:"enabled"`
Notify bool `mapstructure:"notify"`
Name string `mapstructure:"name"`
Message string `mapstructure:"message"`
Group string `mapstructure:"group"`
Source Source
Patterns []LogAlertPattern
}
func (l *LogAlertRule) ToSource(isNotify bool) (*config.Source, error) {
func (l *LogAlertRule) ToSource(isNotify bool, group *config.AlertGroup) (*config.Source, error) {
if err := l.validate(); err != nil {
return nil, err
}
@@ -50,17 +46,20 @@ func (l *LogAlertRule) ToSource(isNotify bool) (*config.Source, error) {
IsNotification: isNotify && l.Notify,
Patterns: patterns,
}
if group != nil {
source.AlertRule.Group = group
}
return source, nil
}
func (l *LogAlertRule) validate() error {
if l.Name == "" {
return fmt.Errorf("name is empty")
return fmt.Errorf("alert name is empty")
}
if !reName.MatchString(l.Name) {
return fmt.Errorf("invalid name")
return fmt.Errorf("alert invalid name: %s", l.Name)
}
return nil