update sql

* feat: implementing relabel

alert/dispatch/consume.go

@@ -14,7 +14,9 @@ import (
 	"github.com/ccfos/nightingale/v6/pkg/poster"
 	promsdk "github.com/ccfos/nightingale/v6/pkg/prom"
 	"github.com/ccfos/nightingale/v6/prom"
+	"github.com/ccfos/nightingale/v6/pushgw/writer"
 
+	"github.com/prometheus/prometheus/prompb"
 	"github.com/toolkits/pkg/concurrent/semaphore"
 	"github.com/toolkits/pkg/logger"
 )
@@ -93,6 +95,8 @@ func (e *Consumer) consumeOne(event *models.AlertCurEvent) {
 		event.RuleNote = fmt.Sprintf("failed to parse rule note: %v", err)
 	}
 
+	e.relabel(event)
+
 	e.persist(event)
 
 	if event.IsRecovered && event.NotifyRecovered == 0 {
@@ -186,6 +190,37 @@ func (e *Consumer) queryRecoveryVal(event *models.AlertCurEvent) {
 	}
 }
 
+func (e *Consumer) relabel(event *models.AlertCurEvent) {
+	rule := e.dispatch.alertRuleCache.Get(event.RuleId)
+	if rule == nil {
+		return
+	}
+
+	// need to keep the original label
+	event.OriginalTags = event.Tags
+	event.OriginalTagsJSON = make([]string, len(event.TagsJSON))
+
+	labels := make([]prompb.Label, len(event.TagsJSON))
+	for i, tag := range event.TagsJSON {
+		label := strings.Split(tag, "=")
+		if len(label) != 2 {
+			logger.Errorf("event%+v relabel: the label length is not 2:%v", event, label)
+			continue
+		}
+		event.OriginalTagsJSON[i] = tag
+		labels[i] = prompb.Label{Name: label[0], Value: label[1]}
+	}
+
+	// relabel process
+	relabels := writer.Process(labels, rule.EventRelabelConfig...)
+	event.TagsJSON = make([]string, len(relabels))
+	for i, label := range relabels {
+		event.TagsJSON[i] = fmt.Sprintf("%s=%s", label.Name, label.Value)
+		event.TagsMap[label.Name] = label.Value
+	}
+	event.Tags = strings.Join(event.TagsJSON, ",,")
+}
+
 func getKey(event *models.AlertCurEvent) string {
 	return common.RuleKey(event.DatasourceId, event.RuleId)
 }

center/router/router.go

@@ -314,6 +314,7 @@ func (rt *Router) Config(r *gin.Engine) {
 		pages.GET("/alert-rule/:arid", rt.auth(), rt.user(), rt.perm("/alert-rules"), rt.alertRuleGet)
 		pages.GET("/alert-rule/:arid/pure", rt.auth(), rt.user(), rt.perm("/alert-rules"), rt.alertRulePureGet)
 		pages.PUT("/busi-group/alert-rule/validate", rt.auth(), rt.user(), rt.perm("/alert-rules/put"), rt.alertRuleValidation)
+		pages.POST("/relabel-test", rt.auth(), rt.user(), rt.relabelTest)
 
 		pages.GET("/busi-groups/recording-rules", rt.auth(), rt.user(), rt.perm("/recording-rules"), rt.recordingRuleGetsByGids)
 		pages.GET("/busi-group/:id/recording-rules", rt.auth(), rt.user(), rt.perm("/recording-rules"), rt.recordingRuleGets)

center/router/router_alert_rule.go

@@ -1,14 +1,18 @@
 package router
 
 import (
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pushgw/pconf"
+	"github.com/ccfos/nightingale/v6/pushgw/writer"
 
 	"github.com/gin-gonic/gin"
+	"github.com/prometheus/prometheus/prompb"
 	"github.com/toolkits/pkg/ginx"
 	"github.com/toolkits/pkg/i18n"
 	"github.com/toolkits/pkg/str"
@@ -402,3 +406,36 @@ func (rt *Router) alertRuleCallbacks(c *gin.Context) {
 
 	ginx.NewRender(c).Data(callbacks, nil)
 }
+
+type alertRuleTestForm struct {
+	Configs []*pconf.RelabelConfig `json:"configs"`
+	Tags    []string               `json:"tags"`
+}
+
+func (rt *Router) relabelTest(c *gin.Context) {
+	var f alertRuleTestForm
+	ginx.BindJSON(c, &f)
+
+	if len(f.Tags) == 0 || len(f.Configs) == 0 {
+		ginx.Bomb(http.StatusBadRequest, "relabel config is empty")
+	}
+
+	labels := make([]prompb.Label, len(f.Tags))
+	for i, tag := range f.Tags {
+		label := strings.Split(tag, "=")
+		if len(label) != 2 {
+			ginx.Bomb(http.StatusBadRequest, "tag:%s format error", tag)
+		}
+
+		labels[i] = prompb.Label{Name: label[0], Value: label[1]}
+	}
+
+	relabels := writer.Process(labels, f.Configs...)
+
+	var tags []string
+	for _, label := range relabels {
+		tags = append(tags, fmt.Sprintf("%s=%s", label.Name, label.Value))
+	}
+
+	ginx.NewRender(c).Data(tags, nil)
+}

docker/initsql/a-n9e.sql

@@ -441,7 +441,7 @@ CREATE TABLE `alert_cur_event` (
     `prom_for_duration` int not null comment 'prometheus for, unit:s',
     `prom_ql` varchar(8192) not null comment 'promql',
     `prom_eval_interval` int not null comment 'evaluate interval',
-    `callbacks` varchar(255) not null default '' comment 'split by space: http://a.com/api/x http://a.com/api/y',
+    `callbacks` varchar(2048) not null default '' comment 'split by space: http://a.com/api/x http://a.com/api/y',
     `runbook_url` varchar(255),
     `notify_recovered` tinyint(1) not null comment 'whether notify when recovery',
     `notify_channels` varchar(255) not null default '' comment 'split by space: sms voice email dingtalk wecom',
@@ -456,6 +456,7 @@ CREATE TABLE `alert_cur_event` (
     `annotations` text not null comment 'annotations',
     `rule_config` text not null comment 'annotations',
     `tags` varchar(1024) not null default '' comment 'merge data_tags rule_tags, split by ,,',
+    `original_tags` varchar(1024) default '' comment 'labels key=val,,k2=v2',
     PRIMARY KEY (`id`),
     KEY (`hash`),
     KEY (`rule_id`),
@@ -481,7 +482,7 @@ CREATE TABLE `alert_his_event` (
     `prom_for_duration` int not null comment 'prometheus for, unit:s',
     `prom_ql` varchar(8192) not null comment 'promql',
     `prom_eval_interval` int not null comment 'evaluate interval',
-    `callbacks` varchar(255) not null default '' comment 'split by space: http://a.com/api/x http://a.com/api/y',
+    `callbacks` varchar(2048) not null default '' comment 'split by space: http://a.com/api/x http://a.com/api/y',
     `runbook_url` varchar(255),
     `notify_recovered` tinyint(1) not null comment 'whether notify when recovery',
     `notify_channels` varchar(255) not null default '' comment 'split by space: sms voice email dingtalk wecom',
@@ -495,6 +496,7 @@ CREATE TABLE `alert_his_event` (
     `recover_time` bigint not null default 0,
     `last_eval_time` bigint not null default 0 comment 'for time filter',
     `tags` varchar(1024) not null default '' comment 'merge data_tags rule_tags, split by ,,',
+    `original_tags` varchar(1024) default '' comment 'labels key=val,,k2=v2',
     `annotations` text not null comment 'annotations',
     `rule_config` text not null comment 'annotations',
     PRIMARY KEY (`id`),

docker/migratesql/migrate.sql

@@ -79,4 +79,8 @@ CREATE TABLE `builtin_payloads` (
 ALTER TABLE users ADD COLUMN last_active_time BIGINT NOT NULL DEFAULT 0;
 
 /* v7.0.0-beta.13 */
-ALTER TABLE recording_rule ADD COLUMN cron_pattern VARCHAR(255) DEFAULT '' COMMENT 'cron pattern';
+ALTER TABLE recording_rule ADD COLUMN cron_pattern VARCHAR(255) DEFAULT '' COMMENT 'cron pattern';
+
+/* v7.0.0-beta.14 */
+ALTER TABLE alert_cur_event ADD COLUMN original_tags VARCHAR(1024) DEFAULT '' COMMENT 'labels key=val,,k2=v2';
+ALTER TABLE alert_his_event ADD COLUMN original_tags VARCHAR(1024) DEFAULT '' COMMENT 'labels key=val,,k2=v2';

models/alert_cur_event.go

@@ -52,6 +52,8 @@ type AlertCurEvent struct {
 	Tags               string            `json:"-"`                         // for db
 	TagsJSON           []string          `json:"tags" gorm:"-"`             // for fe
 	TagsMap            map[string]string `json:"tags_map" gorm:"-"`         // for internal usage
+	OriginalTags       string            `json:"-"`                         // for db
+	OriginalTagsJSON   []string          `json:"original_tags" gorm:"-"`    // for fe
 	Annotations        string            `json:"-"`                         //
 	AnnotationsJSON    map[string]string `json:"annotations" gorm:"-"`      // for fe
 	IsRecovered        bool              `json:"is_recovered" gorm:"-"`     // for notify.py
@@ -289,6 +291,7 @@ func (e *AlertCurEvent) ToHis(ctx *ctx.Context) *AlertHisEvent {
 		TriggerTime:      e.TriggerTime,
 		TriggerValue:     e.TriggerValue,
 		Tags:             e.Tags,
+		OriginalTags:     e.OriginalTags,
 		RecoverTime:      recoverTime,
 		LastEvalTime:     e.LastEvalTime,
 		NotifyCurNumber:  e.NotifyCurNumber,
@@ -301,6 +304,7 @@ func (e *AlertCurEvent) DB2FE() error {
 	e.NotifyGroupsJSON = strings.Fields(e.NotifyGroups)
 	e.CallbacksJSON = strings.Fields(e.Callbacks)
 	e.TagsJSON = strings.Split(e.Tags, ",,")
+	e.OriginalTagsJSON = strings.Split(e.OriginalTags, ",,")
 	json.Unmarshal([]byte(e.Annotations), &e.AnnotationsJSON)
 	json.Unmarshal([]byte(e.RuleConfig), &e.RuleConfigJson)
 	return nil
@@ -311,6 +315,7 @@ func (e *AlertCurEvent) FE2DB() {
 	e.NotifyGroups = strings.Join(e.NotifyGroupsJSON, " ")
 	e.Callbacks = strings.Join(e.CallbacksJSON, " ")
 	e.Tags = strings.Join(e.TagsJSON, ",,")
+	e.OriginalTags = strings.Join(e.OriginalTagsJSON, ",,")
 	b, _ := json.Marshal(e.AnnotationsJSON)
 	e.Annotations = string(b)
 

models/alert_his_event.go

@@ -48,6 +48,8 @@ type AlertHisEvent struct {
 	LastEvalTime       int64             `json:"last_eval_time"`
 	Tags               string            `json:"-"`
 	TagsJSON           []string          `json:"tags" gorm:"-"`
+	OriginalTags       string            `json:"-"`                       // for db
+	OriginalTagsJSON   []string          `json:"original_tags"  gorm:"-"` // for fe
 	Annotations        string            `json:"-"`
 	AnnotationsJSON    map[string]string `json:"annotations" gorm:"-"` // for fe
 	NotifyCurNumber    int               `json:"notify_cur_number"`    // notify: current number
@@ -68,6 +70,7 @@ func (e *AlertHisEvent) DB2FE() {
 	e.NotifyGroupsJSON = strings.Fields(e.NotifyGroups)
 	e.CallbacksJSON = strings.Fields(e.Callbacks)
 	e.TagsJSON = strings.Split(e.Tags, ",,")
+	e.OriginalTagsJSON = strings.Split(e.OriginalTags, ",,")
 
 	if len(e.Annotations) > 0 {
 		err := json.Unmarshal([]byte(e.Annotations), &e.AnnotationsJSON)

models/alert_rule.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/poster"
+	"github.com/ccfos/nightingale/v6/pushgw/pconf"
 
 	"github.com/pkg/errors"
 	"github.com/toolkits/pkg/logger"
@@ -26,60 +27,61 @@ const (
 )
 
 type AlertRule struct {
-	Id                    int64             `json:"id" gorm:"primaryKey"`
-	GroupId               int64             `json:"group_id"`                      // busi group id
-	Cate                  string            `json:"cate"`                          // alert rule cate (prometheus|elasticsearch)
-	DatasourceIds         string            `json:"-" gorm:"datasource_ids"`       // datasource ids
-	DatasourceIdsJson     []int64           `json:"datasource_ids" gorm:"-"`       // for fe
-	Cluster               string            `json:"cluster"`                       // take effect by clusters, seperated by space
-	Name                  string            `json:"name"`                          // rule name
-	Note                  string            `json:"note"`                          // will sent in notify
-	Prod                  string            `json:"prod"`                          // product empty means n9e
-	Algorithm             string            `json:"algorithm"`                     // algorithm (''|holtwinters), empty means threshold
-	AlgoParams            string            `json:"-" gorm:"algo_params"`          // params algorithm need
-	AlgoParamsJson        interface{}       `json:"algo_params" gorm:"-"`          // for fe
-	Delay                 int               `json:"delay"`                         // Time (in seconds) to delay evaluation
-	Severity              int               `json:"severity"`                      // 1: Emergency 2: Warning 3: Notice
-	Severities            []int             `json:"severities" gorm:"-"`           // 1: Emergency 2: Warning 3: Notice
-	Disabled              int               `json:"disabled"`                      // 0: enabled, 1: disabled
-	PromForDuration       int               `json:"prom_for_duration"`             // prometheus for, unit:s
-	PromQl                string            `json:"prom_ql"`                       // just one ql
-	RuleConfig            string            `json:"-" gorm:"rule_config"`          // rule config
-	RuleConfigJson        interface{}       `json:"rule_config" gorm:"-"`          // rule config for fe
-	PromEvalInterval      int               `json:"prom_eval_interval"`            // unit:s
-	EnableStime           string            `json:"-"`                             // split by space: "00:00 10:00 12:00"
-	EnableStimeJSON       string            `json:"enable_stime" gorm:"-"`         // for fe
-	EnableStimesJSON      []string          `json:"enable_stimes" gorm:"-"`        // for fe
-	EnableEtime           string            `json:"-"`                             // split by space: "00:00 10:00 12:00"
-	EnableEtimeJSON       string            `json:"enable_etime" gorm:"-"`         // for fe
-	EnableEtimesJSON      []string          `json:"enable_etimes" gorm:"-"`        // for fe
-	EnableDaysOfWeek      string            `json:"-"`                             // eg: "0 1 2 3 4 5 6 ; 0 1 2"
-	EnableDaysOfWeekJSON  []string          `json:"enable_days_of_week" gorm:"-"`  // for fe
-	EnableDaysOfWeeksJSON [][]string        `json:"enable_days_of_weeks" gorm:"-"` // for fe
-	EnableInBG            int               `json:"enable_in_bg"`                  // 0: global 1: enable one busi-group
-	NotifyRecovered       int               `json:"notify_recovered"`              // whether notify when recovery
-	NotifyChannels        string            `json:"-"`                             // split by space: sms voice email dingtalk wecom
-	NotifyChannelsJSON    []string          `json:"notify_channels" gorm:"-"`      // for fe
-	NotifyGroups          string            `json:"-"`                             // split by space: 233 43
-	NotifyGroupsObj       []UserGroup       `json:"notify_groups_obj" gorm:"-"`    // for fe
-	NotifyGroupsJSON      []string          `json:"notify_groups" gorm:"-"`        // for fe
-	NotifyRepeatStep      int               `json:"notify_repeat_step"`            // notify repeat interval, unit: min
-	NotifyMaxNumber       int               `json:"notify_max_number"`             // notify: max number
-	RecoverDuration       int64             `json:"recover_duration"`              // unit: s
-	Callbacks             string            `json:"-"`                             // split by space: http://a.com/api/x http://a.com/api/y'
-	CallbacksJSON         []string          `json:"callbacks" gorm:"-"`            // for fe
-	RunbookUrl            string            `json:"runbook_url"`                   // sop url
-	AppendTags            string            `json:"-"`                             // split by space: service=n9e mod=api
-	AppendTagsJSON        []string          `json:"append_tags" gorm:"-"`          // for fe
-	Annotations           string            `json:"-"`                             //
-	AnnotationsJSON       map[string]string `json:"annotations" gorm:"-"`          // for fe
-	ExtraConfig           string            `json:"-" gorm:"extra_config"`         // extra config
-	ExtraConfigJSON       interface{}       `json:"extra_config" gorm:"-"`         // for fe
-	CreateAt              int64             `json:"create_at"`
-	CreateBy              string            `json:"create_by"`
-	UpdateAt              int64             `json:"update_at"`
-	UpdateBy              string            `json:"update_by"`
-	UUID                  int64             `json:"uuid" gorm:"-"` // tpl identifier
+	Id                    int64                  `json:"id" gorm:"primaryKey"`
+	GroupId               int64                  `json:"group_id"`                      // busi group id
+	Cate                  string                 `json:"cate"`                          // alert rule cate (prometheus|elasticsearch)
+	DatasourceIds         string                 `json:"-" gorm:"datasource_ids"`       // datasource ids
+	DatasourceIdsJson     []int64                `json:"datasource_ids" gorm:"-"`       // for fe
+	Cluster               string                 `json:"cluster"`                       // take effect by clusters, seperated by space
+	Name                  string                 `json:"name"`                          // rule name
+	Note                  string                 `json:"note"`                          // will sent in notify
+	Prod                  string                 `json:"prod"`                          // product empty means n9e
+	Algorithm             string                 `json:"algorithm"`                     // algorithm (''|holtwinters), empty means threshold
+	AlgoParams            string                 `json:"-" gorm:"algo_params"`          // params algorithm need
+	AlgoParamsJson        interface{}            `json:"algo_params" gorm:"-"`          // for fe
+	Delay                 int                    `json:"delay"`                         // Time (in seconds) to delay evaluation
+	Severity              int                    `json:"severity"`                      // 1: Emergency 2: Warning 3: Notice
+	Severities            []int                  `json:"severities" gorm:"-"`           // 1: Emergency 2: Warning 3: Notice
+	Disabled              int                    `json:"disabled"`                      // 0: enabled, 1: disabled
+	PromForDuration       int                    `json:"prom_for_duration"`             // prometheus for, unit:s
+	PromQl                string                 `json:"prom_ql"`                       // just one ql
+	RuleConfig            string                 `json:"-" gorm:"rule_config"`          // rule config
+	RuleConfigJson        interface{}            `json:"rule_config" gorm:"-"`          // rule config for fe
+	EventRelabelConfig    []*pconf.RelabelConfig `json:"event_relabel_config" gorm:"-"` // event relabel config
+	PromEvalInterval      int                    `json:"prom_eval_interval"`            // unit:s
+	EnableStime           string                 `json:"-"`                             // split by space: "00:00 10:00 12:00"
+	EnableStimeJSON       string                 `json:"enable_stime" gorm:"-"`         // for fe
+	EnableStimesJSON      []string               `json:"enable_stimes" gorm:"-"`        // for fe
+	EnableEtime           string                 `json:"-"`                             // split by space: "00:00 10:00 12:00"
+	EnableEtimeJSON       string                 `json:"enable_etime" gorm:"-"`         // for fe
+	EnableEtimesJSON      []string               `json:"enable_etimes" gorm:"-"`        // for fe
+	EnableDaysOfWeek      string                 `json:"-"`                             // eg: "0 1 2 3 4 5 6 ; 0 1 2"
+	EnableDaysOfWeekJSON  []string               `json:"enable_days_of_week" gorm:"-"`  // for fe
+	EnableDaysOfWeeksJSON [][]string             `json:"enable_days_of_weeks" gorm:"-"` // for fe
+	EnableInBG            int                    `json:"enable_in_bg"`                  // 0: global 1: enable one busi-group
+	NotifyRecovered       int                    `json:"notify_recovered"`              // whether notify when recovery
+	NotifyChannels        string                 `json:"-"`                             // split by space: sms voice email dingtalk wecom
+	NotifyChannelsJSON    []string               `json:"notify_channels" gorm:"-"`      // for fe
+	NotifyGroups          string                 `json:"-"`                             // split by space: 233 43
+	NotifyGroupsObj       []UserGroup            `json:"notify_groups_obj" gorm:"-"`    // for fe
+	NotifyGroupsJSON      []string               `json:"notify_groups" gorm:"-"`        // for fe
+	NotifyRepeatStep      int                    `json:"notify_repeat_step"`            // notify repeat interval, unit: min
+	NotifyMaxNumber       int                    `json:"notify_max_number"`             // notify: max number
+	RecoverDuration       int64                  `json:"recover_duration"`              // unit: s
+	Callbacks             string                 `json:"-"`                             // split by space: http://a.com/api/x http://a.com/api/y'
+	CallbacksJSON         []string               `json:"callbacks" gorm:"-"`            // for fe
+	RunbookUrl            string                 `json:"runbook_url"`                   // sop url
+	AppendTags            string                 `json:"-"`                             // split by space: service=n9e mod=api
+	AppendTagsJSON        []string               `json:"append_tags" gorm:"-"`          // for fe
+	Annotations           string                 `json:"-"`                             //
+	AnnotationsJSON       map[string]string      `json:"annotations" gorm:"-"`          // for fe
+	ExtraConfig           string                 `json:"-" gorm:"extra_config"`         // extra config
+	ExtraConfigJSON       interface{}            `json:"extra_config" gorm:"-"`         // for fe
+	CreateAt              int64                  `json:"create_at"`
+	CreateBy              string                 `json:"create_by"`
+	UpdateAt              int64                  `json:"update_at"`
+	UpdateBy              string                 `json:"update_by"`
+	UUID                  int64                  `json:"uuid" gorm:"-"` // tpl identifier
 }
 
 type PromRuleConfig struct {
@@ -622,6 +624,13 @@ func (ar *AlertRule) DB2FE() error {
 	json.Unmarshal([]byte(ar.Annotations), &ar.AnnotationsJSON)
 	json.Unmarshal([]byte(ar.ExtraConfig), &ar.ExtraConfigJSON)
 
+	// 解析 RuleConfig 字段
+	var ruleConfig struct {
+		EventRelabelConfig []*pconf.RelabelConfig `json:"event_relabel_config"`
+	}
+	json.Unmarshal([]byte(ar.RuleConfig), &ruleConfig)
+	ar.EventRelabelConfig = ruleConfig.EventRelabelConfig
+
 	err := ar.FillDatasourceIds()
 	return err
 }

models/migrate/migrate.go

@@ -60,8 +60,23 @@ func MigrateTables(db *gorm.DB) error {
 		&Board{}, &BoardBusigroup{}, &Users{}, &SsoConfig{}, &models.BuiltinMetric{},
 		&models.MetricFilter{}, &models.BuiltinComponent{}}
 
-	if !columnHasIndex(db, &AlertHisEvent{}, "last_eval_time") {
-		dts = append(dts, &AlertHisEvent{})
+	if !columnHasIndex(db, &AlertHisEvent{}, "original_tags") ||
+		!columnHasIndex(db, &AlertCurEvent{}, "original_tags") {
+		asyncDts := []interface{}{&AlertHisEvent{}, &AlertCurEvent{}}
+
+		go func() {
+			defer func() {
+				if r := recover(); r != nil {
+					logger.Errorf("panic to migrate table: %v", r)
+				}
+			}()
+
+			for _, dt := range asyncDts {
+				if err := db.AutoMigrate(dt); err != nil {
+					logger.Errorf("failed to migrate table: %v", err)
+				}
+			}
+		}()
 	}
 
 	if !db.Migrator().HasTable(&models.BuiltinPayload{}) {
@@ -203,8 +218,14 @@ type TaskRecord struct {
 	EventId int64 `gorm:"column:event_id;bigint(20);not null;default:0;comment:event id;index:idx_event_id"`
 }
 type AlertHisEvent struct {
-	LastEvalTime int64 `gorm:"column:last_eval_time;bigint(20);not null;default:0;comment:for time filter;index:idx_last_eval_time"`
+	LastEvalTime int64  `gorm:"column:last_eval_time;bigint(20);not null;default:0;comment:for time filter;index:idx_last_eval_time"`
+	OriginalTags string `gorm:"column:original_tags;type:varchar(1024);default:'';comment:labels key=val,,k2=v2"`
 }
+
+type AlertCurEvent struct {
+	OriginalTags string `gorm:"column:original_tags;type:varchar(1024);default:'';comment:labels key=val,,k2=v2"`
+}
+
 type Target struct {
 	HostIp       string `gorm:"column:host_ip;varchar(15);default:'';comment:IPv4 string;index:idx_host_ip"`
 	AgentVersion string `gorm:"column:agent_version;varchar(255);default:'';comment:agent version;index:idx_agent_version"`

pushgw/pconf/conf.go

@@ -52,14 +52,16 @@ type WriterOptions struct {
 }
 
 type RelabelConfig struct {
-	SourceLabels  model.LabelNames
-	Separator     string
-	Regex         string
+	SourceLabels  model.LabelNames `json:"source_labels"`
+	Separator     string           `json:"separator"`
+	Regex         string           `json:"regex"`
 	RegexCompiled *regexp.Regexp
-	Modulus       uint64
-	TargetLabel   string
-	Replacement   string
-	Action        string
+	If            string `json:"if"`
+	IfRegex       *regexp.Regexp
+	Modulus       uint64 `json:"modulus"`
+	TargetLabel   string `json:"target_label"`
+	Replacement   string `json:"replacement"`
+	Action        string `json:"action"`
 }
 
 func (p *Pushgw) PreCheck() {

pushgw/writer/relabel.go

@@ -3,24 +3,28 @@ package writer
 import (
 	"crypto/md5"
 	"fmt"
+	"regexp"
 	"sort"
 	"strings"
 
 	"github.com/ccfos/nightingale/v6/pushgw/pconf"
+	"github.com/toolkits/pkg/logger"
+
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/prompb"
 )
 
 const (
-	Replace   string = "replace"
-	Keep      string = "keep"
-	Drop      string = "drop"
-	HashMod   string = "hashmod"
-	LabelMap  string = "labelmap"
-	LabelDrop string = "labeldrop"
-	LabelKeep string = "labelkeep"
-	Lowercase string = "lowercase"
-	Uppercase string = "uppercase"
+	Replace     string = "replace"
+	Keep        string = "keep"
+	Drop        string = "drop"
+	HashMod     string = "hashmod"
+	LabelMap    string = "labelmap"
+	LabelDrop   string = "labeldrop"
+	LabelKeep   string = "labelkeep"
+	Lowercase   string = "lowercase"
+	Uppercase   string = "uppercase"
+	DropIfEqual string = "drop_if_equal"
 )
 
 func Process(labels []prompb.Label, cfgs ...*pconf.RelabelConfig) []prompb.Label {
@@ -55,10 +59,6 @@ func newBuilder(ls []prompb.Label) *LabelBuilder {
 }
 
 func (l *LabelBuilder) set(k, v string) *LabelBuilder {
-	if v == "" {
-		return l.del(k)
-	}
-
 	l.LabelSet[k] = v
 	return l
 }
@@ -96,9 +96,17 @@ func relabel(lset []prompb.Label, cfg *pconf.RelabelConfig) []prompb.Label {
 	}
 
 	regx := cfg.RegexCompiled
+	if regx == nil {
+		regx = compileRegex(cfg.Regex)
+	}
+
+	if regx == nil {
+		return lset
+	}
 
 	val := strings.Join(values, cfg.Separator)
 	lb := newBuilder(lset)
+
 	switch cfg.Action {
 	case Drop:
 		if regx.MatchString(val) {
@@ -109,21 +117,7 @@ func relabel(lset []prompb.Label, cfg *pconf.RelabelConfig) []prompb.Label {
 			return nil
 		}
 	case Replace:
-		indexes := regx.FindStringSubmatchIndex(val)
-		if indexes == nil {
-			break
-		}
-		target := model.LabelName(regx.ExpandString([]byte{}, cfg.TargetLabel, val, indexes))
-		if !target.IsValid() {
-			lb.del(cfg.TargetLabel)
-			break
-		}
-		res := regx.ExpandString([]byte{}, cfg.Replacement, val, indexes)
-		if len(res) == 0 {
-			lb.del(cfg.TargetLabel)
-			break
-		}
-		lb.set(string(target), string(res))
+		return handleReplace(lb, regx, cfg, val, lset)
 	case Lowercase:
 		lb.set(cfg.TargetLabel, strings.ToLower(val))
 	case Uppercase:
@@ -150,13 +144,84 @@ func relabel(lset []prompb.Label, cfg *pconf.RelabelConfig) []prompb.Label {
 				lb.del(l.Name)
 			}
 		}
+	case DropIfEqual:
+		return handleDropIfEqual(lb, cfg, lset)
 	default:
-		panic(fmt.Errorf("relabel: unknown relabel action type %q", cfg.Action))
+		logger.Errorf("relabel: unknown relabel action type %q", cfg.Action)
 	}
 
 	return lb.labels()
 }
 
+func handleReplace(lb *LabelBuilder, regx *regexp.Regexp, cfg *pconf.RelabelConfig, val string, lset []prompb.Label) []prompb.Label {
+	// 如果没有 source_labels，直接设置标签（新增标签）
+	if len(cfg.SourceLabels) == 0 {
+		lb.set(cfg.TargetLabel, cfg.Replacement)
+		return lb.labels()
+	}
+
+	// 如果 Replacement 为空, separator 不为空, 则用已有标签构建新标签
+	if cfg.Replacement == "" && len(cfg.SourceLabels) > 1 {
+		lb.set(cfg.TargetLabel, val)
+		return lb.labels()
+	}
+
+	// 处理正则表达式替换的情况（修改标签值，正则）
+	if regx != nil {
+		indexes := regx.FindStringSubmatchIndex(val)
+		if indexes == nil {
+			return lb.labels()
+		}
+
+		target := model.LabelName(cfg.TargetLabel)
+		if !target.IsValid() {
+			lb.del(cfg.TargetLabel)
+			return lb.labels()
+		}
+
+		res := regx.ExpandString([]byte{}, cfg.Replacement, val, indexes)
+		if len(res) == 0 {
+			lb.del(cfg.TargetLabel)
+		} else {
+			lb.set(string(target), string(res))
+		}
+
+		return lb.labels()
+	}
+
+	// 默认情况，直接设置目标标签值
+	lb.set(cfg.TargetLabel, cfg.Replacement)
+	return lb.labels()
+}
+
+func handleDropIfEqual(lb *LabelBuilder, cfg *pconf.RelabelConfig, lset []prompb.Label) []prompb.Label {
+	if len(cfg.SourceLabels) < 2 {
+		return lb.labels()
+	}
+	firstVal := getValue(lset, cfg.SourceLabels[0])
+	equal := true
+	for _, label := range cfg.SourceLabels[1:] {
+		if getValue(lset, label) != firstVal {
+			equal = false
+			break
+		}
+	}
+	if equal {
+		return nil
+	}
+	return lb.labels()
+}
+
+func compileRegex(expr string) *regexp.Regexp {
+	regex, err := regexp.Compile(expr)
+	if err != nil {
+		logger.Error("failed to compile regexp:", expr, "error:", err)
+		return nil
+	}
+
+	return regex
+}
+
 func sum64(hash [md5.Size]byte) uint64 {
 	var s uint64
 

pushgw/writer/relabel_test.go

@@ -0,0 +1,406 @@
+// @Author: Ciusyan 6/19/24
+
+package writer
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/ccfos/nightingale/v6/pushgw/pconf"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/prompb"
+)
+
+func TestProcess(t *testing.T) {
+	tests := []struct {
+		name     string
+		labels   []prompb.Label
+		cfgs     []*pconf.RelabelConfig
+		expected []prompb.Label
+	}{
+		// 1. 添加新标签 (Adding new label)
+		{
+			name:   "Adding new label",
+			labels: []prompb.Label{{Name: "job", Value: "aa"}},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "replace",
+					TargetLabel: "foo",
+					Replacement: "bar",
+				},
+			},
+			expected: []prompb.Label{{Name: "job", Value: "aa"}, {Name: "foo", Value: "bar"}},
+		},
+		// 2. 更新现有标签 (Updating existing label)
+		{
+			name:   "Updating existing label",
+			labels: []prompb.Label{{Name: "foo", Value: "aaaa"}},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "replace",
+					TargetLabel: "foo",
+					Replacement: "bar",
+				},
+			},
+			expected: []prompb.Label{{Name: "foo", Value: "bar"}},
+		},
+		// 3. 重写现有标签 (Rewriting existing label)
+		{
+			name:   "Rewriting existing label",
+			labels: []prompb.Label{{Name: "instance", Value: "bar:123"}},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "replace",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        "([^:]+):.+",
+					TargetLabel:  "instance",
+					Replacement:  "$1",
+				},
+			},
+			expected: []prompb.Label{{Name: "instance", Value: "bar"}},
+		},
+		{
+			name:   "Rewriting existing label",
+			labels: []prompb.Label{{Name: "instance", Value: "bar:123"}},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "replace",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        ":([0-9]+)$",
+					TargetLabel:  "port",
+					Replacement:  "$1",
+				},
+			},
+			expected: []prompb.Label{{Name: "port", Value: "123"}, {Name: "instance", Value: "bar:123"}},
+		},
+		// 4. 更新度量标准名称 (Updating metric name)
+		{
+			name:   "Updating metric name",
+			labels: []prompb.Label{{Name: "__name__", Value: "foo_suffix"}},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "replace",
+					SourceLabels: model.LabelNames{"__name__"},
+					Regex:        "(.+)_suffix",
+					TargetLabel:  "__name__",
+					Replacement:  "prefix_$1",
+				},
+			},
+			expected: []prompb.Label{{Name: "__name__", Value: "prefix_foo"}},
+		},
+		// 5. 删除不需要/保持需要 的标签 (Removing unneeded labels)
+		{
+			name: "Removing unneeded labels",
+			labels: []prompb.Label{
+				{Name: "job", Value: "a"},
+				{Name: "instance", Value: "xyz"},
+				{Name: "foobar", Value: "baz"},
+				{Name: "foox", Value: "aaa"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action: "labeldrop",
+					Regex:  "foo.+",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "job", Value: "a"},
+				{Name: "instance", Value: "xyz"},
+			},
+		},
+		{
+			name: "keep needed labels",
+			labels: []prompb.Label{
+				{Name: "job", Value: "a"},
+				{Name: "instance", Value: "xyz"},
+				{Name: "foobar", Value: "baz"},
+				{Name: "foox", Value: "aaa"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action: "labelkeep",
+					Regex:  "foo.+",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "foobar", Value: "baz"},
+				{Name: "foox", Value: "aaa"},
+			},
+		},
+		// 6. 删除特定标签值 (Removing the specific label value)
+		{
+			name: "Removing the specific label value",
+			labels: []prompb.Label{
+				{Name: "foo", Value: "bar"},
+				{Name: "baz", Value: "x"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "replace",
+					SourceLabels: model.LabelNames{"foo"},
+					Regex:        "bar",
+					TargetLabel:  "foo",
+					Replacement:  "",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "baz", Value: "x"},
+			},
+		},
+		// 7. 删除不需要的度量标准 (Removing unneeded metrics)
+		{
+			name: "Removing unneeded metrics",
+			labels: []prompb.Label{
+				{Name: "instance", Value: "foobar1"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        "foobar.+",
+				},
+			},
+			expected: nil,
+		},
+		{
+			name: "Removing unneeded metrics 2",
+			labels: []prompb.Label{
+				{Name: "instance", Value: "foobar2"},
+				{Name: "job", Value: "xxx"},
+				{Name: "aaa", Value: "bb"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        "foobar.+",
+				},
+			},
+			expected: nil,
+		},
+		{
+			name: "Removing unneeded metrics 3",
+			labels: []prompb.Label{
+				{Name: "instance", Value: "xxx"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        "foobar.+",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "instance", Value: "xxx"},
+			},
+		},
+		{
+			name: "Removing unneeded metrics 4",
+			labels: []prompb.Label{
+				{Name: "instance", Value: "abc"},
+				{Name: "job", Value: "xyz"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        "foobar.+",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "instance", Value: "abc"},
+				{Name: "job", Value: "xyz"},
+			},
+		},
+		{
+			name: "Removing unneeded metrics with multiple labels",
+			labels: []prompb.Label{
+				{Name: "job", Value: "foo"},
+				{Name: "instance", Value: "bar"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop",
+					SourceLabels: model.LabelNames{"job", "instance"},
+					Regex:        "foo;bar",
+					Separator:    ";",
+				},
+			},
+			expected: nil,
+		},
+		// 8. 按条件删除度量标准 (Dropping metrics on certain condition)
+		{
+			name: "Dropping metrics on certain condition",
+			labels: []prompb.Label{
+				{Name: "real_port", Value: "123"},
+				{Name: "needed_port", Value: "123"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop_if_equal",
+					SourceLabels: model.LabelNames{"real_port", "needed_port"},
+				},
+			},
+			expected: nil,
+		},
+		{
+			name: "Dropping metrics on certain condition 2",
+			labels: []prompb.Label{
+				{Name: "real_port", Value: "123"},
+				{Name: "needed_port", Value: "456"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "drop_if_equal",
+					SourceLabels: model.LabelNames{"real_port", "needed_port"},
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "real_port", Value: "123"},
+				{Name: "needed_port", Value: "456"},
+			},
+		},
+		// 9. 修改标签名称 (Modifying label names)
+		{
+			name: "Modifying label names",
+			labels: []prompb.Label{
+				{Name: "foo_xx", Value: "bb"},
+				{Name: "job", Value: "qq"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "labelmap",
+					Regex:       "foo_(.+)",
+					Replacement: "bar_$1",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "foo_xx", Value: "bb"},
+				{Name: "bar_xx", Value: "bb"},
+				{Name: "job", Value: "qq"},
+			},
+		},
+		// 10. 从多个现有标签构建新标签 (Constructing a label from multiple existing labels)
+		{
+			name: "Constructing a label from multiple existing labels",
+			labels: []prompb.Label{
+				{Name: "host", Value: "hostname"},
+				{Name: "port", Value: "9090"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:       "replace",
+					SourceLabels: model.LabelNames{"host", "port"},
+					Separator:    ":",
+					TargetLabel:  "address",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "host", Value: "hostname"},
+				{Name: "port", Value: "9090"},
+				{Name: "address", Value: "hostname:9090"},
+			},
+		},
+		// 11. 链式重标记规则 (Chaining relabeling rules)
+		{
+			name: "Chaining relabeling rules",
+			labels: []prompb.Label{
+				{Name: "instance", Value: "hostname:9090"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "replace",
+					TargetLabel: "foo",
+					Replacement: "bar",
+				},
+				{
+					Action:       "replace",
+					SourceLabels: model.LabelNames{"instance"},
+					Regex:        "([^:]+):.*",
+					TargetLabel:  "instance",
+					Replacement:  "$1",
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "instance", Value: "hostname"},
+				{Name: "foo", Value: "bar"},
+			},
+		},
+		// 12. 条件重标记 (Conditional relabeling)
+		{
+			name: "Conditional relabeling matches",
+			labels: []prompb.Label{
+				{Name: "label", Value: "x"},
+				{Name: "foo", Value: "aaa"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "replace",
+					If:          `label="x|y"`,
+					TargetLabel: "foo",
+					Replacement: "bar",
+					IfRegex:     compileRegex(`label="x|y"`),
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "label", Value: "x"},
+				{Name: "foo", Value: "bar"},
+			},
+		},
+		{
+			name: "Conditional relabeling matches alternative",
+			labels: []prompb.Label{
+				{Name: "label", Value: "y"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "replace",
+					If:          `label="x|y"`,
+					TargetLabel: "foo",
+					Replacement: "bar",
+					IfRegex:     compileRegex(`label="x|y"`),
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "label", Value: "y"},
+				{Name: "foo", Value: "bar"},
+			},
+		},
+		{
+			name: "Conditional relabeling does not match",
+			labels: []prompb.Label{
+				{Name: "label", Value: "z"},
+			},
+			cfgs: []*pconf.RelabelConfig{
+				{
+					Action:      "replace",
+					If:          `label="x|y"`,
+					TargetLabel: "foo",
+					Replacement: "bar",
+					IfRegex:     compileRegex(`label="x|y"`),
+				},
+			},
+			expected: []prompb.Label{
+				{Name: "label", Value: "z"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := Process(tt.labels, tt.cfgs...)
+			// Sort the slices before comparison
+			sort.Slice(got, func(i, j int) bool {
+				return got[i].Name < got[j].Name
+			})
+			sort.Slice(tt.expected, func(i, j int) bool {
+				return tt.expected[i].Name < tt.expected[j].Name
+			})
+			if !reflect.DeepEqual(got, tt.expected) {
+				t.Errorf("Process() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}