Merge branch 'main' into update-alert-rule

Co-authored-by: Xu Bin <140785332+Reditiny@users.noreply.github.com>

alert/process/process.go

@@ -212,6 +212,14 @@ func (p *Processor) BuildEvent(anomalyPoint common.AnomalyPoint, from string, no
 	event.ExtraConfig = p.rule.ExtraConfigJSON
 	event.PromQl = anomalyPoint.Query
 
+	if p.target != "" {
+		if pt, exist := p.TargetCache.Get(p.target); exist {
+			event.Target = pt
+		} else {
+			logger.Infof("Target[ident: %s] doesn't exist in cache.", p.target)
+		}
+	}
+
 	if event.TriggerValues != "" && strings.Count(event.TriggerValues, "$") > 1 {
 		// TriggerValues 有多个变量，将多个变量都放到 TriggerValue 中
 		event.TriggerValue = event.TriggerValues

center/integration/init.go

@@ -16,6 +16,12 @@ import (
 const SYSTEM = "system"
 
 func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
+	err := models.InitBuiltinPayloads(ctx)
+	if err != nil {
+		logger.Warning("init old builtinPayloads fail ", err)
+		return
+	}
+
 	fp := builtinIntegrationsDir
 	if fp == "" {
 		fp = path.Join(runner.Cwd, "integrations")
@@ -92,6 +98,7 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
 					logger.Warning("update builtin component fail ", old, err)
 				}
 			}
+			component.ID = old.ID
 		}
 
 		// delete uuid is emtpy
@@ -141,13 +148,13 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
 
 					cate := strings.Replace(f, ".json", "", -1)
 					builtinAlert := models.BuiltinPayload{
-						Component: component.Ident,
-						Type:      "alert",
-						Cate:      cate,
-						Name:      alert.Name,
-						Tags:      alert.AppendTags,
-						Content:   string(content),
-						UUID:      alert.UUID,
+						ComponentID: component.ID,
+						Type:        "alert",
+						Cate:        cate,
+						Name:        alert.Name,
+						Tags:        alert.AppendTags,
+						Content:     string(content),
+						UUID:        alert.UUID,
 					}
 
 					old, err := models.BuiltinPayloadGet(ctx, "uuid = ?", alert.UUID)
@@ -165,6 +172,7 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
 					}
 
 					if old.UpdatedBy == SYSTEM {
+						old.ComponentID = component.ID
 						old.Content = string(content)
 						old.Name = alert.Name
 						old.Tags = alert.AppendTags
@@ -231,13 +239,13 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
 				}
 
 				builtinDashboard := models.BuiltinPayload{
-					Component: component.Ident,
-					Type:      "dashboard",
-					Cate:      "",
-					Name:      dashboard.Name,
-					Tags:      dashboard.Tags,
-					Content:   string(content),
-					UUID:      dashboard.UUID,
+					ComponentID: component.ID,
+					Type:        "dashboard",
+					Cate:        "",
+					Name:        dashboard.Name,
+					Tags:        dashboard.Tags,
+					Content:     string(content),
+					UUID:        dashboard.UUID,
 				}
 
 				old, err := models.BuiltinPayloadGet(ctx, "uuid = ?", dashboard.UUID)
@@ -255,6 +263,7 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
 				}
 
 				if old.UpdatedBy == SYSTEM {
+					old.ComponentID = component.ID
 					old.Content = string(content)
 					old.Name = dashboard.Name
 					old.Tags = dashboard.Tags

center/router/router_alert_rule.go

@@ -550,10 +550,17 @@ func (rt *Router) cloneToMachine(c *gin.Context) {
 	reterr := make(map[string]map[string]string)
 
 	for i := range alertRules {
-		reterr[alertRules[i].Name] = make(map[string]string)
+		errMsg := make(map[string]string)
 
 		if alertRules[i].Cate != "prometheus" {
-			reterr[alertRules[i].Name]["all"] = "Only Prometheus rules can be cloned to machines"
+			errMsg["all"] = "Only Prometheus rule can be cloned to machines"
+			reterr[alertRules[i].Name] = errMsg
+			continue
+		}
+
+		if !strings.Contains(alertRules[i].RuleConfig, "ident") {
+			errMsg["all"] = "promql is missing ident"
+			reterr[alertRules[i].Name] = errMsg
 			continue
 		}
 
@@ -562,7 +569,7 @@ func (rt *Router) cloneToMachine(c *gin.Context) {
 
 			newRule := &models.AlertRule{}
 			if err := copier.Copy(newRule, alertRules[i]); err != nil {
-				reterr[alertRules[i].Name][f.IdentList[j]] = fmt.Sprintf("fail to clone rule, err: %s", err)
+				errMsg[f.IdentList[j]] = fmt.Sprintf("fail to clone rule, err: %s", err)
 				continue
 			}
 
@@ -576,17 +583,21 @@ func (rt *Router) cloneToMachine(c *gin.Context) {
 
 			exist, err := models.AlertRuleExists(rt.Ctx, 0, newRule.GroupId, newRule.DatasourceIdsJson, newRule.Name)
 			if err != nil {
-				reterr[alertRules[i].Name][f.IdentList[j]] = err.Error()
+				errMsg[f.IdentList[j]] = err.Error()
 				continue
 			}
 
 			if exist {
-				reterr[alertRules[i].Name][f.IdentList[j]] = fmt.Sprintf("rule already exists, ruleName: %s", newRule.Name)
+				errMsg[f.IdentList[j]] = fmt.Sprintf("rule already exists, ruleName: %s", newRule.Name)
 				continue
 			}
 
 			newRules = append(newRules, newRule)
 		}
+
+		if len(errMsg) > 0 {
+			reterr[alertRules[i].Name] = errMsg
+		}
 	}
 
 	ginx.NewRender(c).Data(reterr, models.InsertAlertRule(rt.Ctx, newRules))

center/router/router_builtin_componet.go

@@ -4,10 +4,15 @@ import (
 	"net/http"
 
 	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
+
 	"github.com/gin-gonic/gin"
 	"github.com/toolkits/pkg/ginx"
+	"gorm.io/gorm"
 )
 
+const SYSTEM = "system"
+
 func (rt *Router) builtinComponentsAdd(c *gin.Context) {
 	var lst []models.BuiltinComponent
 	ginx.BindJSON(c, &lst)
@@ -50,10 +55,31 @@ func (rt *Router) builtinComponentsPut(c *gin.Context) {
 		return
 	}
 
+	if bc.CreatedBy == SYSTEM {
+		req.Ident = bc.Ident
+	}
+
 	username := Username(c)
 	req.UpdatedBy = username
 
-	ginx.NewRender(c).Message(bc.Update(rt.Ctx, req))
+	err = models.DB(rt.Ctx).Transaction(func(tx *gorm.DB) error {
+		tCtx := &ctx.Context{
+			DB: tx,
+		}
+
+		txErr := models.BuiltinMetricBatchUpdateColumn(tCtx, "typ", bc.Ident, req.Ident, req.UpdatedBy)
+		if txErr != nil {
+			return txErr
+		}
+
+		txErr = bc.Update(tCtx, req)
+		if txErr != nil {
+			return txErr
+		}
+		return nil
+	})
+
+	ginx.NewRender(c).Message(err)
 }
 
 func (rt *Router) builtinComponentsDel(c *gin.Context) {

center/router/router_builtin_payload.go

@@ -52,15 +52,15 @@ func (rt *Router) builtinPayloadsAdd(c *gin.Context) {
 					}
 
 					bp := models.BuiltinPayload{
-						Type:      lst[i].Type,
-						Component: lst[i].Component,
-						Cate:      lst[i].Cate,
-						Name:      rule.Name,
-						Tags:      rule.AppendTags,
-						UUID:      rule.UUID,
-						Content:   string(contentBytes),
-						CreatedBy: username,
-						UpdatedBy: username,
+						Type:        lst[i].Type,
+						ComponentID: lst[i].ComponentID,
+						Cate:        lst[i].Cate,
+						Name:        rule.Name,
+						Tags:        rule.AppendTags,
+						UUID:        rule.UUID,
+						Content:     string(contentBytes),
+						CreatedBy:   username,
+						UpdatedBy:   username,
 					}
 
 					if err := bp.Add(rt.Ctx, username); err != nil {
@@ -81,15 +81,15 @@ func (rt *Router) builtinPayloadsAdd(c *gin.Context) {
 			}
 
 			bp := models.BuiltinPayload{
-				Type:      lst[i].Type,
-				Component: lst[i].Component,
-				Cate:      lst[i].Cate,
-				Name:      alertRule.Name,
-				Tags:      alertRule.AppendTags,
-				UUID:      alertRule.UUID,
-				Content:   lst[i].Content,
-				CreatedBy: username,
-				UpdatedBy: username,
+				Type:        lst[i].Type,
+				ComponentID: lst[i].ComponentID,
+				Cate:        lst[i].Cate,
+				Name:        alertRule.Name,
+				Tags:        alertRule.AppendTags,
+				UUID:        alertRule.UUID,
+				Content:     lst[i].Content,
+				CreatedBy:   username,
+				UpdatedBy:   username,
 			}
 
 			if err := bp.Add(rt.Ctx, username); err != nil {
@@ -115,15 +115,15 @@ func (rt *Router) builtinPayloadsAdd(c *gin.Context) {
 					}
 
 					bp := models.BuiltinPayload{
-						Type:      lst[i].Type,
-						Component: lst[i].Component,
-						Cate:      lst[i].Cate,
-						Name:      dashboard.Name,
-						Tags:      dashboard.Tags,
-						UUID:      dashboard.UUID,
-						Content:   string(contentBytes),
-						CreatedBy: username,
-						UpdatedBy: username,
+						Type:        lst[i].Type,
+						ComponentID: lst[i].ComponentID,
+						Cate:        lst[i].Cate,
+						Name:        dashboard.Name,
+						Tags:        dashboard.Tags,
+						UUID:        dashboard.UUID,
+						Content:     string(contentBytes),
+						CreatedBy:   username,
+						UpdatedBy:   username,
 					}
 
 					if err := bp.Add(rt.Ctx, username); err != nil {
@@ -144,15 +144,15 @@ func (rt *Router) builtinPayloadsAdd(c *gin.Context) {
 			}
 
 			bp := models.BuiltinPayload{
-				Type:      lst[i].Type,
-				Component: lst[i].Component,
-				Cate:      lst[i].Cate,
-				Name:      dashboard.Name,
-				Tags:      dashboard.Tags,
-				UUID:      dashboard.UUID,
-				Content:   lst[i].Content,
-				CreatedBy: username,
-				UpdatedBy: username,
+				Type:        lst[i].Type,
+				ComponentID: lst[i].ComponentID,
+				Cate:        lst[i].Cate,
+				Name:        dashboard.Name,
+				Tags:        dashboard.Tags,
+				UUID:        dashboard.UUID,
+				Content:     lst[i].Content,
+				CreatedBy:   username,
+				UpdatedBy:   username,
 			}
 
 			if err := bp.Add(rt.Ctx, username); err != nil {
@@ -171,19 +171,20 @@ func (rt *Router) builtinPayloadsAdd(c *gin.Context) {
 
 func (rt *Router) builtinPayloadsGets(c *gin.Context) {
 	typ := ginx.QueryStr(c, "type", "")
-	component := ginx.QueryStr(c, "component", "")
+	ComponentID := ginx.QueryInt64(c, "component_id", 0)
+
 	cate := ginx.QueryStr(c, "cate", "")
 	query := ginx.QueryStr(c, "query", "")
 
-	lst, err := models.BuiltinPayloadGets(rt.Ctx, typ, component, cate, query)
+	lst, err := models.BuiltinPayloadGets(rt.Ctx, uint64(ComponentID), typ, cate, query)
 	ginx.NewRender(c).Data(lst, err)
 }
 
 func (rt *Router) builtinPayloadcatesGet(c *gin.Context) {
 	typ := ginx.QueryStr(c, "type", "")
-	component := ginx.QueryStr(c, "component", "")
+	ComponentID := ginx.QueryInt64(c, "component_id", 0)
 
-	cates, err := models.BuiltinPayloadCates(rt.Ctx, typ, component)
+	cates, err := models.BuiltinPayloadCates(rt.Ctx, typ, uint64(ComponentID))
 	ginx.NewRender(c).Data(cates, err)
 }
 

docker/initsql/a-n9e.sql

@@ -528,6 +528,7 @@ CREATE TABLE `builtin_components` (
 
 CREATE TABLE `builtin_payloads` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '''unique identifier''',
+  `component_id` bigint(20) NOT NULL DEFAULT 0 COMMENT 'component_id',
   `uuid` bigint(20) NOT NULL COMMENT '''uuid of payload''',
   `type` varchar(191) NOT NULL COMMENT '''type of payload''',
   `component` varchar(191) NOT NULL COMMENT '''component of payload''',

docker/migratesql/migrate.sql

@@ -103,4 +103,7 @@ CREATE TABLE notification_record (
 
 
 /* v7.3.0 2024-08-26 */
-ALTER TABLE `target` ADD COLUMN `host_tags` TEXT COMMENT 'global labels set in conf file';
+ALTER TABLE `target` ADD COLUMN `host_tags` TEXT COMMENT 'global labels set in conf file';
+
+/* v7.4.0 2024-08-28 */
+ALTER TABLE `builtin_payloads` ADD COLUMN `component_id` bigint(20) NOT NULL DEFAULT 0 COMMENT 'component_id';

models/alert_cur_event.go

@@ -67,6 +67,7 @@ type AlertCurEvent struct {
 	Claimant           string            `json:"claimant" gorm:"-"`
 	SubRuleId          int64             `json:"sub_rule_id" gorm:"-"`
 	ExtraInfo          []string          `json:"extra_info" gorm:"-"`
+	Target             *Target           `json:"target" gorm:"-"`
 }
 
 func (e *AlertCurEvent) TableName() string {

models/builtin_component.go

@@ -11,7 +11,7 @@ import (
 // BuiltinComponent represents a builtin component along with its metadata.
 type BuiltinComponent struct {
 	ID        uint64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
-	Ident     string `json:"ident" gorm:"type:varchar(191);not null;index:idx_ident,sort:asc;comment:'identifier of component'"`
+	Ident     string `json:"ident" gorm:"type:varchar(191);not null;uniqueIndex:idx_ident,sort:asc;comment:'identifier of component'"`
 	Logo      string `json:"logo" gorm:"type:varchar(191);not null;comment:'logo of component'"`
 	Readme    string `json:"readme" gorm:"type:text;not null;comment:'readme of component'"`
 	CreatedAt int64  `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`

models/builtin_metrics.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"errors"
+	"fmt"
 	"strings"
 	"time"
 
@@ -217,3 +218,10 @@ func BuiltinMetricCollectors(ctx *ctx.Context, lang, typ, query string) ([]strin
 	err := session.Select("distinct(collector)").Pluck("collector", &collectors).Error
 	return collectors, err
 }
+
+func BuiltinMetricBatchUpdateColumn(ctx *ctx.Context, col, old, new, updatedBy string) error {
+	if old == new {
+		return nil
+	}
+	return DB(ctx).Model(&BuiltinMetric{}).Where(fmt.Sprintf("%s = ?", col), old).Updates(map[string]interface{}{col: new, "updated_by": updatedBy}).Error
+}

models/builtin_payload.go

@@ -9,18 +9,19 @@ import (
 )
 
 type BuiltinPayload struct {
-	ID        int64  `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
-	Type      string `json:"type" gorm:"type:varchar(191);not null;index:idx_type,sort:asc;comment:'type of payload'"`                // Alert Dashboard Collet
-	Component string `json:"component" gorm:"type:varchar(191);not null;index:idx_component,sort:asc;comment:'component of payload'"` // Host MySQL Redis
-	Cate      string `json:"cate" gorm:"type:varchar(191);not null;comment:'category of payload'"`                                    // categraf_v1 telegraf_v1
-	Name      string `json:"name" gorm:"type:varchar(191);not null;index:idx_buildinpayload_name,sort:asc;comment:'name of payload'"`                //
-	Tags      string `json:"tags" gorm:"type:varchar(191);not null;default:'';comment:'tags of payload'"`                             // {"host":"
-	Content   string `json:"content" gorm:"type:longtext;not null;comment:'content of payload'"`
-	UUID      int64  `json:"uuid" gorm:"type:bigint;not null;index:idx_uuid;comment:'uuid of payload'"`
-	CreatedAt int64  `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
-	CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
-	UpdatedAt int64  `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
-	UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
+	ID          int64  `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
+	Type        string `json:"type" gorm:"type:varchar(191);not null;index:idx_type,sort:asc;comment:'type of payload'"`                // Alert Dashboard Collet
+	Component   string `json:"component" gorm:"type:varchar(191);not null;index:idx_component,sort:asc;comment:'component of payload'"` //
+	ComponentID uint64 `json:"component_id" gorm:"type:bigint;index:idx_component,sort:asc;comment:'component_id of payload'"`          // ComponentID which the payload belongs to
+	Cate        string `json:"cate" gorm:"type:varchar(191);not null;comment:'category of payload'"`                                    // categraf_v1 telegraf_v1
+	Name        string `json:"name" gorm:"type:varchar(191);not null;index:idx_buildinpayload_name,sort:asc;comment:'name of payload'"` //
+	Tags        string `json:"tags" gorm:"type:varchar(191);not null;default:'';comment:'tags of payload'"`                             // {"host":"
+	Content     string `json:"content" gorm:"type:longtext;not null;comment:'content of payload'"`
+	UUID        int64  `json:"uuid" gorm:"type:bigint;not null;index:idx_uuid;comment:'uuid of payload'"`
+	CreatedAt   int64  `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
+	CreatedBy   string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
+	UpdatedAt   int64  `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
+	UpdatedBy   string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
 }
 
 func (bp *BuiltinPayload) TableName() string {
@@ -33,9 +34,8 @@ func (bp *BuiltinPayload) Verify() error {
 		return errors.New("type is blank")
 	}
 
-	bp.Component = strings.TrimSpace(bp.Component)
-	if bp.Component == "" {
-		return errors.New("component is blank")
+	if bp.ComponentID == 0 {
+		return errors.New("component_id is blank")
 	}
 
 	if bp.Name == "" {
@@ -47,7 +47,7 @@ func (bp *BuiltinPayload) Verify() error {
 
 func BuiltinPayloadExists(ctx *ctx.Context, bp *BuiltinPayload) (bool, error) {
 	var count int64
-	err := DB(ctx).Model(bp).Where("type = ? AND component = ? AND name = ? AND cate = ?", bp.Type, bp.Component, bp.Name, bp.Cate).Count(&count).Error
+	err := DB(ctx).Model(bp).Where("type = ? AND component_id = ? AND name = ? AND cate = ?", bp.Type, bp.ComponentID, bp.Name, bp.Cate).Count(&count).Error
 	if err != nil {
 		return false, err
 	}
@@ -78,7 +78,7 @@ func (bp *BuiltinPayload) Update(ctx *ctx.Context, req BuiltinPayload) error {
 		return err
 	}
 
-	if bp.Type != req.Type || bp.Component != req.Component || bp.Name != req.Name {
+	if bp.Type != req.Type || bp.ComponentID != req.ComponentID || bp.Name != req.Name {
 		exists, err := BuiltinPayloadExists(ctx, &req)
 		if err != nil {
 			return err
@@ -117,13 +117,13 @@ func BuiltinPayloadGet(ctx *ctx.Context, where string, args ...interface{}) (*Bu
 	return &bp, nil
 }
 
-func BuiltinPayloadGets(ctx *ctx.Context, typ, component, cate, query string) ([]*BuiltinPayload, error) {
+func BuiltinPayloadGets(ctx *ctx.Context, componentId uint64, typ, cate, query string) ([]*BuiltinPayload, error) {
 	session := DB(ctx)
 	if typ != "" {
 		session = session.Where("type = ?", typ)
 	}
-	if component != "" {
-		session = session.Where("component = ?", component)
+	if componentId != 0 {
+		session = session.Where("component_id = ?", componentId)
 	}
 
 	if cate != "" {
@@ -144,9 +144,9 @@ func BuiltinPayloadGets(ctx *ctx.Context, typ, component, cate, query string) ([
 }
 
 // get cates of BuiltinPayload by type and component, return []string
-func BuiltinPayloadCates(ctx *ctx.Context, typ, component string) ([]string, error) {
+func BuiltinPayloadCates(ctx *ctx.Context, typ string, componentID uint64) ([]string, error) {
 	var cates []string
-	err := DB(ctx).Model(new(BuiltinPayload)).Where("type = ? and component = ?", typ, component).Distinct("cate").Pluck("cate", &cates).Error
+	err := DB(ctx).Model(new(BuiltinPayload)).Where("type = ? and component_id = ?", typ, componentID).Distinct("cate").Pluck("cate", &cates).Error
 	return cates, err
 }
 
@@ -163,3 +163,33 @@ func BuiltinPayloadComponents(ctx *ctx.Context, typ, cate string) (string, error
 	}
 	return components[0], nil
 }
+
+// InitBuiltinPayloads 兼容新旧 BuiltinPayload 格式
+func InitBuiltinPayloads(ctx *ctx.Context) error {
+	var lst []*BuiltinPayload
+
+	components, err := BuiltinComponentGets(ctx, "")
+	if err != nil {
+		return err
+	}
+
+	identToId := make(map[string]uint64)
+	for _, component := range components {
+		identToId[component.Ident] = component.ID
+	}
+
+	err = DB(ctx).Where("component_id = 0 or component_id is NULL").Find(&lst).Error
+	if err != nil {
+		return err
+	}
+
+	for _, bp := range lst {
+		componentId, ok := identToId[bp.Component]
+		if !ok {
+			continue
+		}
+		bp.ComponentID = componentId
+	}
+
+	return DB(ctx).Save(&lst).Error
+}

models/migrate/migrate.go

@@ -276,5 +276,6 @@ type SsoConfig struct {
 }
 
 type BuiltinPayloads struct {
-	UUID int64 `json:"uuid" gorm:"type:bigint;not null;index:idx_uuid;comment:'uuid of payload'"`
+	UUID        int64 `json:"uuid" gorm:"type:bigint;not null;index:idx_uuid;comment:'uuid of payload'"`
+	ComponentID int64 `json:"component_id" gorm:"type:bigint;index:idx_component,sort:asc;not null;default:0;comment:'component_id of payload'"`
 }