add sql

center/router/router_heartbeat.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io/ioutil"
+	"sort"
 	"strings"
 	"time"
 
@@ -82,56 +83,87 @@ func HandleHeartbeat(c *gin.Context, ctx *ctx.Context, engineName string, metaSe
 		gid := ginx.QueryInt64(c, "gid", 0)
 		hostIp := strings.TrimSpace(req.HostIp)
 
-		field := make(map[string]interface{})
+		newTarget := models.Target{}
+		targetNeedUpdate := false
 		if gid != 0 && gid != target.GroupId {
-			field["group_id"] = gid
+			newTarget.GroupId = gid
+			targetNeedUpdate = true
 		}
 
 		if hostIp != "" && hostIp != target.HostIp {
-			field["host_ip"] = hostIp
+			newTarget.HostIp = hostIp
+			targetNeedUpdate = true
 		}
 
-		tagsMap := target.GetTagsMap()
-		tagNeedUpdate := false
-		for k, v := range req.GlobalLabels {
+		hostTagsMap := target.GetHostTagsMap()
+		hostTagNeedUpdate := false
+		if len(hostTagsMap) != len(req.GlobalLabels) {
+			hostTagNeedUpdate = true
+		} else {
+			for k, v := range req.GlobalLabels {
+				if v == "" {
+					continue
+				}
+
+				if tagv, ok := hostTagsMap[k]; !ok || tagv != v {
+					hostTagNeedUpdate = true
+					break
+				}
+			}
+		}
+
+		if hostTagNeedUpdate {
+			lst := []string{}
+			for k, v := range req.GlobalLabels {
+				lst = append(lst, k+"="+v)
+			}
+			sort.Strings(lst)
+			newTarget.HostTags = lst
+			targetNeedUpdate = true
+		}
+
+		userTagsMap := target.GetTagsMap()
+		userTagNeedUpdate := false
+		userTags := []string{}
+		for k, v := range userTagsMap {
 			if v == "" {
 				continue
 			}
 
-			if tagv, ok := tagsMap[k]; !ok || tagv != v {
-				tagNeedUpdate = true
-				tagsMap[k] = v
+			if _, ok := req.GlobalLabels[k]; !ok {
+				userTags = append(userTags, k+"="+v)
+			} else { // 该key在hostTags中已经存在
+				userTagNeedUpdate = true
 			}
 		}
 
-		if tagNeedUpdate {
-			lst := []string{}
-			for k, v := range tagsMap {
-				lst = append(lst, k+"="+v)
-			}
-			labels := strings.Join(lst, " ") + " "
-			field["tags"] = labels
+		if userTagNeedUpdate {
+			newTarget.Tags = strings.Join(userTags, " ") + " "
+			targetNeedUpdate = true
 		}
 
 		if req.EngineName != "" && req.EngineName != target.EngineName {
-			field["engine_name"] = req.EngineName
+			newTarget.EngineName = req.EngineName
+			targetNeedUpdate = true
 		}
 
 		if req.AgentVersion != "" && req.AgentVersion != target.AgentVersion {
-			field["agent_version"] = req.AgentVersion
+			newTarget.AgentVersion = req.AgentVersion
+			targetNeedUpdate = true
 		}
 
 		if req.OS != "" && req.OS != target.OS {
-			field["os"] = req.OS
+			newTarget.OS = req.OS
+			targetNeedUpdate = true
 		}
 
-		if len(field) > 0 {
-			err := target.UpdateFieldsMap(ctx, field)
+		if targetNeedUpdate {
+			err := models.DB(ctx).Model(&target).Updates(newTarget).Error
 			if err != nil {
 				logger.Errorf("update target fields failed, err: %v", err)
 			}
 		}
-		logger.Debugf("heartbeat field:%+v target: %v", field, *target)
+		logger.Debugf("heartbeat field:%+v target: %v", newTarget, *target)
 	}
 
 	return req, nil

center/router/router_target.go

@@ -260,9 +260,11 @@ func (rt *Router) validateTags(tags []string) error {
 }
 
 func (rt *Router) addTagsToTarget(target *models.Target, tags []string) error {
+	hostTagsMap := target.GetHostTagsMap()
 	for _, tag := range tags {
 		tagKey := strings.Split(tag, "=")[0]
-		if strings.Contains(target.Tags, tagKey+"=") {
+		if _, ok := hostTagsMap[tagKey]; ok ||
+			strings.Contains(target.Tags, tagKey+"=") {
 			return fmt.Errorf("duplicate tagkey(%s)", tagKey)
 		}
 	}

docker/migratesql/migrate.sql

@@ -99,4 +99,8 @@ CREATE TABLE notification_record (
     `details` VARCHAR(2048),
     `created_at` BIGINT NOT NULL,
     INDEX idx_evt (event_id)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+/* v7.3.0 2024-08-26 */
+ALTER TABLE `target` ADD COLUMN `host_tags` TEXT COMMENT 'global labels set in conf file';

models/alert_rule.go

@@ -192,12 +192,14 @@ func GetHostsQuery(queries []HostQuery) []map[string]interface{} {
 				blank := " "
 				for _, tag := range lst {
 					m["tags like ?"+blank] = "%" + tag + "%"
+					m["host_tags like ?"+blank] = "%" + tag + "%"
 					blank += " "
 				}
 			} else {
 				blank := " "
 				for _, tag := range lst {
 					m["tags not like ?"+blank] = "%" + tag + "%"
+					m["host_tags not like ?"+blank] = "%" + tag + "%"
 					blank += " "
 				}
 			}

models/migrate/migrate.go

@@ -227,10 +227,11 @@ type AlertCurEvent struct {
 }
 
 type Target struct {
-	HostIp       string `gorm:"column:host_ip;varchar(15);default:'';comment:IPv4 string;index:idx_host_ip"`
-	AgentVersion string `gorm:"column:agent_version;varchar(255);default:'';comment:agent version;index:idx_agent_version"`
-	EngineName   string `gorm:"column:engine_name;varchar(255);default:'';comment:engine name;index:idx_engine_name"`
-	OS           string `gorm:"column:os;varchar(31);default:'';comment:os type;index:idx_os"`
+	HostIp       string   `gorm:"column:host_ip;type:varchar(15);default:'';comment:IPv4 string;index:idx_host_ip"`
+	AgentVersion string   `gorm:"column:agent_version;type:varchar(255);default:'';comment:agent version;index:idx_agent_version"`
+	EngineName   string   `gorm:"column:engine_name;type:varchar(255);default:'';comment:engine name;index:idx_engine_name"`
+	OS           string   `gorm:"column:os;type:varchar(31);default:'';comment:os type;index:idx_os"`
+	HostTags     []string `gorm:"column:host_tags;type:text;comment:global labels set in conf file;serializer:json"`
 }
 
 type Datasource struct {

models/target.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/toolkits/pkg/container/set"
+
 	"gorm.io/gorm"
 )
 
@@ -19,7 +20,7 @@ type Target struct {
 	GroupObj     *BusiGroup        `json:"group_obj" gorm:"-"`
 	Ident        string            `json:"ident"`
 	Note         string            `json:"note"`
-	Tags         string            `json:"-"`
+	Tags         string            `json:"-"` // user tags
 	TagsJSON     []string          `json:"tags" gorm:"-"`
 	TagsMap      map[string]string `json:"tags_maps" gorm:"-"` // internal use, append tags to series
 	UpdateAt     int64             `json:"update_at"`
@@ -27,6 +28,7 @@ type Target struct {
 	AgentVersion string            `json:"agent_version"`
 	EngineName   string            `json:"engine_name"`
 	OS           string            `json:"os" gorm:"column:os"`
+	HostTags     []string          `json:"host_tags" gorm:"serializer:json"`
 
 	UnixTime   int64   `json:"unixtime" gorm:"-"`
 	Offset     int64   `json:"offset" gorm:"-"`
@@ -335,12 +337,12 @@ func TargetsGetIdentsByIdentsAndHostIps(ctx *ctx.Context, idents, hostIps []stri
 func TargetGetTags(ctx *ctx.Context, idents []string) ([]string, error) {
 	session := DB(ctx).Model(new(Target))
 
-	var arr []string
+	var arr []*Target
 	if len(idents) > 0 {
 		session = session.Where("ident in ?", idents)
 	}
 
-	err := session.Select("distinct(tags) as tags").Pluck("tags", &arr).Error
+	err := session.Select("tags", "host_tags").Find(&arr).Error
 	if err != nil {
 		return nil, err
 	}
@@ -352,10 +354,13 @@ func TargetGetTags(ctx *ctx.Context, idents []string) ([]string, error) {
 
 	set := make(map[string]struct{})
 	for i := 0; i < cnt; i++ {
-		tags := strings.Fields(arr[i])
+		tags := strings.Fields(arr[i].Tags)
 		for j := 0; j < len(tags); j++ {
 			set[tags[j]] = struct{}{}
 		}
+		for _, ht := range arr[i].HostTags {
+			set[ht] = struct{}{}
+		}
 	}
 
 	cnt = len(set)
@@ -398,7 +403,18 @@ func (t *Target) DelTags(ctx *ctx.Context, tags []string) error {
 
 func (t *Target) FillTagsMap() {
 	t.TagsJSON = strings.Fields(t.Tags)
-	t.TagsMap = t.GetTagsMap()
+	t.TagsMap = make(map[string]string)
+	m := make(map[string]string)
+	allTags := append(t.TagsJSON, t.HostTags...)
+	for _, item := range allTags {
+		arr := strings.Split(item, "=")
+		if len(arr) != 2 {
+			continue
+		}
+		m[arr[0]] = arr[1]
+	}
+
+	t.TagsMap = m
 }
 
 func (t *Target) GetTagsMap() map[string]string {
@@ -412,6 +428,18 @@ func (t *Target) GetTagsMap() map[string]string {
 	return m
 }
 
+func (t *Target) GetHostTagsMap() map[string]string {
+	m := make(map[string]string)
+	for _, item := range t.HostTags {
+		arr := strings.Split(item, "=")
+		if len(arr) != 2 {
+			continue
+		}
+		m[arr[0]] = arr[1]
+	}
+	return m
+}
+
 func (t *Target) FillMeta(meta *HostMeta) {
 	t.MemUtil = meta.MemUtil
 	t.CpuUtil = meta.CpuUtil