refactor

alert/naming/heartbeat.go

@@ -79,25 +79,48 @@ func (n *Naming) loopHeartbeat() {
 
 func (n *Naming) heartbeat() error {
 	var datasourceIds []int64
+	var myDatasourceIds []int64
 	var err error
 
-	// 在页面上维护实例和集群的对应关系
-	datasourceIds, err = models.GetDatasourceIdsByEngineName(n.ctx, n.heartbeatConfig.EngineName)
-	if err != nil {
-		return err
+	// 如果是中心机房的 leader 机器，沟通全量的数据源的 hash ring，给智能告警模块使用
+
+	if n.IamLeader() {
+		datasources, err := models.GetDatasources(n.ctx)
+		if err != nil {
+			logger.Warningf("get datasources err:%v", err)
+			return err
+		}
+		for i := 0; i < len(datasources); i++ {
+			datasourceIds = append(datasourceIds, datasources[i].Id)
+		}
+
+		myDatasourceIds, err = models.GetDatasourceIdsByEngineName(n.ctx, n.heartbeatConfig.EngineName)
+		if err != nil {
+			logger.Warningf("get datasource ids by engine name err:%v", err)
+			return err
+		}
+	} else {
+		// 在页面上维护实例和集群的对应关系
+		datasourceIds, err = models.GetDatasourceIdsByEngineName(n.ctx, n.heartbeatConfig.EngineName)
+		if err != nil {
+			return err
+		}
+
+		myDatasourceIds = datasourceIds
 	}
 
-	if len(datasourceIds) == 0 {
+	// 上报心跳
+	if len(myDatasourceIds) == 0 {
 		err := models.AlertingEngineHeartbeatWithCluster(n.ctx, n.heartbeatConfig.Endpoint, n.heartbeatConfig.EngineName, 0)
 		if err != nil {
 			logger.Warningf("heartbeat with cluster %s err:%v", "", err)
 			n.astats.CounterHeartbeatErrorTotal.WithLabelValues().Inc()
 		}
 	} else {
-		for i := 0; i < len(datasourceIds); i++ {
-			err := models.AlertingEngineHeartbeatWithCluster(n.ctx, n.heartbeatConfig.Endpoint, n.heartbeatConfig.EngineName, datasourceIds[i])
+		for i := 0; i < len(myDatasourceIds); i++ {
+			err := models.AlertingEngineHeartbeatWithCluster(n.ctx, n.heartbeatConfig.Endpoint, n.heartbeatConfig.EngineName, myDatasourceIds[i])
 			if err != nil {
-				logger.Warningf("heartbeat with cluster %d err:%v", datasourceIds[i], err)
+				logger.Warningf("heartbeat with cluster %d err:%v", myDatasourceIds[i], err)
 				n.astats.CounterHeartbeatErrorTotal.WithLabelValues().Inc()
 			}
 		}

center/integration/init.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"encoding/json"
 	"path"
+	"strings"
 
 	"github.com/ccfos/nightingale/v6/models"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
@@ -31,145 +32,145 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
 
 		// get logo name
 		// /api/n9e/integrations/icon/AliYun/aliyun.png
-		// files, err := file.FilesUnder(componentDir + "/icon")
-		// if err == nil && len(files) > 0 {
-		// 	component.Logo = "/api/n9e/integrations/icon/" + component.Ident + "/" + files[0]
-		// } else if err != nil {
-		// 	logger.Warningf("read builtin component icon dir fail %s %v", component.Ident, err)
-		// }
+		files, err := file.FilesUnder(componentDir + "/icon")
+		if err == nil && len(files) > 0 {
+			component.Logo = "/api/n9e/integrations/icon/" + component.Ident + "/" + files[0]
+		} else if err != nil {
+			logger.Warningf("read builtin component icon dir fail %s %v", component.Ident, err)
+		}
 
-		// // get description
-		// files, err = file.FilesUnder(componentDir + "/markdown")
-		// if err == nil && len(files) > 0 {
-		// 	var readmeFile string
-		// 	for _, file := range files {
-		// 		if strings.HasSuffix(strings.ToLower(file), "md") {
-		// 			readmeFile = file
-		// 		}
-		// 	}
-		// 	if readmeFile != "" {
-		// 		component.Readme, _ = file.ReadString(readmeFile)
-		// 	}
-		// } else if err != nil {
-		// 	logger.Warningf("read builtin component markdown dir fail %s %v", component.Ident, err)
-		// }
+		// get description
+		files, err = file.FilesUnder(componentDir + "/markdown")
+		if err == nil && len(files) > 0 {
+			var readmeFile string
+			for _, file := range files {
+				if strings.HasSuffix(strings.ToLower(file), "md") {
+					readmeFile = file
+				}
+			}
+			if readmeFile != "" {
+				component.Readme, _ = file.ReadString(readmeFile)
+			}
+		} else if err != nil {
+			logger.Warningf("read builtin component markdown dir fail %s %v", component.Ident, err)
+		}
 
-		// exists, _ := models.BuiltinComponentExists(ctx, &component)
-		// if !exists {
-		// 	err = component.Add(ctx, "system")
-		// 	if err != nil {
-		// 		logger.Warning("add builtin component fail ", component, err)
-		// 		continue
-		// 	}
-		// }
+		exists, _ := models.BuiltinComponentExists(ctx, &component)
+		if !exists {
+			err = component.Add(ctx, "system")
+			if err != nil {
+				logger.Warning("add builtin component fail ", component, err)
+				continue
+			}
+		}
 
-		// // alerts
-		// files, err = file.FilesUnder(componentDir + "/alerts")
-		// if err == nil && len(files) > 0 {
-		// 	for _, f := range files {
-		// 		fp := componentDir + "/alerts/" + f
-		// 		bs, err := file.ReadBytes(fp)
-		// 		if err != nil {
-		// 			logger.Warning("read builtin component alerts file fail ", f, err)
-		// 			continue
-		// 		}
+		// alerts
+		files, err = file.FilesUnder(componentDir + "/alerts")
+		if err == nil && len(files) > 0 {
+			for _, f := range files {
+				fp := componentDir + "/alerts/" + f
+				bs, err := file.ReadBytes(fp)
+				if err != nil {
+					logger.Warning("read builtin component alerts file fail ", f, err)
+					continue
+				}
 
-		// 		alerts := []models.AlertRule{}
-		// 		err = json.Unmarshal(bs, &alerts)
-		// 		if err != nil {
-		// 			logger.Warning("parse builtin component alerts file fail ", f, err)
-		// 			continue
-		// 		}
+				alerts := []models.AlertRule{}
+				err = json.Unmarshal(bs, &alerts)
+				if err != nil {
+					logger.Warning("parse builtin component alerts file fail ", f, err)
+					continue
+				}
 
-		// 		for _, alert := range alerts {
-		// 			content, err := json.Marshal(alert)
-		// 			if err != nil {
-		// 				logger.Warning("marshal builtin alert fail ", alert, err)
-		// 				continue
-		// 			}
+				for _, alert := range alerts {
+					content, err := json.Marshal(alert)
+					if err != nil {
+						logger.Warning("marshal builtin alert fail ", alert, err)
+						continue
+					}
 
-		// 			cate := strings.Replace(f, ".json", "", -1)
-		// 			builtinAlert := models.BuiltinPayload{
-		// 				Component: component.Ident,
-		// 				Type:      "alert",
-		// 				Cate:      cate,
-		// 				Name:      alert.Name,
-		// 				Content:   string(content),
-		// 			}
+					cate := strings.Replace(f, ".json", "", -1)
+					builtinAlert := models.BuiltinPayload{
+						Component: component.Ident,
+						Type:      "alert",
+						Cate:      cate,
+						Name:      alert.Name,
+						Content:   string(content),
+					}
 
-		// 			exists, err := models.BuiltinPayloadExists(ctx, &builtinAlert)
-		// 			if err != nil {
-		// 				logger.Warning("check builtin alert exists fail ", builtinAlert, err)
-		// 				continue
-		// 			}
+					exists, err := models.BuiltinPayloadExists(ctx, &builtinAlert)
+					if err != nil {
+						logger.Warning("check builtin alert exists fail ", builtinAlert, err)
+						continue
+					}
 
-		// 			if exists {
-		// 				continue
-		// 			}
+					if exists {
+						continue
+					}
 
-		// 			err = builtinAlert.Add(ctx, "system")
-		// 			if err != nil {
-		// 				logger.Warningf("add builtin alert:%+v fail %v", builtinAlert, err)
-		// 				continue
-		// 			}
-		// 		}
-		// 	}
-		// }
+					err = builtinAlert.Add(ctx, "system")
+					if err != nil {
+						logger.Warningf("add builtin alert:%+v fail %v", builtinAlert, err)
+						continue
+					}
+				}
+			}
+		}
 
-		// // dashboards
-		// files, err = file.FilesUnder(componentDir + "/dashboards")
-		// if err == nil && len(files) > 0 {
-		// 	for _, f := range files {
-		// 		fp := componentDir + "/dashboards/" + f
-		// 		bs, err := file.ReadBytes(fp)
-		// 		if err != nil {
-		// 			logger.Warning("read builtin component dashboards file fail ", f, err)
-		// 			continue
-		// 		}
+		// dashboards
+		files, err = file.FilesUnder(componentDir + "/dashboards")
+		if err == nil && len(files) > 0 {
+			for _, f := range files {
+				fp := componentDir + "/dashboards/" + f
+				bs, err := file.ReadBytes(fp)
+				if err != nil {
+					logger.Warning("read builtin component dashboards file fail ", f, err)
+					continue
+				}
 
-		// 		dashboard := BuiltinBoard{}
-		// 		err = json.Unmarshal(bs, &dashboard)
-		// 		if err != nil {
-		// 			logger.Warning("parse builtin component dashboards file fail ", f, err)
-		// 			continue
-		// 		}
+				dashboard := BuiltinBoard{}
+				err = json.Unmarshal(bs, &dashboard)
+				if err != nil {
+					logger.Warning("parse builtin component dashboards file fail ", f, err)
+					continue
+				}
 
-		// 		content, err := json.Marshal(dashboard)
-		// 		if err != nil {
-		// 			logger.Warning("marshal builtin dashboard fail ", dashboard, err)
-		// 			continue
-		// 		}
+				content, err := json.Marshal(dashboard)
+				if err != nil {
+					logger.Warning("marshal builtin dashboard fail ", dashboard, err)
+					continue
+				}
 
-		// 		builtinDashboard := models.BuiltinPayload{
-		// 			Component: component.Ident,
-		// 			Type:      "dashboard",
-		// 			Cate:      "",
-		// 			Name:      dashboard.Name,
-		// 			Content:   string(content),
-		// 		}
+				builtinDashboard := models.BuiltinPayload{
+					Component: component.Ident,
+					Type:      "dashboard",
+					Cate:      "",
+					Name:      dashboard.Name,
+					Content:   string(content),
+				}
 
-		// 		exists, err := models.BuiltinPayloadExists(ctx, &builtinDashboard)
-		// 		if err != nil {
-		// 			logger.Warning("check builtin dashboard exists fail ", builtinDashboard, err)
-		// 			continue
-		// 		}
+				exists, err := models.BuiltinPayloadExists(ctx, &builtinDashboard)
+				if err != nil {
+					logger.Warning("check builtin dashboard exists fail ", builtinDashboard, err)
+					continue
+				}
 
-		// 		if exists {
-		// 			continue
-		// 		}
+				if exists {
+					continue
+				}
 
-		// 		err = builtinDashboard.Add(ctx, "system")
-		// 		if err != nil {
-		// 			logger.Warning("add builtin dashboard fail ", builtinDashboard, err)
-		// 			continue
-		// 		}
-		// 	}
-		// } else if err != nil {
-		// 	logger.Warningf("read builtin component dash dir fail %s %v", component.Ident, err)
-		// }
+				err = builtinDashboard.Add(ctx, "system")
+				if err != nil {
+					logger.Warning("add builtin dashboard fail ", builtinDashboard, err)
+					continue
+				}
+			}
+		} else if err != nil {
+			logger.Warningf("read builtin component dash dir fail %s %v", component.Ident, err)
+		}
 
 		// metrics
-		files, err := file.FilesUnder(componentDir + "/metrics")
+		files, err = file.FilesUnder(componentDir + "/metrics")
 		if err == nil && len(files) > 0 {
 			for _, f := range files {
 				fp := componentDir + "/metrics/" + f

center/router/router_builtin_componet.go

@@ -31,18 +31,11 @@ func (rt *Router) builtinComponentsAdd(c *gin.Context) {
 
 func (rt *Router) builtinComponentsGets(c *gin.Context) {
 	query := ginx.QueryStr(c, "query", "")
-	limit := ginx.QueryInt(c, "limit", 20)
 
-	bc, err := models.BuiltinComponentGets(rt.Ctx, query, limit, ginx.Offset(c, limit))
+	bc, err := models.BuiltinComponentGets(rt.Ctx, query)
 	ginx.Dangerous(err)
 
-	total, err := models.BuiltinComponentCount(rt.Ctx, query)
-	ginx.Dangerous(err)
-
-	ginx.NewRender(c).Data(gin.H{
-		"list":  bc,
-		"total": total,
-	}, nil)
+	ginx.NewRender(c).Data(bc, nil)
 }
 
 func (rt *Router) builtinComponentsPut(c *gin.Context) {

models/builtin_component.go

@@ -86,7 +86,7 @@ func BuiltinComponentDels(ctx *ctx.Context, ids []int64) error {
 	return DB(ctx).Where("id in ?", ids).Delete(new(BuiltinComponent)).Error
 }
 
-func BuiltinComponentGets(ctx *ctx.Context, query string, limit, offset int) ([]*BuiltinComponent, error) {
+func BuiltinComponentGets(ctx *ctx.Context, query string) ([]*BuiltinComponent, error) {
 	session := DB(ctx)
 	if query != "" {
 		queryPattern := "%" + query + "%"
@@ -95,24 +95,11 @@ func BuiltinComponentGets(ctx *ctx.Context, query string, limit, offset int) ([]
 
 	var lst []*BuiltinComponent
 
-	err := session.Limit(limit).Offset(offset).Find(&lst).Error
+	err := session.Order("ident DESC").Find(&lst).Error
 
 	return lst, err
 }
 
-func BuiltinComponentCount(ctx *ctx.Context, query string) (int64, error) {
-	session := DB(ctx).Model(&BuiltinComponent{})
-	if query != "" {
-		queryPattern := "%" + query + "%"
-		session = session.Where("ident LIKE ?", queryPattern)
-	}
-
-	var cnt int64
-	err := session.Count(&cnt).Error
-
-	return cnt, err
-}
-
 func BuiltinComponentGet(ctx *ctx.Context, where string, args ...interface{}) (*BuiltinComponent, error) {
 	var lst []*BuiltinComponent
 	err := DB(ctx).Where(where, args...).Find(&lst).Error