optimize host query (#2385)

Co-authored-by: Xu Bin <140785332+Reditiny@users.noreply.github.com>

.gitignore

@@ -9,6 +9,7 @@
 *.o
 *.a
 *.so
+*.db
 *.sw[po]
 *.tar.gz
 *.[568vq]

alert/alert.go

@@ -96,7 +96,7 @@ func Start(alertc aconf.Alert, pushgwc pconf.Pushgw, syncStats *memsto.Stats, al
 	promClients *prom.PromClientMap, tdendgineClients *tdengine.TdengineClientMap, userCache *memsto.UserCacheType, userGroupCache *memsto.UserGroupCacheType) {
 	alertSubscribeCache := memsto.NewAlertSubscribeCache(ctx, syncStats)
 	recordingRuleCache := memsto.NewRecordingRuleCache(ctx, syncStats)
-	targetsOfAlertRulesCache := memsto.NewTargetOfAlertRuleCache(ctx, alertc.Heartbeat.EngineName, syncStats)
+	targetsOfAlertRulesCache := memsto.NewTargetOfAlertRuleCache(ctx, alertc.Heartbeat.EngineName, syncStats, targetCache)
 
 	go models.InitNotifyConfig(ctx, alertc.Alerting.TemplatesDir)
 

alert/dispatch/dispatch.go

@@ -293,9 +293,9 @@ func (e *Dispatch) Send(rule *models.AlertRule, event *models.AlertCurEvent, not
 	// handle global webhooks
 	if !event.OverrideGlobalWebhook() {
 		if e.alerting.WebhookBatchSend {
-			sender.BatchSendWebhooks(e.ctx, notifyTarget.ToWebhookList(), event, e.Astats)
+			sender.BatchSendWebhooks(e.ctx, notifyTarget.ToWebhookMap(), event, e.Astats)
 		} else {
-			sender.SingleSendWebhooks(e.ctx, notifyTarget.ToWebhookList(), event, e.Astats)
+			sender.SingleSendWebhooks(e.ctx, notifyTarget.ToWebhookMap(), event, e.Astats)
 		}
 	}
 

alert/dispatch/notify_target.go

@@ -76,52 +76,8 @@ func (s *NotifyTarget) ToCallbackList() []string {
 	return callbacks
 }
 
-func (s *NotifyTarget) ToWebhookList() []*models.Webhook {
-	webhooks := make([]*models.Webhook, 0, len(s.webhooks))
-	for _, wh := range s.webhooks {
-		if wh.Batch == 0 {
-			wh.Batch = 1000
-		}
-
-		if wh.Timeout == 0 {
-			wh.Timeout = 10
-		}
-
-		if wh.RetryCount == 0 {
-			wh.RetryCount = 10
-		}
-
-		if wh.RetryInterval == 0 {
-			wh.RetryInterval = 10
-		}
-
-		webhooks = append(webhooks, wh)
-	}
-	return webhooks
-}
-
 func (s *NotifyTarget) ToWebhookMap() map[string]*models.Webhook {
-	webhookMap := make(map[string]*models.Webhook, len(s.webhooks))
-	for _, wh := range s.webhooks {
-		if wh.Batch == 0 {
-			wh.Batch = 1000
-		}
-
-		if wh.Timeout == 0 {
-			wh.Timeout = 10
-		}
-
-		if wh.RetryCount == 0 {
-			wh.RetryCount = 10
-		}
-
-		if wh.RetryInterval == 0 {
-			wh.RetryInterval = 10
-		}
-
-		webhookMap[wh.Url] = wh
-	}
-	return webhookMap
+	return s.webhooks
 }
 
 func (s *NotifyTarget) ToUidList() []int64 {

alert/eval/alert_rule.go

@@ -118,7 +118,7 @@ func (s *Scheduler) syncAlertRules() {
 				}
 				processor := process.NewProcessor(s.aconf.Heartbeat.EngineName, rule, dsId, s.alertRuleCache, s.targetCache, s.targetsOfAlertRuleCache, s.busiGroupCache, s.alertMuteCache, s.datasourceCache, s.ctx, s.stats)
 
-				alertRule := NewAlertRuleWorker(rule, dsId, processor, s.promClients, s.tdengineClients, s.ctx)
+				alertRule := NewAlertRuleWorker(s.ctx, rule, dsId, processor, s.promClients, s.tdengineClients, s.targetCache)
 				alertRuleWorkers[alertRule.Hash()] = alertRule
 			}
 		} else if rule.IsHostRule() {
@@ -127,7 +127,7 @@ func (s *Scheduler) syncAlertRules() {
 				continue
 			}
 			processor := process.NewProcessor(s.aconf.Heartbeat.EngineName, rule, 0, s.alertRuleCache, s.targetCache, s.targetsOfAlertRuleCache, s.busiGroupCache, s.alertMuteCache, s.datasourceCache, s.ctx, s.stats)
-			alertRule := NewAlertRuleWorker(rule, 0, processor, s.promClients, s.tdengineClients, s.ctx)
+			alertRule := NewAlertRuleWorker(s.ctx, rule, 0, processor, s.promClients, s.tdengineClients, s.targetCache)
 			alertRuleWorkers[alertRule.Hash()] = alertRule
 		} else {
 			// 如果 rule 不是通过 prometheus engine 来告警的，则创建为 externalRule

alert/eval/eval.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/ccfos/nightingale/v6/alert/common"
 	"github.com/ccfos/nightingale/v6/alert/process"
+	"github.com/ccfos/nightingale/v6/memsto"
 	"github.com/ccfos/nightingale/v6/models"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/hash"
@@ -46,9 +47,9 @@ type AlertRuleWorker struct {
 
 	Scheduler *cron.Cron
 
-	HostAndDeviceIdentCache sync.Map
+	DeviceIdentHook func(paramQuery models.ParamQuery) ([]string, error)
 
-	DeviceIdentHook func(arw *AlertRuleWorker, paramQuery models.ParamQuery) ([]string, error)
+	TargetCache *memsto.TargetCacheType
 }
 
 const (
@@ -67,20 +68,20 @@ const (
 	Inner JoinType = "inner"
 )
 
-func NewAlertRuleWorker(rule *models.AlertRule, datasourceId int64, Processor *process.Processor, promClients *prom.PromClientMap, tdengineClients *tdengine.TdengineClientMap, ctx *ctx.Context) *AlertRuleWorker {
+func NewAlertRuleWorker(ctx *ctx.Context, rule *models.AlertRule, datasourceId int64, Processor *process.Processor, promClients *prom.PromClientMap, tdengineClients *tdengine.TdengineClientMap, targetCache *memsto.TargetCacheType) *AlertRuleWorker {
 	arw := &AlertRuleWorker{
 		DatasourceId: datasourceId,
 		Quit:         make(chan struct{}),
 		Rule:         rule,
 		Processor:    Processor,
 
-		PromClients:             promClients,
-		TdengineClients:         tdengineClients,
-		Ctx:                     ctx,
-		HostAndDeviceIdentCache: sync.Map{},
-		DeviceIdentHook: func(arw *AlertRuleWorker, paramQuery models.ParamQuery) ([]string, error) {
+		PromClients:     promClients,
+		TdengineClients: tdengineClients,
+		Ctx:             ctx,
+		DeviceIdentHook: func(paramQuery models.ParamQuery) ([]string, error) {
 			return nil, nil
 		},
+		TargetCache: targetCache,
 	}
 
 	interval := rule.PromEvalInterval
@@ -104,9 +105,17 @@ func NewAlertRuleWorker(rule *models.AlertRule, datasourceId int64, Processor *p
 
 	Processor.ScheduleEntry = arw.Scheduler.Entry(entryID)
 
+	Processor.PromEvalInterval = getPromEvalInterval(Processor.ScheduleEntry.Schedule)
 	return arw
 }
 
+func getPromEvalInterval(schedule cron.Schedule) int {
+	now := time.Now()
+	next1 := schedule.Next(now)
+	next2 := schedule.Next(next1)
+	return int(next2.Sub(next1).Seconds())
+}
+
 func (arw *AlertRuleWorker) Key() string {
 	return common.RuleKey(arw.DatasourceId, arw.Rule.Id)
 }
@@ -130,14 +139,16 @@ func (arw *AlertRuleWorker) Start() {
 }
 
 func (arw *AlertRuleWorker) Eval() {
-	arw.Processor.EvalStart = time.Now().Unix()
+	if arw.Processor.PromEvalInterval == 0 {
+		arw.Processor.PromEvalInterval = getPromEvalInterval(arw.Processor.ScheduleEntry.Schedule)
+	}
+
 	cachedRule := arw.Rule
 	if cachedRule == nil {
 		// logger.Errorf("rule_eval:%s Rule not found", arw.Key())
 		return
 	}
 	arw.Processor.Stats.CounterRuleEval.WithLabelValues().Inc()
-	arw.HostAndDeviceIdentCache = sync.Map{}
 
 	typ := cachedRule.GetRuleType()
 	var (
@@ -240,10 +251,15 @@ func (arw *AlertRuleWorker) GetPromAnomalyPoint(ruleConfig string) ([]models.Ano
 		readerClient := arw.PromClients.GetCli(arw.DatasourceId)
 
 		if query.VarEnabled {
-			anomalyPoints := arw.VarFilling(query, readerClient)
-			for _, v := range anomalyPoints {
-				lst = append(lst, v)
+			var anomalyPoints []models.AnomalyPoint
+			if hasLabelLossAggregator(query) || notExactMatch(query) {
+				// 若有聚合函数或非精确匹配则需要先填充变量然后查询，这个方式效率较低
+				anomalyPoints = arw.VarFillingBeforeQuery(query, readerClient)
+			} else {
+				// 先查询再过滤变量，效率较高，但无法处理有聚合函数的情况
+				anomalyPoints = arw.VarFillingAfterQuery(query, readerClient)
 			}
+			lst = append(lst, anomalyPoints...)
 		} else {
 			// 无变量
 			promql := strings.TrimSpace(query.PromQl)
@@ -297,17 +313,18 @@ type sample struct {
 	Timestamp model.Time
 }
 
-// VarFilling 填充变量
+// VarFillingAfterQuery 填充变量，先查询再填充变量
 // 公式: mem_used_percent{host="$host"} > $val 其中 $host 为参数变量，$val 为值变量
 // 实现步骤:
-// 广度优先遍历，保证同一参数变量的子筛选可以覆盖上一层筛选
-// 每个节点先查询无参数的 query, 即 mem_used_percent > curVal, 得到满足值变量的所有结果
+// 依次遍历参数配置节点，保证同一参数变量的子筛选可以覆盖上一层筛选
+// 每个节点先查询无参数的 query, 即 mem_used_percent{} > curVal, 得到满足值变量的所有结果
 // 结果中有满足本节点参数变量的值，加入异常点列表
 // 参数变量的值不满足的组合，需要覆盖上层筛选中产生的异常点
-func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient promsdk.API) map[string]models.AnomalyPoint {
+func (arw *AlertRuleWorker) VarFillingAfterQuery(query models.PromQuery, readerClient promsdk.API) []models.AnomalyPoint {
+	varToLabel := ExtractVarMapping(query.PromQl)
 	fullQuery := removeVal(query.PromQl)
 	// 存储所有的异常点，key 为参数变量的组合，可以实现子筛选对上一层筛选的覆盖
-	anomalyPoints := make(map[string]models.AnomalyPoint)
+	anomalyPointsMap := make(map[string]models.AnomalyPoint)
 	// 统一变量配置格式
 	VarConfigForCalc := &models.ChildVarConfig{
 		ParamVal:        make([]map[string]models.ParamQuery, 1),
@@ -369,12 +386,13 @@ func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient prom
 				curRealQuery := realQuery
 				var cur []string
 				for _, paramKey := range ParamKeys {
-					val := string(seqVals[i].Metric[model.LabelName(paramKey)])
+					val := string(seqVals[i].Metric[model.LabelName(varToLabel[paramKey])])
 					cur = append(cur, val)
-					curRealQuery = strings.Replace(curRealQuery, fmt.Sprintf("$%s", paramKey), val, -1)
+					curRealQuery = fillVar(curRealQuery, paramKey, val)
 				}
+
 				if _, ok := paramPermutation[strings.Join(cur, "-")]; ok {
-					anomalyPoints[strings.Join(cur, "-")] = models.AnomalyPoint{
+					anomalyPointsMap[strings.Join(cur, "-")] = models.AnomalyPoint{
 						Key:       seqVals[i].Metric.String(),
 						Timestamp: seqVals[i].Timestamp.Unix(),
 						Value:     float64(seqVals[i].Value),
@@ -389,12 +407,16 @@ func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient prom
 
 			// 剩余的参数组合为本层筛选不产生异常点的组合，需要覆盖上层筛选中产生的异常点
 			for k, _ := range paramPermutation {
-				delete(anomalyPoints, k)
+				delete(anomalyPointsMap, k)
 			}
 		}
 		curNode = curNode.ChildVarConfigs
 	}
 
+	anomalyPoints := make([]models.AnomalyPoint, 0)
+	for _, point := range anomalyPointsMap {
+		anomalyPoints = append(anomalyPoints, point)
+	}
 	return anomalyPoints
 }
 
@@ -516,6 +538,7 @@ func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.Param
 			return nil, fmt.Errorf("param key: %s, params is empty", paramKey)
 		}
 
+		logger.Infof("rule_eval:%s paramKey: %s, params: %v", arw.Key(), paramKey, params)
 		paramMap[paramKey] = params
 	}
 
@@ -524,7 +547,7 @@ func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.Param
 
 	res := make(map[string]struct{})
 	for i := range permutation {
-		res[strings.Join(permutation[i], "-")] = struct{}{}
+		res[strings.Join(permutation[i], "@@")] = struct{}{}
 	}
 
 	return res, nil
@@ -534,35 +557,21 @@ func (arw *AlertRuleWorker) getHostIdents(paramQuery models.ParamQuery) ([]strin
 	var params []string
 	q, _ := json.Marshal(paramQuery.Query)
 
-	cacheKey := "Host_" + string(q)
-	value, hit := arw.HostAndDeviceIdentCache.Load(cacheKey)
-	if idents, ok := value.([]string); hit && ok {
-		params = idents
-		return params, nil
-	}
-
 	var queries []models.HostQuery
 	err := json.Unmarshal(q, &queries)
 	if err != nil {
 		return nil, err
 	}
 
-	hostsQuery := models.GetHostsQuery(queries)
-	session := models.TargetFilterQueryBuild(arw.Ctx, hostsQuery, 0, 0)
-	var lst []*models.Target
-	err = session.Find(&lst).Error
-	if err != nil {
-		return nil, err
+	hosts := arw.TargetCache.GetHostIdentsQuery(queries)
+	for i := range hosts {
+		params = append(params, hosts[i].Ident)
 	}
-	for i := range lst {
-		params = append(params, lst[i].Ident)
-	}
-	arw.HostAndDeviceIdentCache.Store(cacheKey, params)
 	return params, nil
 }
 
 func (arw *AlertRuleWorker) getDeviceIdents(paramQuery models.ParamQuery) ([]string, error) {
-	return arw.DeviceIdentHook(arw, paramQuery)
+	return arw.DeviceIdentHook(paramQuery)
 }
 
 // 生成所有排列组合
@@ -1198,3 +1207,198 @@ func GetQueryRefAndUnit(query interface{}) (string, string, error) {
 	json.Unmarshal(queryBytes, &queryMap)
 	return queryMap.Ref, queryMap.Unit, nil
 }
+
+// VarFillingBeforeQuery 填充变量，先填充变量再查询，针对有聚合函数的情况
+// 公式: avg(mem_used_percent{host="$host"}) > $val 其中 $host 为参数变量，$val 为值变量
+// 实现步骤:
+// 依次遍历参数配置节点，保证同一参数变量的子筛选可以覆盖上一层筛选
+// 每个节点先填充参数再进行查询, 即先得到完整的 promql avg(mem_used_percent{host="127.0.0.1"}) > 5
+// 再查询得到满足值变量的所有结果加入异常点列表
+// 参数变量的值不满足的组合，需要覆盖上层筛选中产生的异常点
+func (arw *AlertRuleWorker) VarFillingBeforeQuery(query models.PromQuery, readerClient promsdk.API) []models.AnomalyPoint {
+	// 存储异常点的 map，key 为参数变量的组合，可以实现子筛选对上一层筛选的覆盖
+	anomalyPointsMap := sync.Map{}
+	// 统一变量配置格式
+	VarConfigForCalc := &models.ChildVarConfig{
+		ParamVal:        make([]map[string]models.ParamQuery, 1),
+		ChildVarConfigs: query.VarConfig.ChildVarConfigs,
+	}
+	VarConfigForCalc.ParamVal[0] = make(map[string]models.ParamQuery)
+	for _, p := range query.VarConfig.ParamVal {
+		VarConfigForCalc.ParamVal[0][p.Name] = models.ParamQuery{
+			ParamType: p.ParamType,
+			Query:     p.Query,
+		}
+	}
+	// 使用一个统一的参数变量顺序
+	var ParamKeys []string
+	for val, valQuery := range VarConfigForCalc.ParamVal[0] {
+		if valQuery.ParamType == "threshold" {
+			continue
+		}
+		ParamKeys = append(ParamKeys, val)
+	}
+	sort.Slice(ParamKeys, func(i, j int) bool {
+		return ParamKeys[i] < ParamKeys[j]
+	})
+	// 遍历变量配置链表
+	curNode := VarConfigForCalc
+	for curNode != nil {
+		for _, param := range curNode.ParamVal {
+			curPromql := query.PromQl
+			// 取出阈值变量
+			valMap := make(map[string]string)
+			for val, valQuery := range param {
+				if valQuery.ParamType == "threshold" {
+					valMap[val] = getString(valQuery.Query)
+				}
+			}
+			// 替换阈值变量
+			for key, val := range valMap {
+				curPromql = strings.Replace(curPromql, fmt.Sprintf("$%s", key), val, -1)
+			}
+			// 得到参数变量的所有组合
+			paramPermutation, err := arw.getParamPermutation(param, ParamKeys)
+			if err != nil {
+				logger.Errorf("rule_eval:%s, paramPermutation error:%v", arw.Key(), err)
+				continue
+			}
+
+			keyToPromql := make(map[string]string)
+			for paramPermutationKeys, _ := range paramPermutation {
+				realPromql := curPromql
+				split := strings.Split(paramPermutationKeys, "@@")
+				for j := range ParamKeys {
+					realPromql = fillVar(realPromql, ParamKeys[j], split[j])
+				}
+				keyToPromql[paramPermutationKeys] = realPromql
+			}
+
+			// 并发查询
+			wg := sync.WaitGroup{}
+			semaphore := make(chan struct{}, 200)
+			for key, promql := range keyToPromql {
+				wg.Add(1)
+				semaphore <- struct{}{}
+				go func(key, promql string) {
+					defer func() {
+						<-semaphore
+						wg.Done()
+					}()
+					value, _, err := readerClient.Query(context.Background(), promql, time.Now())
+					if err != nil {
+						logger.Errorf("rule_eval:%s, promql:%s, error:%v", arw.Key(), promql, err)
+						return
+					}
+					logger.Infof("rule_eval:%s, promql:%s, value:%+v", arw.Key(), promql, value)
+
+					points := models.ConvertAnomalyPoints(value)
+					if len(points) == 0 {
+						anomalyPointsMap.Delete(key)
+						return
+					}
+					for i := 0; i < len(points); i++ {
+						points[i].Severity = query.Severity
+						points[i].Query = promql
+						points[i].ValuesUnit = map[string]unit.FormattedValue{
+							"v": unit.ValueFormatter(query.Unit, 2, points[i].Value),
+						}
+					}
+					anomalyPointsMap.Store(key, points)
+				}(key, promql)
+			}
+			wg.Wait()
+		}
+		curNode = curNode.ChildVarConfigs
+	}
+	anomalyPoints := make([]models.AnomalyPoint, 0)
+	anomalyPointsMap.Range(func(key, value any) bool {
+		if points, ok := value.([]models.AnomalyPoint); ok {
+			anomalyPoints = append(anomalyPoints, points...)
+		}
+		return true
+	})
+	return anomalyPoints
+}
+
+// 判断 query 中是否有会导致标签丢失的聚合函数
+func hasLabelLossAggregator(query models.PromQuery) bool {
+	noLabelAggregators := []string{
+		"sum", "min", "max", "avg",
+		"stddev", "stdvar",
+		"count", "quantile",
+		"group",
+	}
+	promql := strings.ToLower(query.PromQl)
+
+	for _, fn := range noLabelAggregators {
+		// 检查是否包含这些聚合函数，需要确保函数名后面跟着左括号
+		if strings.Contains(promql, fn+"(") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// 判断 query 中是否有 != =~ !~
+func notExactMatch(query models.PromQuery) bool {
+	promql := strings.ToLower(query.PromQl)
+	if strings.Contains(promql, "!=") || strings.Contains(promql, "=~") || strings.Contains(promql, "!~") {
+		return true
+	}
+	return false
+}
+
+// ExtractVarMapping 从 promql 中提取变量映射关系，为了在 query 之后可以将标签正确的放回 promql
+// 输入: sum(rate(mem_used_percent{host="$my_host"})) by (instance) + avg(node_load1{region="$region"}) > $val
+// 输出: map[string]string{"my_host":"host", "region":"region"}
+func ExtractVarMapping(promql string) map[string]string {
+	varMapping := make(map[string]string)
+
+	// 遍历所有花括号对
+	for {
+		start := strings.Index(promql, "{")
+		if start == -1 {
+			break
+		}
+
+		end := strings.Index(promql, "}")
+		if end == -1 {
+			break
+		}
+
+		// 提取标签键值对
+		labels := promql[start+1 : end]
+		pairs := strings.Split(labels, ",")
+
+		for _, pair := range pairs {
+			// 分割键值对
+			kv := strings.Split(pair, "=")
+			if len(kv) != 2 {
+				continue
+			}
+
+			key := strings.TrimSpace(kv[0])
+			value := strings.Trim(strings.TrimSpace(kv[1]), "\"")
+			value = strings.Trim(value, "'")
+
+			// 检查值是否为变量(以$开头)
+			if strings.HasPrefix(value, "$") {
+				varName := value[1:] // 去掉$前缀
+				varMapping[varName] = key
+			}
+		}
+
+		// 继续处理剩余部分
+		promql = promql[end+1:]
+	}
+
+	return varMapping
+}
+
+func fillVar(curRealQuery string, paramKey string, val string) string {
+	curRealQuery = strings.Replace(curRealQuery, fmt.Sprintf("'$%s'", paramKey), fmt.Sprintf("'%s'", val), -1)
+	curRealQuery = strings.Replace(curRealQuery, fmt.Sprintf("\"$%s\"", paramKey), fmt.Sprintf("\"%s\"", val), -1)
+	return curRealQuery
+}

alert/eval/eval_test.go

@@ -340,7 +340,7 @@ func Test_removeVal(t *testing.T) {
 		{
 			name: "removeVal7",
 			args: args{
-				promql: "mem{test1=\"test1\",test2=\"test2\",test3=\"$test3\"} > $val",
+				promql: "mem{test1=\"test1\",test2=\"test2\",test3='$test3'} > $val",
 			},
 			want: "mem{test1=\"test1\",test2=\"test2\"} > $val",
 		},
@@ -361,16 +361,16 @@ func Test_removeVal(t *testing.T) {
 		{
 			name: "removeVal10",
 			args: args{
-				promql: "mem{test1=\"test1\",test2=\"$test2\"} > $val1 and mem{test3=\"test3\",test4=\"test4\"} > $val2",
+				promql: "mem{test1=\"test1\",test2='$test2'} > $val1 and mem{test3=\"test3\",test4=\"test4\"} > $val2",
 			},
 			want: "mem{test1=\"test1\"} > $val1 and mem{test3=\"test3\",test4=\"test4\"} > $val2",
 		},
 		{
 			name: "removeVal11",
 			args: args{
-				promql: "mem{test1=\"test1\",test2=\"test2\"} > $val1 and mem{test3=\"$test3\",test4=\"test4\"} > $val2",
+				promql: "mem{test1='test1',test2=\"test2\"} > $val1 and mem{test3=\"$test3\",test4=\"test4\"} > $val2",
 			},
-			want: "mem{test1=\"test1\",test2=\"test2\"} > $val1 and mem{test4=\"test4\"} > $val2",
+			want: "mem{test1='test1',test2=\"test2\"} > $val1 and mem{test4=\"test4\"} > $val2",
 		},
 		{
 			name: "removeVal12",
@@ -388,3 +388,71 @@ func Test_removeVal(t *testing.T) {
 		})
 	}
 }
+
+func TestExtractVarMapping(t *testing.T) {
+	tests := []struct {
+		name   string
+		promql string
+		want   map[string]string
+	}{
+		{
+			name:   "单个花括号单个变量",
+			promql: `mem_used_percent{host="$my_host"} > $val`,
+			want:   map[string]string{"my_host": "host"},
+		},
+		{
+			name:   "单个花括号多个变量",
+			promql: `mem_used_percent{host="$my_host",region="$region",env="prod"} > $val`,
+			want:   map[string]string{"my_host": "host", "region": "region"},
+		},
+		{
+			name:   "多个花括号多个变量",
+			promql: `sum(rate(mem_used_percent{host="$my_host"})) by (instance) + avg(node_load1{region="$region"}) > $val`,
+			want:   map[string]string{"my_host": "host", "region": "region"},
+		},
+		{
+			name:   "相同变量出现多次",
+			promql: `sum(rate(mem_used_percent{host="$my_host"})) + avg(node_load1{host="$my_host"}) > $val`,
+			want:   map[string]string{"my_host": "host"},
+		},
+		{
+			name:   "没有变量",
+			promql: `mem_used_percent{host="localhost",region="cn"} > 80`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "没有花括号",
+			promql: `80 > $val`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "格式不规范的标签",
+			promql: `mem_used_percent{host=$my_host,region = $region} > $val`,
+			want:   map[string]string{"my_host": "host", "region": "region"},
+		},
+		{
+			name:   "空花括号",
+			promql: `mem_used_percent{} > $val`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "不完整的花括号",
+			promql: `mem_used_percent{host="$my_host"`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "复杂表达式",
+			promql: `sum(rate(http_requests_total{handler="$handler",code="$code"}[5m])) by (handler) / sum(rate(http_requests_total{handler="$handler"}[5m])) by (handler) * 100 > $threshold`,
+			want:   map[string]string{"handler": "handler", "code": "code"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExtractVarMapping(tt.promql)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ExtractVarMapping() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

alert/process/process.go

@@ -80,8 +80,8 @@ type Processor struct {
 	HandleRecoverEventHook HandleEventFunc
 	EventMuteHook          EventMuteHookFunc
 
-	ScheduleEntry cron.Entry
-	EvalStart     int64
+	ScheduleEntry    cron.Entry
+	PromEvalInterval int
 }
 
 func (p *Processor) Key() string {
@@ -424,6 +424,7 @@ func (p *Processor) handleEvent(events []*models.AlertCurEvent) {
 			p.pendingsUseByRecover.Set(event.Hash, event)
 		}
 
+		event.PromEvalInterval = p.PromEvalInterval
 		if p.rule.PromForDuration == 0 {
 			fireEvents = append(fireEvents, event)
 			if severity > event.Severity {
@@ -442,7 +443,6 @@ func (p *Processor) handleEvent(events []*models.AlertCurEvent) {
 			preTriggerTime = event.TriggerTime
 		}
 
-		event.PromEvalInterval = int(p.ScheduleEntry.Schedule.Next(time.Unix(p.EvalStart, 0)).Unix() - p.EvalStart)
 		if event.LastEvalTime-preTriggerTime+int64(event.PromEvalInterval) >= int64(p.rule.PromForDuration) {
 			fireEvents = append(fireEvents, event)
 			if severity > event.Severity {

alert/sender/webhook.go

@@ -59,17 +59,21 @@ func sendWebhook(webhook *models.Webhook, event interface{}, stats *astats.Stats
 	if webhook != nil {
 		insecureSkipVerify = webhook.SkipVerify
 	}
-	client := http.Client{
-		Timeout: time.Duration(conf.Timeout) * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify},
-		},
+
+	if conf.Client == nil {
+		logger.Warningf("event_%s, event:%s, url: [%s], error: [%s]", channel, string(bs), conf.Url, "client is nil")
+		conf.Client = &http.Client{
+			Timeout: time.Duration(conf.Timeout) * time.Second,
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify},
+			},
+		}
 	}
 
 	stats.AlertNotifyTotal.WithLabelValues(channel).Inc()
 	var resp *http.Response
 	var body []byte
-	resp, err = client.Do(req)
+	resp, err = conf.Client.Do(req)
 
 	if err != nil {
 		stats.AlertNotifyErrorTotal.WithLabelValues(channel).Inc()
@@ -91,7 +95,7 @@ func sendWebhook(webhook *models.Webhook, event interface{}, stats *astats.Stats
 	return false, string(body), nil
 }
 
-func SingleSendWebhooks(ctx *ctx.Context, webhooks []*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
+func SingleSendWebhooks(ctx *ctx.Context, webhooks map[string]*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
 	for _, conf := range webhooks {
 		retryCount := 0
 		for retryCount < 3 {
@@ -106,7 +110,7 @@ func SingleSendWebhooks(ctx *ctx.Context, webhooks []*models.Webhook, event *mod
 	}
 }
 
-func BatchSendWebhooks(ctx *ctx.Context, webhooks []*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
+func BatchSendWebhooks(ctx *ctx.Context, webhooks map[string]*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
 	for _, conf := range webhooks {
 		logger.Infof("push event:%+v to queue:%v", event, conf)
 		PushEvent(ctx, conf, event, stats)

center/router/router_target.go

@@ -28,17 +28,19 @@ func (rt *Router) targetGetsByHostFilter(c *gin.Context) {
 	var f TargetQuery
 	ginx.BindJSON(c, &f)
 
-	query := models.GetHostsQuery(f.Filters)
-
-	hosts, err := models.TargetGetsByFilter(rt.Ctx, query, f.Limit, (f.P-1)*f.Limit)
-	ginx.Dangerous(err)
-
-	total, err := models.TargetCountByFilter(rt.Ctx, query)
-	ginx.Dangerous(err)
+	// todo 这里也走缓存吗？有 limit 和 offset
+	hosts := rt.TargetCache.GetHostIdentsQuery(f.Filters)
+	//query := models.GetHostsQuery(f.Filters)
+	//
+	//hosts, err := models.TargetGetsByFilter(rt.Ctx, query, f.Limit, (f.P-1)*f.Limit)
+	//ginx.Dangerous(err)
+	//
+	//total, err := models.TargetCountByFilter(rt.Ctx, query)
+	//ginx.Dangerous(err)
 
 	ginx.NewRender(c).Data(gin.H{
 		"list":  hosts,
-		"total": total,
+		"total": len(hosts),
 	}, nil)
 }
 
@@ -537,7 +539,7 @@ func (rt *Router) checkTargetPerm(c *gin.Context, idents []string) {
 
 func (rt *Router) targetsOfAlertRule(c *gin.Context) {
 	engineName := ginx.QueryStr(c, "engine_name", "")
-	m, err := models.GetTargetsOfHostAlertRule(rt.Ctx, engineName)
+	m, err := models.GetTargetsOfHostAlertRule(rt.Ctx, engineName, rt.TargetCache.GetHostIdentsQuery)
 	ret := make(map[string]map[int64][]string)
 	for en, v := range m {
 		if en != engineName {

docker/compose-bridge/etc-nightingale/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/compose-host-network-metric-log/etc-nightingale/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/compose-host-network/etc-nightingale/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/compose-postgres/n9eetc_pg/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/migratesql/migrate.sql

@@ -120,4 +120,8 @@ CREATE TABLE `target_busi_group` (
 
 /* v7.7.2 2024-12-02 */
 ALTER TABLE alert_subscribe MODIFY COLUMN rule_ids varchar(1024);
-ALTER TABLE alert_subscribe MODIFY COLUMN busi_groups varchar(4096);
+ALTER TABLE alert_subscribe MODIFY COLUMN busi_groups varchar(4096);
+
+/* v8.0.0-beta.1 2024-12-13 */
+ALTER TABLE `alert_rule` ADD COLUMN `cron_pattern` VARCHAR(64);
+ALTER TABLE `builtin_components` MODIFY COLUMN `logo` mediumtext COMMENT '''logo of component''';

docker/sqlite.sql

@@ -17,6 +17,8 @@ CREATE TABLE `users` (
     `update_by` varchar(64) not null default ''
 );
 
+CREATE UNIQUE INDEX idx_users_username ON `users` (username);
+
 insert into `users`(id, username, nickname, password, roles, create_at, create_by, update_at, update_by) values(1, 'root', '超管', 'root.2020', 'Admin', strftime('%s', 'now'), 'system', strftime('%s', 'now'), 'system');
 
 CREATE TABLE `user_group` (
@@ -182,8 +184,9 @@ CREATE TABLE `board` (
     `create_by` varchar(64) not null default '',
     `update_at` bigint not null default 0,
     `update_by` varchar(64) not null default '',
-    unique (`group_id`, `name`)
+    `public_cate` bigint not null default 0
 );
+CREATE UNIQUE INDEX idx_board_group_id_name ON `board` (group_id, name);
 CREATE INDEX `idx_board_ident` ON `board` (`ident` asc);
 
 -- for dashboard new version
@@ -192,6 +195,15 @@ CREATE TABLE `board_payload` (
     `payload` mediumtext not null
 );
 
+CREATE TABLE `chart` (
+    `id` integer primary key autoincrement,
+    `group_id` integer not null,
+    `configs` text,
+    `weight` integer not null default 0
+);
+
+CREATE INDEX idx_chart_group_id ON `chart` (group_id);
+
 CREATE TABLE `chart_share` (
     `id` integer primary key autoincrement,
     `cluster` varchar(128) not null,
@@ -238,7 +250,9 @@ CREATE TABLE `alert_rule` (
     `create_at` bigint not null default 0,
     `create_by` varchar(64) not null default '',
     `update_at` bigint not null default 0,
-    `update_by` varchar(64) not null default ''
+    `update_by` varchar(64) not null default '',
+    `cron_pattern` varchar(64),
+    `datasource_queries` text
 );
 CREATE INDEX `idx_alert_rule_group_id` ON `alert_rule` (`group_id` asc);
 CREATE INDEX `idx_alert_rule_update_at` ON `alert_rule` (`update_at` asc);
@@ -308,11 +322,18 @@ CREATE TABLE `target` (
     `tags` varchar(512) not null default '',
     `host_ip` varchar(15) default '',
     `agent_version` varchar(255) default '',
+    `host_tags` text,
     `engine_name` varchar(255) default '',
+    `os` varchar(31) default '',
     `update_at` bigint not null default 0
 );
-CREATE INDEX `idx_target_group_id` ON `target` (`group_id` asc);
 
+CREATE INDEX `idx_target_group_id` ON `target` (`group_id` asc);
+CREATE UNIQUE INDEX idx_target_ident ON `target` (ident);
+CREATE INDEX idx_host_ip ON `target` (host_ip);
+CREATE INDEX idx_agent_version ON `target` (agent_version);
+CREATE INDEX idx_engine_name ON `target` (engine_name);
+CREATE INDEX idx_os ON `target` (os);
 
 CREATE TABLE `metric_view` (
     `id` integer primary key autoincrement,
@@ -337,12 +358,14 @@ CREATE TABLE `recording_rule` (
     `disabled` tinyint(1) not null default 0,
     `prom_ql` varchar(8192) not null,
     `prom_eval_interval` int not null,
+    `cron_pattern` varchar(255) default '',
     `append_tags` varchar(255) default '',
     `query_configs` text not null,
     `create_at` bigint default '0',
     `create_by` varchar(64) default '',
     `update_at` bigint default '0',
-    `update_by` varchar(64) default ''
+    `update_by` varchar(64) default '',
+    `datasource_queries` text
 );
 CREATE INDEX `idx_recording_rule_group_id` ON `recording_rule` (`group_id` asc);
 CREATE INDEX `idx_recording_rule_update_at` ON `recording_rule` (`update_at` asc);
@@ -430,6 +453,7 @@ CREATE TABLE `alert_his_event` (
     `trigger_value` varchar(2048) not null,
     `recover_time` bigint not null default 0,
     `last_eval_time` bigint not null default 0,
+    `original_tags` varchar(8192),
     `tags` varchar(1024) not null default '',
     `annotations` text not null,
     `rule_config` text not null
@@ -459,6 +483,8 @@ CREATE INDEX `idx_builtin_components_ident` ON `builtin_components` (`ident` asc
 
 CREATE TABLE `builtin_payloads` (
   `id` integer primary key autoincrement,
+  `component_id` integer not null default 0,
+  `uuid` integer not null,
   `type` varchar(191) not null,
   `component` varchar(191) not null,
   `cate` varchar(191) not null,
@@ -474,6 +500,20 @@ CREATE INDEX `idx_builtin_payloads_component` ON `builtin_payloads` (`component`
 CREATE INDEX `idx_builtin_payloads_name` ON `builtin_payloads` (`name` asc);
 CREATE INDEX `idx_builtin_payloads_cate` ON `builtin_payloads` (`cate` asc);
 CREATE INDEX `idx_builtin_payloads_type` ON `builtin_payloads` (`type` asc);
+CREATE INDEX idx_uuid ON `builtin_payloads` (uuid);
+
+
+CREATE TABLE `notification_record` (
+    `id` integer primary key autoincrement,
+    `event_id` integer not null,
+    `sub_id` integer,
+    `channel` varchar(255) not null,
+    `status` integer,
+    `target` varchar(1024) not null,
+    `details` varchar(2048) default '',
+    `created_at` integer not null
+);
+CREATE INDEX idx_evt ON notification_record (event_id);
 
 CREATE TABLE `task_tpl` (
     `id`        integer primary key autoincrement,
@@ -553,6 +593,8 @@ CREATE TABLE `datasource`
     `updated_by` varchar(64) not null default ''
 );
 
+CREATE UNIQUE INDEX idx_datasource_name ON datasource (name);
+
 CREATE TABLE `builtin_cate` (
     `id` integer primary key autoincrement,
     `name` varchar(191) not null,
@@ -570,6 +612,8 @@ CREATE TABLE `notify_tpl` (
     `update_by` varchar(64) not null default ''
 );
 
+CREATE UNIQUE INDEX idx_notify_tpl_channel ON notify_tpl (channel);
+
 CREATE TABLE `sso_config` (
     `id` integer primary key autoincrement,
     `name` varchar(191) not null unique,
@@ -577,6 +621,8 @@ CREATE TABLE `sso_config` (
     `update_at` bigint not null default 0
 );
 
+CREATE UNIQUE INDEX idx_sso_config_name ON sso_config (name);
+
 CREATE TABLE `es_index_pattern` (
     `id` integer primary key autoincrement,
     `datasource_id` bigint not null default 0,
@@ -591,6 +637,8 @@ CREATE TABLE `es_index_pattern` (
     unique (`datasource_id`, `name`)
 );
 
+CREATE UNIQUE INDEX idx_es_index_pattern_datasource_id_name ON es_index_pattern (datasource_id, name);
+
 CREATE TABLE `builtin_metrics` (
     `id` integer primary key autoincrement,
     `collector` varchar(191) NOT NULL,
@@ -603,13 +651,15 @@ CREATE TABLE `builtin_metrics` (
     `created_at` bigint NOT NULL DEFAULT 0,
     `created_by` varchar(191) NOT NULL DEFAULT '',
     `updated_at` bigint NOT NULL DEFAULT 0,
-    `updated_by` varchar(191) NOT NULL DEFAULT ''
+    `updated_by` varchar(191) NOT NULL DEFAULT '',
+    `uuid integer` not null default 0
 );
--- CREATE UNIQUE INDEX `idx_builtin_metrics_collector_typ_name` ON `builtin_metrics` (`lang`,`collector`, `typ`, `name` asc);
--- CREATE INDEX `idx_builtin_metrics_collector` ON `builtin_metrics` (`collector` asc);
--- CREATE INDEX `idx_builtin_metrics_typ` ON `builtin_metrics` (`typ` asc);
--- CREATE INDEX `idx_builtin_metrics_name` ON `builtin_metrics` (`name` asc);
--- CREATE INDEX `idx_builtin_metrics_lang` ON `builtin_metrics` (`lang` asc);
+
+CREATE UNIQUE INDEX idx_collector_typ_name ON builtin_metrics (lang, collector, typ, name);
+CREATE INDEX idx_collector ON builtin_metrics (collector);
+CREATE INDEX idx_typ ON builtin_metrics (typ);
+CREATE INDEX idx_builtinmetric_name ON builtin_metrics (name);
+CREATE INDEX idx_lang ON builtin_metrics (lang);
 
 
 CREATE TABLE `metric_filter` (
@@ -624,6 +674,14 @@ CREATE TABLE `metric_filter` (
 );
 CREATE INDEX `idx_metric_filter_name` ON `metric_filter` (`name` asc);
 
+CREATE TABLE `target_busi_group` (
+    `id` integer primary key autoincrement,
+    `target_ident` varchar(191) not null,
+    `group_id` integer not null,
+    `update_at` integer not null
+);
+
+CREATE UNIQUE INDEX idx_target_busi_group ON target_busi_group (target_ident, group_id);
 
 CREATE TABLE `task_meta`
 (

etc/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
@@ -73,14 +73,14 @@ DefaultRoles = ["Standard"]
 OpenRSA = false
 
 [DB]
+# mysql postgres sqlite
+DBType = "sqlite"
 # postgres: host=%s port=%s user=%s dbname=%s password=%s sslmode=%s
 # postgres: DSN="host=127.0.0.1 port=5432 user=root dbname=n9e_v6 password=1234 sslmode=disable"
-# sqlite: DSN="/path/to/filename.db"
-DSN = "root:1234@tcp(127.0.0.1:3306)/n9e_v6?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"
+# mysql: DSN="root:1234@tcp(localhost:3306)/n9e_v6?charset=utf8mb4&parseTime=True&loc=Local"
+DSN = "n9e.db"
 # enable debug mode or not
 Debug = false
-# mysql postgres sqlite
-DBType = "mysql"
 # unit: s
 MaxLifetime = 7200
 # max open connections
@@ -98,8 +98,8 @@ Address = "127.0.0.1:6379"
 # DB = 0
 # UseTLS = false
 # TLSMinVersion = "1.2"
-# standalone cluster sentinel
-RedisType = "standalone"
+# standalone cluster sentinel miniredis
+RedisType = "miniredis"
 # Mastername for sentinel type
 # MasterName = "mymaster"
 # SentinelUsername = ""
@@ -138,6 +138,9 @@ ForceUseServerTS = true
 # [Pushgw.WriterOpt]
 # QueueMaxSize = 1000000
 # QueuePopSize = 1000
+# AllQueueMaxSize = 1000000
+# fresh time, unit ms
+# AllQueueMaxSizeInterval = 200
 
 [[Pushgw.Writers]] 
 # Url = "http://127.0.0.1:8480/insert/0/prometheus/api/v1/write"

etc/edge/edge.toml

@@ -54,7 +54,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/flashcatcloud/ibex v1.3.5
 	github.com/gin-contrib/pprof v1.4.0
 	github.com/gin-gonic/gin v1.9.1
+	github.com/glebarez/sqlite v1.11.0
 	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
@@ -42,13 +43,25 @@ require (
 	gorm.io/driver/mysql v1.4.4
 	gorm.io/driver/postgres v1.4.5
 	gorm.io/driver/sqlite v1.5.5
-	gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde
+	gorm.io/gorm v1.25.7
 )
 
-require github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+require (
+	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/glebarez/go-sqlite v1.21.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/yuin/gopher-lua v1.1.1 // indirect
+	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/sqlite v1.23.1 // indirect
+)
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
+	github.com/alicebob/miniredis/v2 v2.33.0
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -97,11 +110,11 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/automaxprocs v1.5.2 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/image v0.18.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

go.sum

@@ -15,6 +15,10 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.33.0 h1:uvTF0EDeu9RLnUEG27Db5I68ESoIxTiXbNUiji6lZrA=
+github.com/alicebob/miniredis/v2 v2.33.0/go.mod h1:MhP4a3EU7aENRi9aO+tHfTBZicLqQevyi/DJpoj6mi0=
 github.com/aws/aws-sdk-go v1.44.302 h1:ST3ko6GrJKn3Xi+nAvxjG3uk/V1pW8KC52WLeIxqqNk=
 github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -49,6 +53,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/expr-lang/expr v1.16.1 h1:Na8CUcMdyGbnNpShY7kzcHCU7WqxuL+hnxgHZ4vaz/A=
 github.com/expr-lang/expr v1.16.1/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
@@ -67,6 +73,10 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
 github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
+github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
+github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
+github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -116,6 +126,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA=
+github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -273,6 +285,9 @@ github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE=
 github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62/go.mod h1:65XQgovT59RWatovFwnwocoUxiI/eENTnOY5GK3STuY=
@@ -328,6 +343,8 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -364,8 +381,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
@@ -401,8 +418,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -425,8 +442,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -444,8 +461,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -500,7 +517,15 @@ gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E=
 gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE=
 gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
-gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg=
-gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
+modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

integrations/MongoDB/dashboards/mongo_by_exporter.json

@@ -1,5 +1,5 @@
 {
-    "name": "MongoDB by instance",
+    "name": "MongoDB Overview by exporter",
     "tags": "Prometheus MongoDB",
     "ident": "",
     "configs": {
@@ -9,11 +9,11 @@
                 "id": "939298f2-b21f-4e2f-9142-c10946cc4032",
                 "layout": {
                     "h": 1,
+                    "i": "939298f2-b21f-4e2f-9142-c10946cc4032",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 0,
-                    "i": "939298f2-b21f-4e2f-9142-c10946cc4032",
-                    "isResizable": false
+                    "y": 0
                 },
                 "name": "Basic Info",
                 "type": "row"
@@ -32,12 +32,12 @@
                 "description": "instance count",
                 "id": "91970d24-3f04-4424-a1ed-73e7d28f5706",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "91970d24-3f04-4424-a1ed-73e7d28f5706",
+                    "isResizable": true,
                     "w": 6,
                     "x": 0,
-                    "y": 1,
-                    "i": "91970d24-3f04-4424-a1ed-73e7d28f5706",
-                    "isResizable": true
+                    "y": 1
                 },
                 "name": "Up",
                 "options": {
@@ -77,54 +77,39 @@
                 "version": "2.0.0"
             },
             {
-                "type": "stat",
-                "id": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
-                "layout": {
-                    "h": 4,
-                    "w": 6,
-                    "x": 6,
-                    "y": 1,
-                    "i": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
-                    "isResizable": true
-                },
-                "version": "3.0.0",
-                "datasourceCate": "prometheus",
-                "datasourceValue": "${prom}",
-                "targets": [
-                    {
-                        "expr": "mongodb_ss_uptime{instance=\"$instance\"}",
-                        "refId": "A",
-                        "maxDataPoints": 240
-                    }
-                ],
-                "transformations": [
-                    {
-                        "id": "organize",
-                        "options": {}
-                    }
-                ],
-                "name": "Uptime",
-                "description": "Uptime",
-                "maxPerRow": 4,
                 "custom": {
-                    "textMode": "value",
-                    "graphMode": "none",
-                    "colorMode": "value",
                     "calc": "lastNotNull",
-                    "valueField": "Value",
                     "colSpan": 1,
+                    "colorMode": "value",
+                    "textMode": "value",
                     "textSize": {
                         "title": null
                     },
-                    "orientation": "auto"
+                    "valueField": "Value"
                 },
+                "datasourceCate": "prometheus",
+                "datasourceValue": "${prom}",
+                "description": "Uptime",
+                "id": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
+                "layout": {
+                    "h": 7,
+                    "i": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
+                    "isResizable": true,
+                    "w": 6,
+                    "x": 6,
+                    "y": 1
+                },
+                "name": "Uptime",
                 "options": {
+                    "standardOptions": {
+                        "util": "humantimeSeconds"
+                    },
                     "thresholds": {
                         "steps": [
                             {
                                 "color": "#634CD9",
-                                "value": null,
-                                "type": "base"
+                                "type": "base",
+                                "value": null
                             }
                         ]
                     },
@@ -147,51 +132,34 @@
                             },
                             "type": "range"
                         }
-                    ],
-                    "standardOptions": {
-                        "util": "seconds",
-                        "decimals": 2
-                    }
+                    ]
                 },
-                "overrides": [
+                "targets": [
                     {
-                        "matcher": {
-                            "id": "byFrameRefID"
-                        },
-                        "properties": {
-                            "thresholds": {
-                                "steps": [
-                                    {
-                                        "color": "#6C53B1",
-                                        "value": null,
-                                        "type": "base"
-                                    }
-                                ]
-                            },
-                            "standardOptions": {
-                                "decimals": 0
-                            }
-                        }
+                        "expr": "mongodb_ss_uptime{instance=\"$instance\"}",
+                        "refId": "A"
                     }
-                ]
+                ],
+                "type": "stat",
+                "version": "2.0.0"
             },
             {
                 "type": "timeseries",
                 "id": "8446dded-9e11-4ee9-bdad-769b193ddf3e",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "8446dded-9e11-4ee9-bdad-769b193ddf3e",
+                    "isResizable": true,
                     "w": 6,
                     "x": 12,
-                    "y": 1,
-                    "i": "8446dded-9e11-4ee9-bdad-769b193ddf3e",
-                    "isResizable": true
+                    "y": 1
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "mongodb_ss_mem_resident * 1024 * 1024",
+                        "expr": "mongodb_ss_mem_resident{instance='$instance'} * 1024 * 1024",
                         "legend": "{{type}}",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -219,8 +187,7 @@
                         "selectMode": "single"
                     },
                     "standardOptions": {
-                        "util": "bytesIEC",
-                        "decimals": 2
+                        "util": "bytesIEC"
                     },
                     "thresholds": {
                         "steps": [
@@ -275,12 +242,12 @@
                 "description": "Page faults indicate that requests are processed from disk either because an index is missing or there is not enough memory for the data set. Consider increasing memory or sharding out.",
                 "id": "3eda28e7-2480-4ddc-b346-89ced1c33034",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "3eda28e7-2480-4ddc-b346-89ced1c33034",
+                    "isResizable": true,
                     "w": 6,
                     "x": 18,
-                    "y": 1,
-                    "i": "3eda28e7-2480-4ddc-b346-89ced1c33034",
-                    "isResizable": true
+                    "y": 1
                 },
                 "name": "Page Faults",
                 "options": {
@@ -333,12 +300,12 @@
                 "description": "Network traffic (bytes)",
                 "id": "528d0485-f947-470d-95f3-59eae157ebb6",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "528d0485-f947-470d-95f3-59eae157ebb6",
+                    "isResizable": true,
                     "w": 6,
                     "x": 0,
-                    "y": 5,
-                    "i": "528d0485-f947-470d-95f3-59eae157ebb6",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Network I/O",
                 "options": {
@@ -395,12 +362,12 @@
                 "description": "Number of connections Keep in mind the hard limit on the maximum number of connections set by your distribution.",
                 "id": "067e97c3-4e57-447f-a9dc-a49627b6ce18",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "067e97c3-4e57-447f-a9dc-a49627b6ce18",
+                    "isResizable": true,
                     "w": 6,
                     "x": 6,
-                    "y": 5,
-                    "i": "067e97c3-4e57-447f-a9dc-a49627b6ce18",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Connections",
                 "options": {
@@ -450,12 +417,12 @@
                 "description": "Number of assertion errors, Asserts are not important by themselves, but you can correlate spikes with other graphs.",
                 "id": "9e9b7356-cf0e-4e5f-95f5-00258c576bf4",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "9e9b7356-cf0e-4e5f-95f5-00258c576bf4",
+                    "isResizable": true,
                     "w": 6,
                     "x": 12,
-                    "y": 5,
-                    "i": "9e9b7356-cf0e-4e5f-95f5-00258c576bf4",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Assert Events",
                 "options": {
@@ -505,12 +472,12 @@
                 "description": "Number of operations waiting to acquire locks, Any number of queued operations for long periods of time is an indication of possible issues. Find the cause and fix it before requests get stuck in the queue.",
                 "id": "2698f0f8-a76a-499b-99cf-30504f0f4db6",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "2698f0f8-a76a-499b-99cf-30504f0f4db6",
+                    "isResizable": true,
                     "w": 6,
                     "x": 18,
-                    "y": 5,
-                    "i": "2698f0f8-a76a-499b-99cf-30504f0f4db6",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Lock Queue",
                 "options": {
@@ -547,11 +514,11 @@
                 "id": "2bdb8cc9-92f4-449e-8f70-a4c470a21604",
                 "layout": {
                     "h": 1,
+                    "i": "2bdb8cc9-92f4-449e-8f70-a4c470a21604",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 9,
-                    "i": "2bdb8cc9-92f4-449e-8f70-a4c470a21604",
-                    "isResizable": false
+                    "y": 15
                 },
                 "name": "Operation Info",
                 "type": "row"
@@ -574,12 +541,12 @@
                 "description": "Number of requests received Shows how many times a command is executed per second on average during the selected interval.",
                 "id": "c2819508-95e7-4c63-aeae-ce19f92469cd",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "c2819508-95e7-4c63-aeae-ce19f92469cd",
+                    "isResizable": true,
                     "w": 12,
                     "x": 0,
-                    "y": 10,
-                    "i": "c2819508-95e7-4c63-aeae-ce19f92469cd",
-                    "isResizable": true
+                    "y": 16
                 },
                 "name": "Command Operations",
                 "options": {
@@ -625,12 +592,12 @@
                 "type": "timeseries",
                 "id": "7030d97a-d69f-4916-a415-ec57503ab1ed",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "7030d97a-d69f-4916-a415-ec57503ab1ed",
+                    "isResizable": true,
                     "w": 12,
                     "x": 12,
-                    "y": 10,
-                    "i": "7030d97a-d69f-4916-a415-ec57503ab1ed",
-                    "isResizable": true
+                    "y": 16
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
@@ -704,19 +671,19 @@
                 "type": "timeseries",
                 "id": "1c3b73d5-c25c-449f-995d-26acc9c621e1",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "1c3b73d5-c25c-449f-995d-26acc9c621e1",
+                    "isResizable": true,
                     "w": 8,
                     "x": 0,
-                    "y": 15,
-                    "i": "1c3b73d5-c25c-449f-995d-26acc9c621e1",
-                    "isResizable": true
+                    "y": 23
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "rate(mongodb_ss_opLatencies_latency{}[5m]) / rate(mongodb_ss_opLatencies_latency{}[5m]) / 1000",
+                        "expr": "rate(mongodb_ss_opLatencies_latency{instance='$instance'}[5m]) / rate(mongodb_ss_opLatencies_latency{instance='$instance'}[5m]) / 1000",
                         "legend": "{{op_type}}",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -799,12 +766,12 @@
                 "description": "",
                 "id": "e642183c-8ba2-4f60-abc6-c65de49e7577",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "e642183c-8ba2-4f60-abc6-c65de49e7577",
+                    "isResizable": true,
                     "w": 8,
                     "x": 8,
-                    "y": 15,
-                    "i": "e642183c-8ba2-4f60-abc6-c65de49e7577",
-                    "isResizable": true
+                    "y": 23
                 },
                 "name": "Query Efficiency",
                 "options": {
@@ -861,12 +828,12 @@
                 "description": "number of cursors Helps identify why connections are increasing. Shows active cursors compared to cursors being automatically killed after 10 minutes due to an application not closing the connection.",
                 "id": "8b5a4f44-3291-4822-ab73-f56be6c62674",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "8b5a4f44-3291-4822-ab73-f56be6c62674",
+                    "isResizable": true,
                     "w": 8,
                     "x": 16,
-                    "y": 15,
-                    "i": "8b5a4f44-3291-4822-ab73-f56be6c62674",
-                    "isResizable": true
+                    "y": 23
                 },
                 "name": "Cursors",
                 "options": {
@@ -903,11 +870,11 @@
                 "id": "06946b19-94b4-4f72-bd87-70f87989257d",
                 "layout": {
                     "h": 1,
+                    "i": "06946b19-94b4-4f72-bd87-70f87989257d",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 20,
-                    "i": "06946b19-94b4-4f72-bd87-70f87989257d",
-                    "isResizable": false
+                    "y": 30
                 },
                 "name": "Cache Info",
                 "panels": [],
@@ -917,19 +884,19 @@
                 "type": "timeseries",
                 "id": "bb0ae571-43a1-430b-8f63-256f6f1ebee6",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "bb0ae571-43a1-430b-8f63-256f6f1ebee6",
+                    "isResizable": true,
                     "w": 6,
                     "x": 0,
-                    "y": 21,
-                    "i": "bb0ae571-43a1-430b-8f63-256f6f1ebee6",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "mongodb_ss_wt_cache_bytes_currently_in_the_cache{}",
+                        "expr": "mongodb_ss_wt_cache_bytes_currently_in_the_cache{instance='$instance'}",
                         "legend": "total",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -975,8 +942,7 @@
                         "selectMode": "single"
                     },
                     "standardOptions": {
-                        "util": "bytesIEC",
-                        "decimals": 2
+                        "util": "bytesIEC"
                     },
                     "thresholds": {
                         "steps": [
@@ -1017,19 +983,19 @@
                 "type": "timeseries",
                 "id": "f1ffd169-2a1a-42bc-9647-0e6621be0fef",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "f1ffd169-2a1a-42bc-9647-0e6621be0fef",
+                    "isResizable": true,
                     "w": 6,
                     "x": 6,
-                    "y": 21,
-                    "i": "f1ffd169-2a1a-42bc-9647-0e6621be0fef",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "rate(mongodb_ss_wt_cache_bytes_read_into_cache{}[5m])",
+                        "expr": "rate(mongodb_ss_wt_cache_bytes_read_into_cache{instance='$instance'}[5m])",
                         "legend": "read",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -1104,19 +1070,19 @@
                 "type": "timeseries",
                 "id": "43ee140d-ae6d-474a-9892-fa4743d7f97e",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "43ee140d-ae6d-474a-9892-fa4743d7f97e",
+                    "isResizable": true,
                     "w": 6,
                     "x": 12,
-                    "y": 21,
-                    "i": "43ee140d-ae6d-474a-9892-fa4743d7f97e",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "100 * sum(mongodb_ss_wt_cache_tracked_dirty_pages_in_the_cache{}) / sum(mongodb_ss_wt_cache_pages_currently_held_in_the_cache{})",
+                        "expr": "100 * sum(mongodb_ss_wt_cache_tracked_dirty_pages_in_the_cache{instance='$instance'}) / sum(mongodb_ss_wt_cache_pages_currently_held_in_the_cache{instance='$instance'})",
                         "legend": "dirty rate",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -1185,19 +1151,19 @@
                 "type": "timeseries",
                 "id": "1a22c31a-859a-400c-af2a-ae83c308d0f2",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "1a22c31a-859a-400c-af2a-ae83c308d0f2",
+                    "isResizable": true,
                     "w": 6,
                     "x": 18,
-                    "y": 21,
-                    "i": "1a22c31a-859a-400c-af2a-ae83c308d0f2",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "rate(mongodb_mongod_wiredtiger_cache_evicted_total{}[5m])",
+                        "expr": "rate(mongodb_mongod_wiredtiger_cache_evicted_total{instance='$instance'}[5m])",
                         "legend": "evicted pages",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -1265,95 +1231,125 @@
                 "id": "b0016f4a-c565-4276-a08d-bacdf94b6b5a",
                 "layout": {
                     "h": 1,
+                    "i": "b0016f4a-c565-4276-a08d-bacdf94b6b5a",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 26,
-                    "i": "b0016f4a-c565-4276-a08d-bacdf94b6b5a",
-                    "isResizable": false
+                    "y": 45
                 },
                 "name": "ReplSet Info",
                 "type": "row"
             },
             {
-                "type": "timeseries",
-                "id": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
-                "layout": {
-                    "h": 5,
-                    "w": 12,
-                    "x": 0,
-                    "y": 27,
-                    "i": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
-                    "isResizable": true
+                "custom": {
+                    "calc": "lastNotNull",
+                    "colSpan": 1,
+                    "colorMode": "value",
+                    "textMode": "value",
+                    "textSize": {},
+                    "valueField": "Value"
                 },
-                "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
-                "targets": [
-                    {
-                        "expr": "mongodb_mongod_replset_member_replication_lag{instance=\"$instance\"}",
-                        "legend": "",
-                        "refId": "A",
-                        "maxDataPoints": 240
-                    }
-                ],
-                "transformations": [
-                    {
-                        "id": "organize",
-                        "options": {}
-                    }
-                ],
-                "name": "Replset Lag Seconds",
-                "description": "replica set member master-slave synchronization delay",
-                "maxPerRow": 4,
+                "description": "",
+                "id": "6187ceee-7c25-43f2-be1b-c44ad612ab52",
+                "layout": {
+                    "h": 7,
+                    "i": "6187ceee-7c25-43f2-be1b-c44ad612ab52",
+                    "isResizable": true,
+                    "w": 12,
+                    "x": 0,
+                    "y": 46
+                },
+                "name": "Replset Election",
                 "options": {
-                    "tooltip": {
-                        "mode": "all",
-                        "sort": "none"
-                    },
-                    "legend": {
-                        "displayMode": "hidden",
-                        "heightInPercentage": 30,
-                        "placement": "bottom",
-                        "behaviour": "showItem",
-                        "selectMode": "single"
-                    },
                     "standardOptions": {
+                        "decimals": 1,
                         "util": "seconds"
                     },
                     "thresholds": {
                         "steps": [
                             {
-                                "color": "#6C53B1",
-                                "value": null,
-                                "type": "base"
+                                "color": "#634CD9",
+                                "type": "base",
+                                "value": null
                             }
                         ]
-                    }
+                    },
+                    "valueMappings": [
+                        {
+                            "match": {
+                                "to": 1800
+                            },
+                            "result": {
+                                "color": "#f24526"
+                            },
+                            "type": "range"
+                        },
+                        {
+                            "match": {
+                                "from": 1800
+                            },
+                            "result": {
+                                "color": "#53b503"
+                            },
+                            "type": "range"
+                        }
+                    ]
                 },
+                "targets": [
+                    {
+                        "expr": "time() - mongodb_mongod_replset_member_election_date",
+                        "refId": "A"
+                    }
+                ],
+                "type": "stat",
+                "version": "2.0.0"
+            },
+            {
                 "custom": {
                     "drawStyle": "lines",
-                    "lineInterpolation": "smooth",
-                    "spanNulls": false,
-                    "lineWidth": 2,
                     "fillOpacity": 0.3,
                     "gradientMode": "opacity",
-                    "stack": "off",
-                    "scaleDistribution": {
-                        "type": "linear"
-                    },
-                    "showPoints": "none",
-                    "pointSize": 5
+                    "lineInterpolation": "smooth",
+                    "lineWidth": 2,
+                    "stack": "off"
                 },
-                "overrides": [
-                    {
-                        "matcher": {
-                            "id": "byFrameRefID"
-                        },
-                        "properties": {
-                            "rightYAxisDisplay": "off"
-                        }
+                "datasourceCate": "prometheus",
+                "datasourceValue": "${prom}",
+                "description": "replica set member master-slave synchronization delay",
+                "id": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
+                "layout": {
+                    "h": 7,
+                    "i": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
+                    "isResizable": true,
+                    "w": 12,
+                    "x": 12,
+                    "y": 46
+                },
+                "name": "Replset Lag Seconds",
+                "options": {
+                    "legend": {
+                        "displayMode": "hidden"
+                    },
+                    "standardOptions": {
+                        "util": "seconds"
+                    },
+                    "thresholds": {},
+                    "tooltip": {
+                        "mode": "all",
+                        "sort": "none"
                     }
-                ]
+                },
+                "targets": [
+                    {
+                        "expr": "mongodb_mongod_replset_member_replication_lag{instance=\"$instance\"}",
+                        "legend": "lag",
+                        "refId": "A"
+                    }
+                ],
+                "type": "timeseries",
+                "version": "2.0.0"
             }
         ],
         "var": [
@@ -1375,4 +1371,4 @@
         "version": "3.0.0"
     },
     "uuid": 1717556328065329000
-}
+}

memsto/host_alert_rule_targets.go

@@ -19,9 +19,11 @@ type TargetsOfAlertRuleCacheType struct {
 
 	sync.RWMutex
 	targets map[string]map[int64][]string // key: ident
+
+	targetCache *TargetCacheType
 }
 
-func NewTargetOfAlertRuleCache(ctx *ctx.Context, engineName string, stats *Stats) *TargetsOfAlertRuleCacheType {
+func NewTargetOfAlertRuleCache(ctx *ctx.Context, engineName string, stats *Stats, targetCache *TargetCacheType) *TargetsOfAlertRuleCacheType {
 	tc := &TargetsOfAlertRuleCacheType{
 		statTotal:       -1,
 		statLastUpdated: -1,
@@ -29,6 +31,7 @@ func NewTargetOfAlertRuleCache(ctx *ctx.Context, engineName string, stats *Stats
 		engineName:      engineName,
 		stats:           stats,
 		targets:         make(map[string]map[int64][]string),
+		targetCache:     targetCache,
 	}
 
 	tc.SyncTargets()
@@ -86,7 +89,7 @@ func (tc *TargetsOfAlertRuleCacheType) loopSyncTargets() {
 }
 
 func (tc *TargetsOfAlertRuleCacheType) syncTargets() error {
-	m, err := models.GetTargetsOfHostAlertRule(tc.ctx, tc.engineName)
+	m, err := models.GetTargetsOfHostAlertRule(tc.ctx, tc.engineName, tc.targetCache.GetHostIdentsQuery)
 	if err != nil {
 		return err
 	}

memsto/notify_config.go

@@ -1,25 +1,28 @@
 package memsto
 
 import (
+	"crypto/tls"
 	"encoding/json"
+	"net/http"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/ccfos/nightingale/v6/pkg/tplx"
-
-	"github.com/BurntSushi/toml"
 	"github.com/ccfos/nightingale/v6/alert/aconf"
 	"github.com/ccfos/nightingale/v6/dumper"
 	"github.com/ccfos/nightingale/v6/models"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
+	"github.com/ccfos/nightingale/v6/pkg/poster"
+	"github.com/ccfos/nightingale/v6/pkg/tplx"
+
+	"github.com/BurntSushi/toml"
 	"github.com/toolkits/pkg/logger"
 )
 
 type NotifyConfigCacheType struct {
 	ctx         *ctx.Context
 	ConfigCache *ConfigCache
-	webhooks    []*models.Webhook
+	webhooks    map[string]*models.Webhook
 	smtp        aconf.SMTPConfig
 	script      models.NotifyScript
 
@@ -47,6 +50,7 @@ func NewNotifyConfigCache(ctx *ctx.Context, configCache *ConfigCache) *NotifyCon
 	w := &NotifyConfigCacheType{
 		ctx:         ctx,
 		ConfigCache: configCache,
+		webhooks:    make(map[string]*models.Webhook),
 	}
 	w.SyncNotifyConfigs()
 	return w
@@ -85,11 +89,60 @@ func (w *NotifyConfigCacheType) syncNotifyConfigs() error {
 	}
 
 	if strings.TrimSpace(cval) != "" {
-		err = json.Unmarshal([]byte(cval), &w.webhooks)
+		var webhooks []*models.Webhook
+		err = json.Unmarshal([]byte(cval), &webhooks)
 		if err != nil {
 			dumper.PutSyncRecord("webhooks", start.Unix(), -1, -1, "failed to unmarshal configs.webhook: "+err.Error())
 			logger.Errorf("failed to unmarshal webhooks:%s error:%v", cval, err)
 		}
+
+		newWebhooks := make(map[string]*models.Webhook, len(webhooks))
+		for i := 0; i < len(webhooks); i++ {
+			if webhooks[i].Batch == 0 {
+				webhooks[i].Batch = 1000
+			}
+
+			if webhooks[i].Timeout == 0 {
+				webhooks[i].Timeout = 10
+			}
+
+			if webhooks[i].RetryCount == 0 {
+				webhooks[i].RetryCount = 10
+			}
+
+			if webhooks[i].RetryInterval == 0 {
+				webhooks[i].RetryInterval = 10
+			}
+
+			if webhooks[i].Client == nil {
+				transport := &http.Transport{
+					TLSClientConfig: &tls.Config{InsecureSkipVerify: webhooks[i].SkipVerify},
+				}
+				if poster.UseProxy(webhooks[i].Url) {
+					transport.Proxy = http.ProxyFromEnvironment
+				}
+				webhooks[i].Client = &http.Client{
+					Timeout:   time.Second * time.Duration(webhooks[i].Timeout),
+					Transport: transport,
+				}
+			}
+
+			newWebhooks[webhooks[i].Url] = webhooks[i]
+		}
+
+		for url, wh := range newWebhooks {
+			if oldWh, has := w.webhooks[url]; has && oldWh.Hash() != wh.Hash() {
+				w.webhooks[url] = wh
+			} else {
+				w.webhooks[url] = wh
+			}
+		}
+
+		for url := range w.webhooks {
+			if _, has := newWebhooks[url]; !has {
+				delete(w.webhooks, url)
+			}
+		}
 	}
 
 	dumper.PutSyncRecord("webhooks", start.Unix(), time.Since(start).Milliseconds(), len(w.webhooks), "success, webhooks:\n"+cval)
@@ -133,7 +186,7 @@ func (w *NotifyConfigCacheType) syncNotifyConfigs() error {
 	return nil
 }
 
-func (w *NotifyConfigCacheType) GetWebhooks() []*models.Webhook {
+func (w *NotifyConfigCacheType) GetWebhooks() map[string]*models.Webhook {
 	w.RWMutex.RLock()
 	defer w.RWMutex.RUnlock()
 	return w.webhooks

memsto/target_cache.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"log"
 	"math"
+	"regexp"
+	"strings"
 	"sync"
 	"time"
 
@@ -27,7 +29,8 @@ type TargetCacheType struct {
 	redis           storage.Redis
 
 	sync.RWMutex
-	targets map[string]*models.Target // key: ident
+	targets       map[string]*models.Target // key: ident
+	groupToIdents map[int64][]string        // key: group_id
 }
 
 func NewTargetCache(ctx *ctx.Context, stats *Stats, redis storage.Redis) *TargetCacheType {
@@ -61,9 +64,10 @@ func (tc *TargetCacheType) StatChanged(total, lastUpdated int64) bool {
 	return true
 }
 
-func (tc *TargetCacheType) Set(m map[string]*models.Target, total, lastUpdated int64) {
+func (tc *TargetCacheType) Set(m map[string]*models.Target, groupToIdents map[int64][]string, total, lastUpdated int64) {
 	tc.Lock()
 	tc.targets = m
+	tc.groupToIdents = groupToIdents
 	tc.Unlock()
 
 	// only one goroutine used, so no need lock
@@ -160,6 +164,7 @@ func (tc *TargetCacheType) syncTargets() error {
 	}
 
 	m := make(map[string]*models.Target)
+	groupToIdents := make(map[int64][]string)
 
 	metaMap := tc.GetHostMetas(lst)
 	if len(metaMap) > 0 {
@@ -172,9 +177,12 @@ func (tc *TargetCacheType) syncTargets() error {
 
 	for i := 0; i < len(lst); i++ {
 		m[lst[i].Ident] = lst[i]
+		for _, groupID := range lst[i].GroupIds {
+			groupToIdents[groupID] = append(groupToIdents[groupID], lst[i].Ident)
+		}
 	}
 
-	tc.Set(m, stat.Total, stat.LastUpdated)
+	tc.Set(m, groupToIdents, stat.Total, stat.LastUpdated)
 
 	ms := time.Since(start).Milliseconds()
 	tc.stats.GaugeCronDuration.WithLabelValues("sync_targets").Set(float64(ms))
@@ -292,3 +300,214 @@ func (tc *TargetCacheType) GetHostMetas(targets []*models.Target) map[string]*mo
 
 	return metaMap
 }
+
+func (tc *TargetCacheType) getAllHostIdentsWithoutLock() []string {
+	var idents []string
+	for ident, _ := range tc.targets {
+		idents = append(idents, ident)
+	}
+	return idents
+}
+
+func (tc *TargetCacheType) getHostIdentsByGroupIdsWithoutLock(groupIDs []int64) []string {
+	var targetIdents []string
+	for _, groupID := range groupIDs {
+		if idents, has := tc.groupToIdents[groupID]; has {
+			targetIdents = append(targetIdents, idents...)
+		}
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) getHostIdentsExcludeGroupIdsWithoutLock(groupIDs []int64) []string {
+	var targetIdents []string
+	exclude := make(map[string]struct{})
+	for _, id := range groupIDs {
+		if idents, has := tc.groupToIdents[id]; has {
+			for _, ident := range idents {
+				exclude[ident] = struct{}{}
+			}
+		}
+	}
+
+	for ident, _ := range tc.targets {
+		if _, ok := exclude[ident]; ok {
+			continue
+		}
+		targetIdents = append(targetIdents, ident)
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) getHostsByIdentsWithoutLock(idents []string) []*models.Target {
+	var targets []*models.Target
+	for _, ident := range idents {
+		if target, has := tc.targets[ident]; has {
+			targets = append(targets, target)
+		}
+	}
+	return targets
+}
+
+func (tc *TargetCacheType) getHostIdentsExcludeIdentsWithoutLock(idents []string) []string {
+	var targetIdents []string
+	exclude := make(map[string]struct{})
+	for _, id := range idents {
+		exclude[id] = struct{}{}
+	}
+
+	for ident, _ := range tc.targets {
+		if _, ok := exclude[ident]; ok {
+			continue
+		}
+		targetIdents = append(targetIdents, ident)
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) getHostIdentsMatchIdentsWithoutLock(identPatterns []string) []string {
+	var targetIdents []string
+	for ident, _ := range tc.targets {
+		for _, identPattern := range identPatterns {
+			// 模糊匹配转正则
+			if ok, _ := regexp.Match(strings.Replace(identPattern, "*", ".*", -1), []byte(ident)); ok {
+				targetIdents = append(targetIdents, ident)
+				break
+			}
+		}
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) getHostIdentsByTagsWithoutLock(tags []string) []string {
+	var targetIdents []string
+
+	tagMap := make(map[string]struct{})
+	for _, tag := range tags {
+		tagMap[tag] = struct{}{}
+	}
+
+	for ident, target := range tc.targets {
+		for _, tag := range target.TagsJSON {
+			if _, ok := tagMap[tag]; ok {
+				targetIdents = append(targetIdents, ident)
+				break
+			}
+		}
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) getHostIdentsExcludeTagsWithoutLock(tags []string) []string {
+	var targetIdents []string
+
+	for ident, target := range tc.targets {
+		exclude := false
+		curTags := make(map[string]struct{})
+		for _, tag := range target.TagsJSON {
+			curTags[tag] = struct{}{}
+		}
+		for _, tag := range tags {
+			if _, ok := curTags[tag]; ok {
+				exclude = true
+				break
+			}
+		}
+		if !exclude {
+			targetIdents = append(targetIdents, ident)
+		}
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) getHostIdentsMatchExcludeIdentsWithoutLock(identPatterns []string) []string {
+	var targetIdents []string
+	exclude := make(map[string]struct{})
+	for _, id := range identPatterns {
+		exclude[id] = struct{}{}
+	}
+
+	for ident, _ := range tc.targets {
+		has := false
+		for _, identPattern := range identPatterns {
+			if ok, _ := regexp.Match(strings.Replace(identPattern, "*", ".*", -1), []byte(ident)); ok {
+				has = true
+				break
+			}
+		}
+		if !has {
+			targetIdents = append(targetIdents, ident)
+		}
+	}
+	return targetIdents
+}
+
+func (tc *TargetCacheType) GetHostIdentsQuery(queries []models.HostQuery) []*models.Target {
+	tc.Lock()
+	defer tc.Unlock()
+
+	targetIdents := tc.getAllHostIdentsWithoutLock()
+
+	for _, q := range queries {
+		var cur []string
+		switch q.Key {
+		case "group_ids":
+			ids := models.ParseInt64(q.Values)
+			if q.Op == "==" {
+				cur = tc.getHostIdentsByGroupIdsWithoutLock(ids)
+			} else {
+				cur = tc.getHostIdentsExcludeGroupIdsWithoutLock(ids)
+			}
+		case "tags":
+			var tags []string
+			for _, v := range q.Values {
+				if v == nil {
+					continue
+				}
+				tags = append(tags, v.(string))
+			}
+			if q.Op == "==" {
+				cur = tc.getHostIdentsByTagsWithoutLock(tags)
+			} else {
+				cur = tc.getHostIdentsExcludeTagsWithoutLock(tags)
+			}
+		case "hosts":
+			var idents []string
+			for _, v := range q.Values {
+				if v == nil {
+					continue
+				}
+				idents = append(idents, v.(string))
+			}
+			if q.Op == "==" {
+				cur = idents
+			} else if q.Op == "!=" {
+				cur = tc.getHostIdentsExcludeIdentsWithoutLock(idents)
+			} else if q.Op == "=~" {
+				cur = tc.getHostIdentsMatchIdentsWithoutLock(idents)
+			} else if q.Op == "!~" {
+				cur = tc.getHostIdentsMatchExcludeIdentsWithoutLock(idents)
+			}
+		default:
+			// all_hosts 与其他未知条件不改变已有集合
+			cur = targetIdents
+		}
+		targetIdents = intersection(targetIdents, cur)
+	}
+	return tc.getHostsByIdentsWithoutLock(targetIdents)
+}
+
+func intersection(a, b []string) []string {
+	m := make(map[string]struct{})
+	for _, v := range a {
+		m[v] = struct{}{}
+	}
+
+	var c []string
+	for _, v := range b {
+		if _, ok := m[v]; ok {
+			c = append(c, v)
+		}
+	}
+	return c
+}

models/alert_rule.go

@@ -372,12 +372,14 @@ func GetHostsQuery(queries []HostQuery) []map[string]interface{} {
 					blank += " "
 				}
 			} else {
-				blank := " "
+				var args []interface{}
+				var query []string
 				for _, tag := range lst {
-					m["tags not like ?"+blank] = "%" + tag + "%"
-					m["host_tags not like ?"+blank] = "%" + tag + "%"
-					blank += " "
+					query = append(query, "tags not like ?",
+						"(host_tags not like ? or host_tags is null)")
+					args = append(args, "%"+tag+"%", "%"+tag+"%")
 				}
+				m[strings.Join(query, " and ")] = args
 			}
 		case "hosts":
 			lst := []string{}
@@ -398,11 +400,13 @@ func GetHostsQuery(queries []HostQuery) []map[string]interface{} {
 					blank += " "
 				}
 			} else if q.Op == "!~" {
-				blank := " "
+				var args []interface{}
+				var query []string
 				for _, host := range lst {
-					m["ident not like ?"+blank] = strings.ReplaceAll(host, "*", "%")
-					blank += " "
+					query = append(query, "ident not like ?")
+					args = append(args, strings.ReplaceAll(host, "*", "%"))
 				}
+				m[strings.Join(query, " and ")] = args
 			}
 		}
 		query = append(query, m)
@@ -1227,7 +1231,7 @@ func AlertRuleUpgradeToV6(ctx *ctx.Context, dsm map[string]Datasource) error {
 	return nil
 }
 
-func GetTargetsOfHostAlertRule(ctx *ctx.Context, engineName string) (map[string]map[int64][]string, error) {
+func GetTargetsOfHostAlertRule(ctx *ctx.Context, engineName string, getTargetFunc func(queries []HostQuery) []*Target) (map[string]map[int64][]string, error) {
 	if !ctx.IsCenter {
 		m, err := poster.GetByUrls[map[string]map[int64][]string](ctx, "/v1/n9e/targets-of-alert-rule?engine_name="+engineName)
 		return m, err
@@ -1251,16 +1255,9 @@ func GetTargetsOfHostAlertRule(ctx *ctx.Context, engineName string) (map[string]
 			continue
 		}
 
-		query := GetHostsQuery(rule.Queries)
-		session := TargetFilterQueryBuild(ctx, query, 0, 0)
-		var lst []*Target
-		err := session.Find(&lst).Error
-		if err != nil {
-			logger.Errorf("failed to query targets: %v", err)
-			continue
-		}
+		hosts := getTargetFunc(rule.Queries)
 
-		for _, target := range lst {
+		for _, target := range hosts {
 			if _, exists := m[target.EngineName]; !exists {
 				m[target.EngineName] = make(map[int64][]string)
 			}

models/builtin_component.go

@@ -20,6 +20,17 @@ type BuiltinComponent struct {
 	UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
 }
 
+type PostgresBuiltinComponent struct {
+	ID        uint64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
+	Ident     string `json:"ident" gorm:"type:varchar(191);not null;uniqueIndex:idx_ident,sort:asc;comment:'identifier of component'"`
+	Logo      string `json:"logo" gorm:"type:text;comment:'logo of component'"`
+	Readme    string `json:"readme" gorm:"type:text;not null;comment:'readme of component'"`
+	CreatedAt int64  `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
+	CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
+	UpdatedAt int64  `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
+	UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
+}
+
 func (bc *BuiltinComponent) TableName() string {
 	return "builtin_components"
 }

models/busi_group.go

@@ -115,68 +115,75 @@ func BusiGroupExists(ctx *ctx.Context, where string, args ...interface{}) (bool,
 	return num > 0, err
 }
 
+// RegisterGroupDelCheckEntries 提供给外部注册删除 group 时需要检查的表
+func RegisterGroupDelCheckEntries(e []CheckEntry) {
+	entries = append(entries, e...)
+}
+
+type CheckEntry struct {
+	Entry        interface{}
+	ErrorMessage string
+	FieldName    string
+}
+
+var entries = []CheckEntry{
+	{
+		Entry:        &AlertRule{},
+		ErrorMessage: "Some alert rules still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &AlertMute{},
+		ErrorMessage: "Some alert mutes still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &AlertSubscribe{},
+		ErrorMessage: "Some alert subscribes still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &Board{},
+		ErrorMessage: "Some Board still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &Target{},
+		ErrorMessage: "Some targets still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &RecordingRule{},
+		ErrorMessage: "Some recording rules still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &TaskTpl{},
+		ErrorMessage: "Some recovery scripts still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &TaskRecord{},
+		ErrorMessage: "Some Task Record records still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &TargetBusiGroup{},
+		ErrorMessage: "Some target busigroups still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+}
+
 func (bg *BusiGroup) Del(ctx *ctx.Context) error {
-	has, err := Exists(DB(ctx).Model(&AlertMute{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
+	for _, e := range entries {
+		has, err := Exists(DB(ctx).Model(e.Entry).Where(fmt.Sprintf("%s=?", e.FieldName), bg.Id))
+		if err != nil {
+			return err
+		}
 
-	if has {
-		return errors.New("Some alert mutes still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&AlertSubscribe{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some alert subscribes still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&TargetBusiGroup{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some targets still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&Board{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some dashboards still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&TaskTpl{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some recovery scripts still in the BusiGroup")
-	}
-
-	// hasCR, err := Exists(DB(ctx).Table("collect_rule").Where("group_id=?", bg.Id))
-	// if err != nil {
-	// 	return err
-	// }
-
-	// if hasCR {
-	// 	return errors.New("Some collect rules still in the BusiGroup")
-	// }
-
-	has, err = Exists(DB(ctx).Model(&AlertRule{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some alert rules still in the BusiGroup")
+		if has {
+			return errors.New(e.ErrorMessage)
+		}
 	}
 
 	return DB(ctx).Transaction(func(tx *gorm.DB) error {

models/configs.go

@@ -130,7 +130,7 @@ func ConfigsGetAll(ctx *ctx.Context) ([]*Configs, error) { // select built-in ty
 	}
 
 	var lst []*Configs
-	err := DB(ctx).Model(&Configs{}).Select("ckey, cval").
+	err := DB(ctx).Model(&Configs{}).Select("id, ckey, cval").
 		Where("ckey!='' and external=? ", 0).Find(&lst).Error
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to query configs")

models/es_index_pattern.go

@@ -20,6 +20,7 @@ type EsIndexPattern struct {
 	CreateBy                   string `json:"create_by"`
 	UpdateAt                   int64  `json:"update_at"`
 	UpdateBy                   string `json:"update_by"`
+	CrossClusterEnabled        int    `json:"cross_cluster_enabled"`
 }
 
 func (t *EsIndexPattern) TableName() string {

models/migrate/migrate.go

@@ -38,13 +38,22 @@ func MigrateIbexTables(db *gorm.DB) {
 
 	for i := 0; i < 100; i++ {
 		tableName := fmt.Sprintf("task_host_%d", i)
-		err := db.Table(tableName).AutoMigrate(&imodels.TaskHost{})
-		if err != nil {
-			logger.Errorf("failed to migrate table:%s %v", tableName, err)
+		exists := db.Migrator().HasTable(tableName)
+		if exists {
+			continue
+		} else {
+			err := db.Table(tableName).AutoMigrate(&imodels.TaskHost{})
+			if err != nil {
+				logger.Errorf("failed to migrate table:%s %v", tableName, err)
+			}
 		}
 	}
 }
 
+func isPostgres(db *gorm.DB) bool {
+	dialect := db.Dialector.Name()
+	return dialect == "postgres"
+}
 func MigrateTables(db *gorm.DB) error {
 	var tableOptions string
 	switch db.Dialector.(type) {
@@ -54,12 +63,17 @@ func MigrateTables(db *gorm.DB) error {
 	if tableOptions != "" {
 		db = db.Set("gorm:table_options", tableOptions)
 	}
-
 	dts := []interface{}{&RecordingRule{}, &AlertRule{}, &AlertSubscribe{}, &AlertMute{},
 		&TaskRecord{}, &ChartShare{}, &Target{}, &Configs{}, &Datasource{}, &NotifyTpl{},
 		&Board{}, &BoardBusigroup{}, &Users{}, &SsoConfig{}, &models.BuiltinMetric{},
-		&models.MetricFilter{}, &models.BuiltinComponent{}, &models.NotificaitonRecord{},
-		&models.TargetBusiGroup{}}
+		&models.MetricFilter{}, &models.NotificaitonRecord{},
+		&models.TargetBusiGroup{}, &EsIndexPatternMigrate{}}
+
+	if isPostgres(db) {
+		dts = append(dts, &models.PostgresBuiltinComponent{})
+	} else {
+		dts = append(dts, &models.BuiltinComponent{})
+	}
 
 	if !db.Migrator().HasColumn(&imodels.TaskSchedulerHealth{}, "scheduler") {
 		dts = append(dts, &imodels.TaskSchedulerHealth{})
@@ -78,7 +92,7 @@ func MigrateTables(db *gorm.DB) error {
 
 			for _, dt := range asyncDts {
 				if err := db.AutoMigrate(dt); err != nil {
-					logger.Errorf("failed to migrate table: %v", err)
+					logger.Errorf("failed to migrate table %+v err:%v", dt, err)
 				}
 			}
 		}()
@@ -174,14 +188,20 @@ func InsertPermPoints(db *gorm.DB) {
 	})
 
 	for _, op := range ops {
-		exists, err := models.Exists(db.Model(&models.RoleOperation{}).Where("operation = ? and role_name = ?", op.Operation, op.RoleName))
+		var count int64
+
+		err := db.Raw("SELECT COUNT(*) FROM role_operation WHERE operation = ? AND role_name = ?",
+			op.Operation, op.RoleName).Scan(&count).Error
+
 		if err != nil {
 			logger.Errorf("check role operation exists failed, %v", err)
 			continue
 		}
-		if exists {
+
+		if count > 0 {
 			continue
 		}
+
 		err = db.Create(&op).Error
 		if err != nil {
 			logger.Errorf("insert role operation failed, %v", err)
@@ -299,3 +319,11 @@ type TaskHostDoing struct {
 func (TaskHostDoing) TableName() string {
 	return "task_host_doing"
 }
+
+type EsIndexPatternMigrate struct {
+	CrossClusterEnabled int `gorm:"column:cross_cluster_enabled;type:int;default:0"`
+}
+
+func (EsIndexPatternMigrate) TableName() string {
+	return "es_index_pattern"
+}

models/migrate/migrate_test.go

@@ -0,0 +1,69 @@
+package migrate
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/ccfos/nightingale/v6/models"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/schema"
+)
+
+func TestInsertPermPoints(t *testing.T) {
+	db, err := gorm.Open(mysql.Open("root:1234@tcp(127.0.0.1:3306)/n9e_v6?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"), &gorm.Config{NamingStrategy: schema.NamingStrategy{
+		SingularTable: true,
+	}})
+	if err != nil {
+		fmt.Printf("failed to connect database: %v", err)
+	}
+
+	var ops []models.RoleOperation
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/alert-mutes/put",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/log/index-patterns",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/help/variable-configs",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Admin",
+		Operation: "/permissions",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/ibex-settings",
+	})
+
+	db = db.Debug()
+	for _, op := range ops {
+		var count int64
+
+		err := db.Raw("SELECT COUNT(*) FROM role_operation WHERE operation = ? AND role_name = ?",
+			op.Operation, op.RoleName).Scan(&count).Error
+		fmt.Printf("count: %d\n", count)
+
+		if err != nil {
+			fmt.Printf("check role operation exists failed, %v", err)
+			continue
+		}
+
+		if count > 0 {
+			continue
+		}
+
+		err = db.Create(&op).Error
+		if err != nil {
+			fmt.Printf("insert role operation failed, %v", err)
+		}
+	}
+}

models/notify_config.go

@@ -1,5 +1,12 @@
 package models
 
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/toolkits/pkg/str"
+)
+
 const WEBHOOKKEY = "webhook"
 const NOTIFYSCRIPT = "notify_script"
 const NOTIFYCHANNEL = "notify_channel"
@@ -24,6 +31,11 @@ type Webhook struct {
 	RetryCount    int               `json:"retry_count"`
 	RetryInterval int               `json:"retry_interval"`
 	Batch         int               `json:"batch"`
+	Client        *http.Client      `json:"-"`
+}
+
+func (w *Webhook) Hash() string {
+	return str.MD5(fmt.Sprintf("%d_%t_%s_%s_%s_%d_%v_%t_%s_%d_%d_%d", w.Type, w.Enable, w.Url, w.BasicAuthUser, w.BasicAuthPass, w.Timeout, w.HeaderMap, w.SkipVerify, w.Note, w.RetryCount, w.RetryInterval, w.Batch))
 }
 
 type NotifyScript struct {

models/target.go

@@ -185,8 +185,16 @@ func BuildTargetWhereWithQuery(query string) BuildTargetWhereOption {
 		if query != "" {
 			arr := strings.Fields(query)
 			for i := 0; i < len(arr); i++ {
-				q := "%" + arr[i] + "%"
-				session = session.Where("ident like ? or host_ip like ? or note like ? or tags like ? or host_tags like ? or os like ?", q, q, q, q, q, q)
+				if strings.HasPrefix(arr[i], "-") {
+					q := "%" + arr[i][1:] + "%"
+					session = session.Where("ident not like ? and host_ip not like ? and "+
+						"note not like ? and tags not like ? and (host_tags not like ? or "+
+						"host_tags is null) and os not like ?", q, q, q, q, q, q)
+				} else {
+					q := "%" + arr[i] + "%"
+					session = session.Where("ident like ? or host_ip like ? or note like ? or "+
+						"tags like ? or host_tags like ? or os like ?", q, q, q, q, q, q)
+				}
 			}
 		}
 		return session
@@ -197,6 +205,8 @@ func BuildTargetWhereWithDowntime(downtime int64) BuildTargetWhereOption {
 	return func(session *gorm.DB) *gorm.DB {
 		if downtime > 0 {
 			session = session.Where("target.update_at < ?", time.Now().Unix()-downtime)
+		} else if downtime < 0 {
+			session = session.Where("target.update_at > ?", time.Now().Unix()+downtime)
 		}
 		return session
 	}
@@ -270,7 +280,11 @@ func TargetFilterQueryBuild(ctx *ctx.Context, query []map[string]interface{}, li
 	for _, q := range query {
 		tx := DB(ctx).Model(&Target{})
 		for k, v := range q {
-			tx = tx.Or(k, v)
+			if strings.Count(k, "?") > 1 {
+				tx = tx.Or(k, v.([]interface{})...)
+			} else {
+				tx = tx.Or(k, v)
+			}
 		}
 		sub = sub.Where(tx)
 	}

pkg/cfg/cfg.go

@@ -29,16 +29,22 @@ func LoadConfigByDir(configDir string, configPtr interface{}) error {
 	if err != nil {
 		return fmt.Errorf("failed to list files under: %s : %v", configDir, err)
 	}
+
+	found := false
+
 	s := NewFileScanner()
 	for _, fpath := range files {
 		switch {
 		case strings.HasSuffix(fpath, ".toml"):
+			found = true
 			s.Read(path.Join(configDir, fpath))
 			tBuf = append(tBuf, s.Data()...)
 			tBuf = append(tBuf, []byte("\n")...)
 		case strings.HasSuffix(fpath, ".json"):
+			found = true
 			loaders = append(loaders, &multiconfig.JSONLoader{Path: path.Join(configDir, fpath)})
 		case strings.HasSuffix(fpath, ".yaml") || strings.HasSuffix(fpath, ".yml"):
+			found = true
 			loaders = append(loaders, &multiconfig.YAMLLoader{Path: path.Join(configDir, fpath)})
 		}
 		if s.Err() != nil {
@@ -46,6 +52,10 @@ func LoadConfigByDir(configDir string, configPtr interface{}) error {
 		}
 	}
 
+	if !found {
+		return fmt.Errorf("fail to found config file, config dir path: %v", configDir)
+	}
+
 	if len(tBuf) != 0 {
 		loaders = append(loaders, &multiconfig.TOMLLoader{Reader: bytes.NewReader(tBuf)})
 	}

pkg/ormx/database_init_test.go

@@ -0,0 +1,60 @@
+package ormx
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gorm.io/driver/mysql"
+	"gorm.io/driver/postgres"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func TestDataBaseInit(t *testing.T) {
+	tests := []struct {
+		name   string
+		config DBConfig
+	}{
+		{
+			name: "MySQL",
+			config: DBConfig{
+				DBType: "mysql",
+				DSN:    "root:1234@tcp(127.0.0.1:3306)/test?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true",
+			},
+		},
+		{
+			name: "Postgres",
+			config: DBConfig{
+				DBType: "postgres",
+				DSN:    "host=127.0.0.1 port=5432 user=postgres dbname=test password=1234 sslmode=disable",
+			},
+		},
+		{
+			name: "SQLite",
+			config: DBConfig{
+				DBType: "sqlite",
+				DSN:    "./test.db",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := createDatabase(tt.config, &gorm.Config{})
+			assert.NoError(t, err)
+			var dialector gorm.Dialector
+			switch tt.config.DBType {
+			case "mysql":
+				dialector = mysql.Open(tt.config.DSN)
+			case "postgres":
+				dialector = postgres.Open(tt.config.DSN)
+			case "sqlite":
+				dialector = sqlite.Open(tt.config.DSN)
+			}
+			db, err := gorm.Open(dialector, &gorm.Config{})
+			assert.NoError(t, err)
+			err = DataBaseInit(tt.config, db)
+			assert.NoError(t, err)
+		})
+	}
+}

pkg/ormx/ormx.go

@@ -2,14 +2,15 @@ package ormx
 
 import (
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"time"
 
 	tklog "github.com/toolkits/pkg/logger"
+    "github.com/glebarez/sqlite"
 	"gorm.io/driver/mysql"
 	"gorm.io/driver/postgres"
-	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
@@ -70,6 +71,234 @@ func (l *TKitLogger) Printf(s string, i ...interface{}) {
 	}
 }
 
+func createDatabase(c DBConfig, gconfig *gorm.Config) error {
+	switch strings.ToLower(c.DBType) {
+	case "mysql":
+		return createMysqlDatabase(c.DSN, gconfig)
+	case "postgres":
+		return createPostgresDatabase(c.DSN, gconfig)
+	case "sqlite":
+		return createSqliteDatabase(c.DSN, gconfig)
+	default:
+		return fmt.Errorf("dialector(%s) not supported", c.DBType)
+	}
+}
+
+func createSqliteDatabase(dsn string, gconfig *gorm.Config) error {
+	tempDialector := sqlite.Open(dsn)
+
+	_, err := gorm.Open(tempDialector, gconfig)
+	if err != nil {
+		return fmt.Errorf("failed to open temporary connection: %v", err)
+	}
+	
+	fmt.Println("sqlite file created")
+
+	return nil
+}
+
+func createPostgresDatabase(dsn string, gconfig *gorm.Config) error {
+	dsnParts := strings.Split(dsn, " ")
+	dbName := ""
+	connectionWithoutDB := ""
+	for _, part := range dsnParts {
+		if strings.HasPrefix(part, "dbname=") {
+			dbName = part[strings.Index(part, "=")+1:]
+		} else {
+			connectionWithoutDB += part
+			connectionWithoutDB += " "
+		}
+	}
+
+	createDBQuery := fmt.Sprintf("CREATE DATABASE %s ENCODING='UTF8' LC_COLLATE='en_US.UTF-8' LC_CTYPE='en_US.UTF-8';", dbName)
+
+	tempDialector := postgres.Open(connectionWithoutDB)
+
+	tempDB, err := gorm.Open(tempDialector, gconfig)
+	if err != nil {
+		return fmt.Errorf("failed to open temporary connection: %v", err)
+	}
+
+	result := tempDB.Exec(createDBQuery)
+	if result.Error != nil {
+		return fmt.Errorf("failed to execute create database query: %v", result.Error)
+	}
+
+	return nil
+}
+
+func createMysqlDatabase(dsn string, gconfig *gorm.Config) error {
+	dsnParts := strings.SplitN(dsn, "/", 2)
+	if len(dsnParts) != 2 {
+		return fmt.Errorf("failed to parse DSN: %s", dsn)
+	}
+
+	connectionInfo := dsnParts[0]
+	dbInfo := dsnParts[1]
+	dbName := dbInfo
+
+	queryIndex := strings.Index(dbInfo, "?")
+	if queryIndex != -1 {
+		dbName = dbInfo[:queryIndex]
+	} else {
+		return fmt.Errorf("failed to parse database name from DSN: %s", dsn)
+	}
+
+	connectionWithoutDB := connectionInfo + "/?" + dbInfo[queryIndex+1:]
+	createDBQuery := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s CHARACTER SET utf8mb4", dbName)
+
+	tempDialector := mysql.Open(connectionWithoutDB)
+
+	tempDB, err := gorm.Open(tempDialector, gconfig)
+	if err != nil {
+		return fmt.Errorf("failed to open temporary connection: %v", err)
+	}
+
+	result := tempDB.Exec(createDBQuery)
+	if result.Error != nil {
+		return fmt.Errorf("failed to execute create database query: %v", result.Error)
+	}
+
+	return nil
+}
+
+func checkDatabaseExist(c DBConfig) (bool, error) {
+	switch strings.ToLower(c.DBType) {
+	case "mysql":
+		return checkMysqlDatabaseExist(c)
+	case "postgres":
+		return checkPostgresDatabaseExist(c)
+	case "sqlite":
+		return checkSqliteDatabaseExist(c)
+	default:
+		return false, fmt.Errorf("dialector(%s) not supported", c.DBType)
+	}
+
+}
+
+func checkSqliteDatabaseExist(c DBConfig) (bool, error) {
+	if _, err := os.Stat(c.DSN); os.IsNotExist(err) {
+		fmt.Printf("sqlite file not exists: %s\n", c.DSN)
+		return false, nil
+	} else {
+		return true, nil
+	}
+}
+
+func checkPostgresDatabaseExist(c DBConfig) (bool, error) {
+	dsnParts := strings.Split(c.DSN, " ")
+	dbName := ""
+	connectionWithoutDB := ""
+	for _, part := range dsnParts {
+		if strings.HasPrefix(part, "dbname=") {
+			dbName = part[strings.Index(part, "=")+1:]
+		} else {
+			connectionWithoutDB += part
+			connectionWithoutDB += " "
+		}
+	}
+
+	dialector := postgres.Open(connectionWithoutDB)
+
+	gconfig := &gorm.Config{
+		NamingStrategy: schema.NamingStrategy{
+			TablePrefix:   c.TablePrefix,
+			SingularTable: true,
+		},
+		Logger: gormLogger,
+	}
+
+	db, err := gorm.Open(dialector, gconfig)
+	if err != nil {
+		return false, fmt.Errorf("failed to open database: %v", err)
+	}
+
+    var databases []string
+    query := genQuery(c)
+    if err := db.Raw(query).Scan(&databases).Error; err != nil {
+        return false, fmt.Errorf("failed to query: %v", err)
+    }
+
+    for _, database := range databases {
+        if database == dbName {
+			fmt.Println("Database exist")
+            return true, nil
+        }
+    }
+
+    return false, nil
+}
+
+func checkMysqlDatabaseExist(c DBConfig) (bool, error) {
+	dsnParts := strings.SplitN(c.DSN, "/", 2)
+    if len(dsnParts) != 2 {
+        return false, fmt.Errorf("failed to parse DSN: %s", c.DSN)
+    }
+
+    connectionInfo := dsnParts[0]
+    dbInfo := dsnParts[1]
+    dbName := dbInfo
+
+    queryIndex := strings.Index(dbInfo, "?")
+    if queryIndex != -1 {
+        dbName = dbInfo[:queryIndex]
+    } else {
+        return false, fmt.Errorf("failed to parse database name from DSN: %s", c.DSN)
+    }
+
+    connectionWithoutDB := connectionInfo + "/?" + dbInfo[queryIndex+1:]
+
+    var dialector gorm.Dialector
+    switch strings.ToLower(c.DBType) {
+    case "mysql":
+        dialector = mysql.Open(connectionWithoutDB)
+    case "postgres":
+        dialector = postgres.Open(connectionWithoutDB)
+    default:
+        return false, fmt.Errorf("unsupported database type: %s", c.DBType)
+    }
+
+    gconfig := &gorm.Config{
+        NamingStrategy: schema.NamingStrategy{
+            TablePrefix:   c.TablePrefix,
+            SingularTable: true,
+        },
+        Logger: gormLogger,
+    }
+
+    db, err := gorm.Open(dialector, gconfig)
+    if err != nil {
+        return false, fmt.Errorf("failed to open database: %v", err)
+    }
+
+    var databases []string
+    query := genQuery(c)
+    if err := db.Raw(query).Scan(&databases).Error; err != nil {
+        return false, fmt.Errorf("failed to query: %v", err)
+    }
+
+    for _, database := range databases {
+        if database == dbName {
+            return true, nil
+        }
+    }
+
+    return false, nil
+}
+
+func genQuery(c DBConfig) string {
+	switch strings.ToLower(c.DBType) {
+    case "mysql":
+        return "SHOW DATABASES"
+    case "postgres":
+        return "SELECT datname FROM pg_database"
+    case "sqlite":
+        return ""
+    default:
+        return ""
+	}
+}
+
 // New Create gorm.DB instance
 func New(c DBConfig) (*gorm.DB, error) {
 	var dialector gorm.Dialector
@@ -95,9 +324,30 @@ func New(c DBConfig) (*gorm.DB, error) {
 		Logger: gormLogger,
 	}
 
+	dbExist, checkErr := checkDatabaseExist(c)
+	if checkErr != nil {
+		return nil, checkErr
+	}
+	if !dbExist {
+		fmt.Println("Database not exist, trying to create it")
+		createErr := createDatabase(c, gconfig)
+		if createErr != nil {
+			return nil, fmt.Errorf("failed to create database: %v", createErr)
+		}
+
+		db, err := gorm.Open(dialector, gconfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to reopen database after creation: %v", err)
+		}
+		err = DataBaseInit(c, db)
+		if err != nil {
+			return nil, fmt.Errorf("failed to init database: %v", err)
+		}
+	}
+
 	db, err := gorm.Open(dialector, gconfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to open database: %v", err)
 	}
 
 	if c.Debug {

pkg/poster/post.go

@@ -63,7 +63,7 @@ func GetByUrl[T any](url string, cfg conf.CenterApi) (T, error) {
 		Timeout: time.Duration(cfg.Timeout) * time.Millisecond,
 	}
 
-	if useProxy(url) {
+	if UseProxy(url) {
 		client.Transport = ProxyTransporter
 	}
 
@@ -147,7 +147,7 @@ func PostByUrl[T any](url string, cfg conf.CenterApi, v interface{}) (t T, err e
 		Timeout: time.Duration(cfg.Timeout) * time.Millisecond,
 	}
 
-	if useProxy(url) {
+	if UseProxy(url) {
 		client.Transport = ProxyTransporter
 	}
 
@@ -195,7 +195,7 @@ var ProxyTransporter = &http.Transport{
 	Proxy: http.ProxyFromEnvironment,
 }
 
-func useProxy(url string) bool {
+func UseProxy(url string) bool {
 	// N9E_PROXY_URL=oapi.dingtalk.com,feishu.com
 	patterns := os.Getenv("N9E_PROXY_URL")
 	if patterns != "" {
@@ -228,7 +228,7 @@ func PostJSON(url string, timeout time.Duration, v interface{}, retries ...int)
 		Timeout: timeout,
 	}
 
-	if useProxy(url) {
+	if UseProxy(url) {
 		client.Transport = ProxyTransporter
 	}
 

pushgw/pconf/conf.go

@@ -24,8 +24,10 @@ type Pushgw struct {
 }
 
 type WriterGlobalOpt struct {
-	QueueMaxSize int
-	QueuePopSize int
+	QueueMaxSize            int
+	QueuePopSize            int
+	AllQueueMaxSize         int
+	AllQueueMaxSizeInterval int
 }
 
 type WriterOptions struct {
@@ -77,6 +79,14 @@ func (p *Pushgw) PreCheck() {
 		p.WriterOpt.QueuePopSize = 1000
 	}
 
+	if p.WriterOpt.AllQueueMaxSize <= 0 {
+		p.WriterOpt.AllQueueMaxSize = 10000000
+	}
+
+	if p.WriterOpt.AllQueueMaxSizeInterval <= 0 {
+		p.WriterOpt.AllQueueMaxSizeInterval = 200
+	}
+
 	if p.WriteConcurrency <= 0 {
 		p.WriteConcurrency = 5000
 	}

pushgw/router/router_remotewrite.go

@@ -114,7 +114,7 @@ func (rt *Router) remoteWrite(c *gin.Context) {
 
 	var (
 		ignoreIdent = ginx.QueryBool(c, "ignore_ident", false)
-		ignoreHost  = ginx.QueryBool(c, "ignore_host", false)
+		ignoreHost  = ginx.QueryBool(c, "ignore_host", true) // 默认值改成 true，要不然答疑成本太高。发版的时候通知 telegraf 用户，让他们设置 ignore_host=false
 		ids         = make(map[string]struct{})
 	)
 

pushgw/writer/writer.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ccfos/nightingale/v6/pkg/fasttime"
@@ -138,9 +139,10 @@ func (w WriterType) Post(req []byte, headers ...map[string]string) error {
 }
 
 type WritersType struct {
-	pushgw   pconf.Pushgw
-	backends map[string]WriterType
-	queues   map[string]*IdentQueue
+	pushgw      pconf.Pushgw
+	backends    map[string]WriterType
+	queues      map[string]*IdentQueue
+	allQueueLen atomic.Value
 	sync.RWMutex
 }
 
@@ -160,14 +162,30 @@ func (ws *WritersType) ReportQueueStats(ident string, identQueue *IdentQueue) (i
 	}
 }
 
+func (ws *WritersType) SetAllQueueLen() {
+	for {
+		curMetricLen := 0
+		ws.RLock()
+		for _, q := range ws.queues {
+			curMetricLen += q.list.Len()
+		}
+		ws.RUnlock()
+		ws.allQueueLen.Store(curMetricLen)
+		time.Sleep(time.Duration(ws.pushgw.WriterOpt.AllQueueMaxSizeInterval) * time.Millisecond)
+	}
+}
+
 func NewWriters(pushgwConfig pconf.Pushgw) *WritersType {
 	writers := &WritersType{
-		backends: make(map[string]WriterType),
-		queues:   make(map[string]*IdentQueue),
-		pushgw:   pushgwConfig,
+		backends:    make(map[string]WriterType),
+		queues:      make(map[string]*IdentQueue),
+		pushgw:      pushgwConfig,
+		allQueueLen: atomic.Value{},
 	}
 
 	writers.Init()
+
+	go writers.SetAllQueueLen()
 	go writers.CleanExpQueue()
 	return writers
 }
@@ -217,6 +235,13 @@ func (ws *WritersType) PushSample(ident string, v interface{}) {
 	}
 
 	identQueue.ts = time.Now().Unix()
+	curLen := ws.allQueueLen.Load().(int)
+	if curLen > ws.pushgw.WriterOpt.AllQueueMaxSize {
+		logger.Warningf("Write %+v full, metric count over limit: %d", v, curLen)
+		CounterPushQueueErrorTotal.WithLabelValues(ident).Inc()
+		return
+	}
+
 	succ := identQueue.list.PushFront(v)
 	if !succ {
 		logger.Warningf("Write channel(%s) full, current channel size: %d", ident, identQueue.list.Len())
@@ -245,6 +270,7 @@ func (ws *WritersType) StartConsumer(identQueue *IdentQueue) {
 
 func (ws *WritersType) Init() error {
 	opts := ws.pushgw.Writers
+	ws.allQueueLen.Store(0)
 
 	for i := 0; i < len(opts); i++ {
 		tlsConf, err := opts[i].ClientConfig.TLSConfig()

storage/redis.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/ccfos/nightingale/v6/pkg/tlsx"
 	"github.com/redis/go-redis/v9"
 	"github.com/toolkits/pkg/logger"
@@ -28,6 +29,7 @@ type Redis redis.Cmdable
 
 func NewRedis(cfg RedisConfig) (Redis, error) {
 	var redisClient Redis
+
 	switch cfg.RedisType {
 	case "standalone", "":
 		redisOptions := &redis.Options{
@@ -88,6 +90,16 @@ func NewRedis(cfg RedisConfig) (Redis, error) {
 
 		redisClient = redis.NewFailoverClient(redisOptions)
 
+	case "miniredis":
+		s, err := miniredis.Run()
+		if err != nil {
+			fmt.Println("failed to init miniredis:", err)
+			os.Exit(1)
+		}
+		redisClient = redis.NewClient(&redis.Options{
+			Addr: s.Addr(),
+		})
+
 	default:
 		fmt.Println("failed to init redis , redis type is illegal:", cfg.RedisType)
 		os.Exit(1)

storage/redis_test.go

@@ -0,0 +1,44 @@
+package storage
+
+import (
+	"context"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMiniRedisMGet(t *testing.T) {
+	s, err := miniredis.Run()
+	if err != nil {
+		t.Fatalf("failed to start miniredis: %v", err)
+	}
+	defer s.Close()
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr: s.Addr(),
+	})
+
+	err = rdb.Ping(context.Background()).Err()
+	if err != nil {
+		t.Fatalf("failed to ping miniredis: %v", err)
+	}
+
+	mp := make(map[string]interface{})
+	mp["key1"] = "value1"
+	mp["key2"] = "value2"
+	mp["key3"] = "value3"
+
+	err = MSet(context.Background(), rdb, mp)
+	if err != nil {
+		t.Fatalf("failed to set miniredis value: %v", err)
+	}
+
+	ctx := context.Background()
+	keys := []string{"key1", "key2", "key3", "key4"}
+	vals := MGet(ctx, rdb, keys)
+
+	expected := [][]byte{[]byte("value1"), []byte("value2"), []byte("value3")}
+	assert.Equal(t, expected, vals)
+}