code refactor

Co-authored-by: Xu Bin <140785332+Reditiny@users.noreply.github.com>

.gitignore

@@ -9,6 +9,7 @@
 *.o
 *.a
 *.so
+*.db
 *.sw[po]
 *.tar.gz
 *.[568vq]
@@ -64,3 +65,5 @@ queries.active
 
 /n9e-*
 n9e.sql
+
+!/datasource

alert/alert.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/ccfos/nightingale/v6/dscache"
+
 	"github.com/ccfos/nightingale/v6/alert/aconf"
 	"github.com/ccfos/nightingale/v6/alert/astats"
 	"github.com/ccfos/nightingale/v6/alert/dispatch"
@@ -21,12 +23,11 @@ import (
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/httpx"
 	"github.com/ccfos/nightingale/v6/pkg/logx"
+	"github.com/ccfos/nightingale/v6/pkg/macros"
 	"github.com/ccfos/nightingale/v6/prom"
 	"github.com/ccfos/nightingale/v6/pushgw/pconf"
 	"github.com/ccfos/nightingale/v6/pushgw/writer"
 	"github.com/ccfos/nightingale/v6/storage"
-	"github.com/ccfos/nightingale/v6/tdengine"
-
 	"github.com/flashcatcloud/ibex/src/cmd/ibex"
 )
 
@@ -66,11 +67,12 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 
 	promClients := prom.NewPromClient(ctx)
 	dispatch.InitRegisterQueryFunc(promClients)
-	tdengineClients := tdengine.NewTdengineClient(ctx, config.Alert.Heartbeat)
 
 	externalProcessors := process.NewExternalProcessors()
 
-	Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, taskTplsCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache)
+	macros.RegisterMacro(macros.MacroInVain)
+	dscache.Init(ctx, false)
+	Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, taskTplsCache, dsCache, ctx, promClients, userCache, userGroupCache)
 
 	r := httpx.GinEngine(config.Global.RunMode, config.HTTP,
 		configCvalCache.PrintBodyPaths, configCvalCache.PrintAccessLog)
@@ -93,7 +95,7 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 
 func Start(alertc aconf.Alert, pushgwc pconf.Pushgw, syncStats *memsto.Stats, alertStats *astats.Stats, externalProcessors *process.ExternalProcessorsType, targetCache *memsto.TargetCacheType, busiGroupCache *memsto.BusiGroupCacheType,
 	alertMuteCache *memsto.AlertMuteCacheType, alertRuleCache *memsto.AlertRuleCacheType, notifyConfigCache *memsto.NotifyConfigCacheType, taskTplsCache *memsto.TaskTplCache, datasourceCache *memsto.DatasourceCacheType, ctx *ctx.Context,
-	promClients *prom.PromClientMap, tdendgineClients *tdengine.TdengineClientMap, userCache *memsto.UserCacheType, userGroupCache *memsto.UserGroupCacheType) {
+	promClients *prom.PromClientMap, userCache *memsto.UserCacheType, userGroupCache *memsto.UserGroupCacheType) {
 	alertSubscribeCache := memsto.NewAlertSubscribeCache(ctx, syncStats)
 	recordingRuleCache := memsto.NewRecordingRuleCache(ctx, syncStats)
 	targetsOfAlertRulesCache := memsto.NewTargetOfAlertRuleCache(ctx, alertc.Heartbeat.EngineName, syncStats)
@@ -106,7 +108,7 @@ func Start(alertc aconf.Alert, pushgwc pconf.Pushgw, syncStats *memsto.Stats, al
 	record.NewScheduler(alertc, recordingRuleCache, promClients, writers, alertStats, datasourceCache)
 
 	eval.NewScheduler(alertc, externalProcessors, alertRuleCache, targetCache, targetsOfAlertRulesCache,
-		busiGroupCache, alertMuteCache, datasourceCache, promClients, tdendgineClients, naming, ctx, alertStats)
+		busiGroupCache, alertMuteCache, datasourceCache, promClients, naming, ctx, alertStats)
 
 	dp := dispatch.NewDispatch(alertRuleCache, userCache, userGroupCache, alertSubscribeCache, targetCache, notifyConfigCache, taskTplsCache, alertc.Alerting, ctx, alertStats)
 	consumer := dispatch.NewConsumer(alertc.Alerting, ctx, dp, promClients)

alert/dispatch/dispatch.go

@@ -293,9 +293,9 @@ func (e *Dispatch) Send(rule *models.AlertRule, event *models.AlertCurEvent, not
 	// handle global webhooks
 	if !event.OverrideGlobalWebhook() {
 		if e.alerting.WebhookBatchSend {
-			sender.BatchSendWebhooks(e.ctx, notifyTarget.ToWebhookList(), event, e.Astats)
+			sender.BatchSendWebhooks(e.ctx, notifyTarget.ToWebhookMap(), event, e.Astats)
 		} else {
-			sender.SingleSendWebhooks(e.ctx, notifyTarget.ToWebhookList(), event, e.Astats)
+			sender.SingleSendWebhooks(e.ctx, notifyTarget.ToWebhookMap(), event, e.Astats)
 		}
 	}
 

alert/dispatch/notify_target.go

@@ -76,52 +76,8 @@ func (s *NotifyTarget) ToCallbackList() []string {
 	return callbacks
 }
 
-func (s *NotifyTarget) ToWebhookList() []*models.Webhook {
-	webhooks := make([]*models.Webhook, 0, len(s.webhooks))
-	for _, wh := range s.webhooks {
-		if wh.Batch == 0 {
-			wh.Batch = 1000
-		}
-
-		if wh.Timeout == 0 {
-			wh.Timeout = 10
-		}
-
-		if wh.RetryCount == 0 {
-			wh.RetryCount = 10
-		}
-
-		if wh.RetryInterval == 0 {
-			wh.RetryInterval = 10
-		}
-
-		webhooks = append(webhooks, wh)
-	}
-	return webhooks
-}
-
 func (s *NotifyTarget) ToWebhookMap() map[string]*models.Webhook {
-	webhookMap := make(map[string]*models.Webhook, len(s.webhooks))
-	for _, wh := range s.webhooks {
-		if wh.Batch == 0 {
-			wh.Batch = 1000
-		}
-
-		if wh.Timeout == 0 {
-			wh.Timeout = 10
-		}
-
-		if wh.RetryCount == 0 {
-			wh.RetryCount = 10
-		}
-
-		if wh.RetryInterval == 0 {
-			wh.RetryInterval = 10
-		}
-
-		webhookMap[wh.Url] = wh
-	}
-	return webhookMap
+	return s.webhooks
 }
 
 func (s *NotifyTarget) ToUidList() []int64 {

alert/eval/alert_rule.go

@@ -13,8 +13,6 @@ import (
 	"github.com/ccfos/nightingale/v6/memsto"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/prom"
-	"github.com/ccfos/nightingale/v6/tdengine"
-
 	"github.com/toolkits/pkg/logger"
 )
 
@@ -33,8 +31,7 @@ type Scheduler struct {
 	alertMuteCache          *memsto.AlertMuteCacheType
 	datasourceCache         *memsto.DatasourceCacheType
 
-	promClients     *prom.PromClientMap
-	tdengineClients *tdengine.TdengineClientMap
+	promClients *prom.PromClientMap
 
 	naming *naming.Naming
 
@@ -45,7 +42,7 @@ type Scheduler struct {
 func NewScheduler(aconf aconf.Alert, externalProcessors *process.ExternalProcessorsType, arc *memsto.AlertRuleCacheType,
 	targetCache *memsto.TargetCacheType, toarc *memsto.TargetsOfAlertRuleCacheType,
 	busiGroupCache *memsto.BusiGroupCacheType, alertMuteCache *memsto.AlertMuteCacheType, datasourceCache *memsto.DatasourceCacheType,
-	promClients *prom.PromClientMap, tdengineClients *tdengine.TdengineClientMap, naming *naming.Naming, ctx *ctx.Context, stats *astats.Stats) *Scheduler {
+	promClients *prom.PromClientMap, naming *naming.Naming, ctx *ctx.Context, stats *astats.Stats) *Scheduler {
 	scheduler := &Scheduler{
 		aconf:      aconf,
 		alertRules: make(map[string]*AlertRuleWorker),
@@ -59,9 +56,8 @@ func NewScheduler(aconf aconf.Alert, externalProcessors *process.ExternalProcess
 		alertMuteCache:          alertMuteCache,
 		datasourceCache:         datasourceCache,
 
-		promClients:     promClients,
-		tdengineClients: tdengineClients,
-		naming:          naming,
+		promClients: promClients,
+		naming:      naming,
 
 		ctx:   ctx,
 		stats: stats,
@@ -95,7 +91,7 @@ func (s *Scheduler) syncAlertRules() {
 		}
 
 		ruleType := rule.GetRuleType()
-		if rule.IsPrometheusRule() || rule.IsLokiRule() || rule.IsTdengineRule() {
+		if rule.IsPrometheusRule() || rule.IsLokiRule() || rule.IsTdengineRule() || rule.IsClickHouseRule() || rule.IsElasticSearch() {
 			datasourceIds := s.datasourceCache.GetIDsByDsCateAndQueries(rule.Cate, rule.DatasourceQueries)
 			for _, dsId := range datasourceIds {
 				if !naming.DatasourceHashRing.IsHit(strconv.FormatInt(dsId, 10), fmt.Sprintf("%d", rule.Id), s.aconf.Heartbeat.Endpoint) {
@@ -118,7 +114,7 @@ func (s *Scheduler) syncAlertRules() {
 				}
 				processor := process.NewProcessor(s.aconf.Heartbeat.EngineName, rule, dsId, s.alertRuleCache, s.targetCache, s.targetsOfAlertRuleCache, s.busiGroupCache, s.alertMuteCache, s.datasourceCache, s.ctx, s.stats)
 
-				alertRule := NewAlertRuleWorker(rule, dsId, processor, s.promClients, s.tdengineClients, s.ctx)
+				alertRule := NewAlertRuleWorker(rule, dsId, processor, s.promClients, s.ctx)
 				alertRuleWorkers[alertRule.Hash()] = alertRule
 			}
 		} else if rule.IsHostRule() {
@@ -127,7 +123,7 @@ func (s *Scheduler) syncAlertRules() {
 				continue
 			}
 			processor := process.NewProcessor(s.aconf.Heartbeat.EngineName, rule, 0, s.alertRuleCache, s.targetCache, s.targetsOfAlertRuleCache, s.busiGroupCache, s.alertMuteCache, s.datasourceCache, s.ctx, s.stats)
-			alertRule := NewAlertRuleWorker(rule, 0, processor, s.promClients, s.tdengineClients, s.ctx)
+			alertRule := NewAlertRuleWorker(rule, 0, processor, s.promClients, s.ctx)
 			alertRuleWorkers[alertRule.Hash()] = alertRule
 		} else {
 			// 如果 rule 不是通过 prometheus engine 来告警的，则创建为 externalRule
@@ -153,6 +149,7 @@ func (s *Scheduler) syncAlertRules() {
 	for hash, rule := range alertRuleWorkers {
 		if _, has := s.alertRules[hash]; !has {
 			rule.Prepare()
+			time.Sleep(time.Duration(20) * time.Millisecond)
 			rule.Start()
 			s.alertRules[hash] = rule
 		}

alert/eval/eval.go

@@ -15,14 +15,15 @@ import (
 
 	"github.com/ccfos/nightingale/v6/alert/common"
 	"github.com/ccfos/nightingale/v6/alert/process"
+	"github.com/ccfos/nightingale/v6/dscache"
 	"github.com/ccfos/nightingale/v6/models"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/hash"
 	"github.com/ccfos/nightingale/v6/pkg/parser"
 	promsdk "github.com/ccfos/nightingale/v6/pkg/prom"
+	promql2 "github.com/ccfos/nightingale/v6/pkg/promql"
 	"github.com/ccfos/nightingale/v6/pkg/unit"
 	"github.com/ccfos/nightingale/v6/prom"
-	"github.com/ccfos/nightingale/v6/tdengine"
 	"github.com/prometheus/common/model"
 
 	"github.com/robfig/cron/v3"
@@ -40,14 +41,15 @@ type AlertRuleWorker struct {
 
 	Processor *process.Processor
 
-	PromClients     *prom.PromClientMap
-	TdengineClients *tdengine.TdengineClientMap
-	Ctx             *ctx.Context
+	PromClients *prom.PromClientMap
+	Ctx         *ctx.Context
 
 	Scheduler *cron.Cron
 
 	HostAndDeviceIdentCache sync.Map
 
+	LastSeriesStore map[uint64]models.DataResp
+
 	DeviceIdentHook func(arw *AlertRuleWorker, paramQuery models.ParamQuery) ([]string, error)
 }
 
@@ -59,6 +61,10 @@ const (
 	QUERY_DATA      = "query_data"
 )
 
+const (
+	JoinMark = "@@"
+)
+
 type JoinType string
 
 const (
@@ -67,7 +73,7 @@ const (
 	Inner JoinType = "inner"
 )
 
-func NewAlertRuleWorker(rule *models.AlertRule, datasourceId int64, Processor *process.Processor, promClients *prom.PromClientMap, tdengineClients *tdengine.TdengineClientMap, ctx *ctx.Context) *AlertRuleWorker {
+func NewAlertRuleWorker(rule *models.AlertRule, datasourceId int64, Processor *process.Processor, promClients *prom.PromClientMap, ctx *ctx.Context) *AlertRuleWorker {
 	arw := &AlertRuleWorker{
 		DatasourceId: datasourceId,
 		Quit:         make(chan struct{}),
@@ -75,12 +81,12 @@ func NewAlertRuleWorker(rule *models.AlertRule, datasourceId int64, Processor *p
 		Processor:    Processor,
 
 		PromClients:             promClients,
-		TdengineClients:         tdengineClients,
 		Ctx:                     ctx,
 		HostAndDeviceIdentCache: sync.Map{},
 		DeviceIdentHook: func(arw *AlertRuleWorker, paramQuery models.ParamQuery) ([]string, error) {
 			return nil, nil
 		},
+		LastSeriesStore: make(map[uint64]models.DataResp),
 	}
 
 	interval := rule.PromEvalInterval
@@ -104,9 +110,17 @@ func NewAlertRuleWorker(rule *models.AlertRule, datasourceId int64, Processor *p
 
 	Processor.ScheduleEntry = arw.Scheduler.Entry(entryID)
 
+	Processor.PromEvalInterval = getPromEvalInterval(Processor.ScheduleEntry.Schedule)
 	return arw
 }
 
+func getPromEvalInterval(schedule cron.Schedule) int {
+	now := time.Now()
+	next1 := schedule.Next(now)
+	next2 := schedule.Next(next1)
+	return int(next2.Sub(next1).Seconds())
+}
+
 func (arw *AlertRuleWorker) Key() string {
 	return common.RuleKey(arw.DatasourceId, arw.Rule.Id)
 }
@@ -125,12 +139,15 @@ func (arw *AlertRuleWorker) Prepare() {
 }
 
 func (arw *AlertRuleWorker) Start() {
-	logger.Infof("eval:%s started", arw.Key())
 	arw.Scheduler.Start()
 }
 
 func (arw *AlertRuleWorker) Eval() {
-	arw.Processor.EvalStart = time.Now().Unix()
+	logger.Infof("eval:%s started", arw.Key())
+	if arw.Processor.PromEvalInterval == 0 {
+		arw.Processor.PromEvalInterval = getPromEvalInterval(arw.Processor.ScheduleEntry.Schedule)
+	}
+
 	cachedRule := arw.Rule
 	if cachedRule == nil {
 		// logger.Errorf("rule_eval:%s Rule not found", arw.Key())
@@ -151,12 +168,10 @@ func (arw *AlertRuleWorker) Eval() {
 		anomalyPoints, err = arw.GetPromAnomalyPoint(cachedRule.RuleConfig)
 	case models.HOST:
 		anomalyPoints, err = arw.GetHostAnomalyPoint(cachedRule.RuleConfig)
-	case models.TDENGINE:
-		anomalyPoints, recoverPoints, err = arw.GetTdengineAnomalyPoint(cachedRule, arw.Processor.DatasourceId())
 	case models.LOKI:
 		anomalyPoints, err = arw.GetPromAnomalyPoint(cachedRule.RuleConfig)
 	default:
-		return
+		anomalyPoints, recoverPoints = arw.GetAnomalyPoint(cachedRule, arw.Processor.DatasourceId())
 	}
 
 	if err != nil {
@@ -240,10 +255,15 @@ func (arw *AlertRuleWorker) GetPromAnomalyPoint(ruleConfig string) ([]models.Ano
 		readerClient := arw.PromClients.GetCli(arw.DatasourceId)
 
 		if query.VarEnabled {
-			anomalyPoints := arw.VarFilling(query, readerClient)
-			for _, v := range anomalyPoints {
-				lst = append(lst, v)
+			var anomalyPoints []models.AnomalyPoint
+			if hasLabelLossAggregator(query) || notExactMatch(query) {
+				// 若有聚合函数或非精确匹配则需要先填充变量然后查询，这个方式效率较低
+				anomalyPoints = arw.VarFillingBeforeQuery(query, readerClient)
+			} else {
+				// 先查询再过滤变量，效率较高，但无法处理有聚合函数的情况
+				anomalyPoints = arw.VarFillingAfterQuery(query, readerClient)
 			}
+			lst = append(lst, anomalyPoints...)
 		} else {
 			// 无变量
 			promql := strings.TrimSpace(query.PromQl)
@@ -297,17 +317,18 @@ type sample struct {
 	Timestamp model.Time
 }
 
-// VarFilling 填充变量
+// VarFillingAfterQuery 填充变量，先查询再填充变量
 // 公式: mem_used_percent{host="$host"} > $val 其中 $host 为参数变量，$val 为值变量
 // 实现步骤:
-// 广度优先遍历，保证同一参数变量的子筛选可以覆盖上一层筛选
-// 每个节点先查询无参数的 query, 即 mem_used_percent > curVal, 得到满足值变量的所有结果
+// 依次遍历参数配置节点，保证同一参数变量的子筛选可以覆盖上一层筛选
+// 每个节点先查询无参数的 query, 即 mem_used_percent{} > curVal, 得到满足值变量的所有结果
 // 结果中有满足本节点参数变量的值，加入异常点列表
 // 参数变量的值不满足的组合，需要覆盖上层筛选中产生的异常点
-func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient promsdk.API) map[string]models.AnomalyPoint {
+func (arw *AlertRuleWorker) VarFillingAfterQuery(query models.PromQuery, readerClient promsdk.API) []models.AnomalyPoint {
+	varToLabel := ExtractVarMapping(query.PromQl)
 	fullQuery := removeVal(query.PromQl)
 	// 存储所有的异常点，key 为参数变量的组合，可以实现子筛选对上一层筛选的覆盖
-	anomalyPoints := make(map[string]models.AnomalyPoint)
+	anomalyPointsMap := make(map[string]models.AnomalyPoint)
 	// 统一变量配置格式
 	VarConfigForCalc := &models.ChildVarConfig{
 		ParamVal:        make([]map[string]models.ParamQuery, 1),
@@ -359,7 +380,7 @@ func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient prom
 			}
 			seqVals := getSamples(value)
 			// 得到参数变量的所有组合
-			paramPermutation, err := arw.getParamPermutation(param, ParamKeys)
+			paramPermutation, err := arw.getParamPermutation(param, ParamKeys, varToLabel, query.PromQl, readerClient)
 			if err != nil {
 				logger.Errorf("rule_eval:%s, paramPermutation error:%v", arw.Key(), err)
 				continue
@@ -369,12 +390,13 @@ func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient prom
 				curRealQuery := realQuery
 				var cur []string
 				for _, paramKey := range ParamKeys {
-					val := string(seqVals[i].Metric[model.LabelName(paramKey)])
+					val := string(seqVals[i].Metric[model.LabelName(varToLabel[paramKey])])
 					cur = append(cur, val)
-					curRealQuery = strings.Replace(curRealQuery, fmt.Sprintf("$%s", paramKey), val, -1)
+					curRealQuery = fillVar(curRealQuery, paramKey, val)
 				}
-				if _, ok := paramPermutation[strings.Join(cur, "-")]; ok {
-					anomalyPoints[strings.Join(cur, "-")] = models.AnomalyPoint{
+
+				if _, ok := paramPermutation[strings.Join(cur, JoinMark)]; ok {
+					anomalyPointsMap[strings.Join(cur, JoinMark)] = models.AnomalyPoint{
 						Key:       seqVals[i].Metric.String(),
 						Timestamp: seqVals[i].Timestamp.Unix(),
 						Value:     float64(seqVals[i].Value),
@@ -383,18 +405,22 @@ func (arw *AlertRuleWorker) VarFilling(query models.PromQuery, readerClient prom
 						Query:     curRealQuery,
 					}
 					// 生成异常点后，删除该参数组合
-					delete(paramPermutation, strings.Join(cur, "-"))
+					delete(paramPermutation, strings.Join(cur, JoinMark))
 				}
 			}
 
 			// 剩余的参数组合为本层筛选不产生异常点的组合，需要覆盖上层筛选中产生的异常点
 			for k, _ := range paramPermutation {
-				delete(anomalyPoints, k)
+				delete(anomalyPointsMap, k)
 			}
 		}
 		curNode = curNode.ChildVarConfigs
 	}
 
+	anomalyPoints := make([]models.AnomalyPoint, 0)
+	for _, point := range anomalyPointsMap {
+		anomalyPoints = append(anomalyPoints, point)
+	}
 	return anomalyPoints
 }
 
@@ -475,7 +501,7 @@ func removeVal(promql string) string {
 }
 
 // 获取参数变量的所有组合
-func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.ParamQuery, paramKeys []string) (map[string]struct{}, error) {
+func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.ParamQuery, paramKeys []string, varToLabel map[string]string, originPromql string, readerClient promsdk.API) (map[string]struct{}, error) {
 
 	// 参数变量查询，得到参数变量值
 	paramMap := make(map[string][]string)
@@ -507,7 +533,15 @@ func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.Param
 			if err != nil {
 				logger.Errorf("query:%s fail to unmarshalling into string slice, error:%v", paramQuery.Query, err)
 			}
-			params = query
+			if len(query) == 0 {
+				paramsKeyAllLabel, err := getParamKeyAllLabel(varToLabel[paramKey], originPromql, readerClient)
+				if err != nil {
+					logger.Errorf("rule_eval:%s, fail to getParamKeyAllLabel, error:%v query:%s", arw.Key(), err, paramQuery.Query)
+				}
+				params = paramsKeyAllLabel
+			} else {
+				params = query
+			}
 		default:
 			return nil, fmt.Errorf("unknown param type: %s", paramQuery.ParamType)
 		}
@@ -516,6 +550,7 @@ func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.Param
 			return nil, fmt.Errorf("param key: %s, params is empty", paramKey)
 		}
 
+		logger.Infof("rule_eval:%s paramKey: %s, params: %v", arw.Key(), paramKey, params)
 		paramMap[paramKey] = params
 	}
 
@@ -524,12 +559,63 @@ func (arw *AlertRuleWorker) getParamPermutation(paramVal map[string]models.Param
 
 	res := make(map[string]struct{})
 	for i := range permutation {
-		res[strings.Join(permutation[i], "-")] = struct{}{}
+		res[strings.Join(permutation[i], JoinMark)] = struct{}{}
 	}
 
 	return res, nil
 }
 
+func getParamKeyAllLabel(paramKey string, promql string, client promsdk.API) ([]string, error) {
+	labels, metricName, err := promql2.GetLabelsAndMetricNameWithReplace(promql, "$")
+	if err != nil {
+		return nil, fmt.Errorf("promql:%s, get labels error:%v", promql, err)
+	}
+	labelstrs := make([]string, 0)
+	for _, label := range labels {
+		if strings.HasPrefix(label.Value, "$") {
+			continue
+		}
+		labelstrs = append(labelstrs, label.Name+label.Op+label.Value)
+	}
+	pr := metricName + "{" + strings.Join(labelstrs, ",") + "}"
+
+	value, _, err := client.Query(context.Background(), pr, time.Now())
+	if err != nil {
+		return nil, fmt.Errorf("promql: %s query error: %v", pr, err)
+	}
+	labelValuesMap := make(map[string]struct{})
+
+	switch value.Type() {
+	case model.ValVector:
+		vector := value.(model.Vector)
+		for _, sample := range vector {
+			for labelName, labelValue := range sample.Metric {
+				// 只处理ParamKeys中指定的label
+				if string(labelName) == paramKey {
+					labelValuesMap[string(labelValue)] = struct{}{}
+				}
+			}
+		}
+	case model.ValMatrix:
+		matrix := value.(model.Matrix)
+		for _, series := range matrix {
+			for labelName, labelValue := range series.Metric {
+				// 只处理ParamKeys中指定的label
+				if string(labelName) == paramKey {
+					labelValuesMap[string(labelValue)] = struct{}{}
+				}
+			}
+		}
+	}
+
+	result := make([]string, 0)
+	for labelValue, _ := range labelValuesMap {
+		result = append(result, labelValue)
+	}
+
+	return result, nil
+}
+
 func (arw *AlertRuleWorker) getHostIdents(paramQuery models.ParamQuery) ([]string, error) {
 	var params []string
 	q, _ := json.Marshal(paramQuery.Query)
@@ -594,69 +680,6 @@ func combine(paramKeys []string, paraMap map[string][]string, index int, current
 	}
 }
 
-func (arw *AlertRuleWorker) GetTdengineAnomalyPoint(rule *models.AlertRule, dsId int64) ([]models.AnomalyPoint, []models.AnomalyPoint, error) {
-	// 获取查询和规则判断条件
-	points := []models.AnomalyPoint{}
-	recoverPoints := []models.AnomalyPoint{}
-	ruleConfig := strings.TrimSpace(rule.RuleConfig)
-	if ruleConfig == "" {
-		logger.Warningf("rule_eval:%d promql is blank", rule.Id)
-		arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_RULE_CONFIG, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
-		return points, recoverPoints, errors.New("rule config is nil")
-	}
-
-	var ruleQuery models.RuleQuery
-	err := json.Unmarshal([]byte(ruleConfig), &ruleQuery)
-	if err != nil {
-		logger.Warningf("rule_eval:%d promql parse error:%s", rule.Id, err.Error())
-		arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_RULE_CONFIG, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
-		return points, recoverPoints, err
-	}
-
-	arw.Inhibit = ruleQuery.Inhibit
-	if len(ruleQuery.Queries) > 0 {
-		seriesStore := make(map[uint64]models.DataResp)
-		// 将不同查询的 hash 索引分组存放
-		seriesTagIndexes := make(map[string]map[uint64][]uint64)
-
-		for _, query := range ruleQuery.Queries {
-			seriesTagIndex := make(map[uint64][]uint64)
-
-			arw.Processor.Stats.CounterQueryDataTotal.WithLabelValues(fmt.Sprintf("%d", arw.DatasourceId)).Inc()
-			cli := arw.TdengineClients.GetCli(dsId)
-			if cli == nil {
-				logger.Warningf("rule_eval:%d tdengine client is nil", rule.Id)
-				arw.Processor.Stats.CounterQueryDataErrorTotal.WithLabelValues(fmt.Sprintf("%d", arw.DatasourceId)).Inc()
-				arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_CLIENT, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
-				continue
-			}
-
-			series, err := cli.Query(query)
-			arw.Processor.Stats.CounterQueryDataTotal.WithLabelValues(fmt.Sprintf("%d", arw.DatasourceId)).Inc()
-			if err != nil {
-				logger.Warningf("rule_eval rid:%d query data error: %v", rule.Id, err)
-				arw.Processor.Stats.CounterQueryDataErrorTotal.WithLabelValues(fmt.Sprintf("%d", arw.DatasourceId)).Inc()
-				arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), QUERY_DATA, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
-				return points, recoverPoints, err
-			}
-			//  此条日志很重要，是告警判断的现场值
-			logger.Debugf("rule_eval rid:%d req:%+v resp:%+v", rule.Id, query, series)
-			MakeSeriesMap(series, seriesTagIndex, seriesStore)
-			ref, err := GetQueryRef(query)
-			if err != nil {
-				logger.Warningf("rule_eval rid:%d query ref error: %v query:%+v", rule.Id, err, query)
-				arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_RULE_CONFIG, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
-				continue
-			}
-			seriesTagIndexes[ref] = seriesTagIndex
-		}
-
-		points, recoverPoints = GetAnomalyPoint(rule.Id, ruleQuery, seriesTagIndexes, seriesStore)
-	}
-
-	return points, recoverPoints, nil
-}
-
 func (arw *AlertRuleWorker) GetHostAnomalyPoint(ruleConfig string) ([]models.AnomalyPoint, error) {
 	var lst []models.AnomalyPoint
 	var severity int
@@ -799,122 +822,6 @@ func (arw *AlertRuleWorker) GetHostAnomalyPoint(ruleConfig string) ([]models.Ano
 	return lst, nil
 }
 
-func GetAnomalyPoint(ruleId int64, ruleQuery models.RuleQuery, seriesTagIndexes map[string]map[uint64][]uint64, seriesStore map[uint64]models.DataResp) ([]models.AnomalyPoint, []models.AnomalyPoint) {
-	points := []models.AnomalyPoint{}
-	recoverPoints := []models.AnomalyPoint{}
-
-	if len(ruleQuery.Triggers) == 0 {
-		return points, recoverPoints
-	}
-
-	if len(seriesTagIndexes) == 0 {
-		return points, recoverPoints
-	}
-
-	unitMap := make(map[string]string)
-	for _, query := range ruleQuery.Queries {
-		ref, unit, err := GetQueryRefAndUnit(query)
-		if err != nil {
-			continue
-		}
-		unitMap[ref] = unit
-	}
-
-	for _, trigger := range ruleQuery.Triggers {
-		// seriesTagIndex 的 key 仅做分组使用，value 为每组 series 的 hash
-		seriesTagIndex := ProcessJoins(ruleId, trigger, seriesTagIndexes, seriesStore)
-
-		for _, seriesHash := range seriesTagIndex {
-			valuesUnitMap := make(map[string]unit.FormattedValue)
-
-			sort.Slice(seriesHash, func(i, j int) bool {
-				return seriesHash[i] < seriesHash[j]
-			})
-
-			m := make(map[string]interface{})
-			var ts int64
-			var sample models.DataResp
-			var value float64
-			for _, serieHash := range seriesHash {
-				series, exists := seriesStore[serieHash]
-				if !exists {
-					logger.Warningf("rule_eval rid:%d series:%+v not found", ruleId, series)
-					continue
-				}
-				t, v, exists := series.Last()
-				if !exists {
-					logger.Warningf("rule_eval rid:%d series:%+v value not found", ruleId, series)
-					continue
-				}
-
-				if !strings.Contains(trigger.Exp, "$"+series.Ref) {
-					// 表达式中不包含该变量
-					continue
-				}
-
-				if u, exists := unitMap[series.Ref]; exists {
-					valuesUnitMap[series.Ref] = unit.ValueFormatter(u, 2, v)
-				}
-
-				m["$"+series.Ref] = v
-				m["$"+series.Ref+"."+series.MetricName()] = v
-				ts = int64(t)
-				sample = series
-				value = v
-			}
-			isTriggered := parser.Calc(trigger.Exp, m)
-			//  此条日志很重要，是告警判断的现场值
-			logger.Infof("rule_eval rid:%d trigger:%+v exp:%s res:%v m:%v", ruleId, trigger, trigger.Exp, isTriggered, m)
-
-			var values string
-			for k, v := range m {
-				if !strings.Contains(k, ".") {
-					continue
-				}
-				values += fmt.Sprintf("%s:%v ", k, v)
-			}
-
-			point := models.AnomalyPoint{
-				Key:           sample.MetricName(),
-				Labels:        sample.Metric,
-				Timestamp:     int64(ts),
-				Value:         value,
-				Values:        values,
-				Severity:      trigger.Severity,
-				Triggered:     isTriggered,
-				Query:         fmt.Sprintf("query:%+v trigger:%+v", ruleQuery.Queries, trigger),
-				RecoverConfig: trigger.RecoverConfig,
-				ValuesUnit:    valuesUnitMap,
-			}
-
-			if sample.Query != "" {
-				point.Query = sample.Query
-			}
-			// 恢复条件判断经过讨论是只在表达式模式下支持，表达式模式会通过 isTriggered 判断是告警点还是恢复点
-			// 1. 不设置恢复判断，满足恢复条件产生 recoverPoint 恢复，无数据不产生 anomalyPoint 恢复
-			// 2. 设置满足条件才恢复，仅可通过产生 recoverPoint 恢复，不能通过不产生 anomalyPoint 恢复
-			// 3. 设置无数据不恢复，仅可通过产生 recoverPoint 恢复，不产生 anomalyPoint 恢复
-			if isTriggered {
-				points = append(points, point)
-			} else {
-				switch trigger.RecoverConfig.JudgeType {
-				case models.Origin:
-					// 对齐原实现 do nothing
-				case models.RecoverOnCondition:
-					// 额外判断恢复条件，满足才恢复
-					fulfill := parser.Calc(trigger.RecoverConfig.RecoverExp, m)
-					if !fulfill {
-						continue
-					}
-				}
-				recoverPoints = append(recoverPoints, point)
-			}
-		}
-	}
-
-	return points, recoverPoints
-}
-
 func flatten(rehashed map[uint64][][]uint64) map[uint64][]uint64 {
 	seriesTagIndex := make(map[uint64][]uint64)
 	var i uint64
@@ -1198,3 +1105,429 @@ func GetQueryRefAndUnit(query interface{}) (string, string, error) {
 	json.Unmarshal(queryBytes, &queryMap)
 	return queryMap.Ref, queryMap.Unit, nil
 }
+
+// VarFillingBeforeQuery 填充变量，先填充变量再查询，针对有聚合函数的情况
+// 公式: avg(mem_used_percent{host="$host"}) > $val 其中 $host 为参数变量，$val 为值变量
+// 实现步骤:
+// 依次遍历参数配置节点，保证同一参数变量的子筛选可以覆盖上一层筛选
+// 每个节点先填充参数再进行查询, 即先得到完整的 promql avg(mem_used_percent{host="127.0.0.1"}) > 5
+// 再查询得到满足值变量的所有结果加入异常点列表
+// 参数变量的值不满足的组合，需要覆盖上层筛选中产生的异常点
+func (arw *AlertRuleWorker) VarFillingBeforeQuery(query models.PromQuery, readerClient promsdk.API) []models.AnomalyPoint {
+	varToLabel := ExtractVarMapping(query.PromQl)
+	// 存储异常点的 map，key 为参数变量的组合，可以实现子筛选对上一层筛选的覆盖
+	anomalyPointsMap := sync.Map{}
+	// 统一变量配置格式
+	VarConfigForCalc := &models.ChildVarConfig{
+		ParamVal:        make([]map[string]models.ParamQuery, 1),
+		ChildVarConfigs: query.VarConfig.ChildVarConfigs,
+	}
+	VarConfigForCalc.ParamVal[0] = make(map[string]models.ParamQuery)
+	for _, p := range query.VarConfig.ParamVal {
+		VarConfigForCalc.ParamVal[0][p.Name] = models.ParamQuery{
+			ParamType: p.ParamType,
+			Query:     p.Query,
+		}
+	}
+	// 使用一个统一的参数变量顺序
+	var ParamKeys []string
+	for val, valQuery := range VarConfigForCalc.ParamVal[0] {
+		if valQuery.ParamType == "threshold" {
+			continue
+		}
+		ParamKeys = append(ParamKeys, val)
+	}
+	sort.Slice(ParamKeys, func(i, j int) bool {
+		return ParamKeys[i] < ParamKeys[j]
+	})
+	// 遍历变量配置链表
+	curNode := VarConfigForCalc
+	for curNode != nil {
+		for _, param := range curNode.ParamVal {
+			curPromql := query.PromQl
+			// 取出阈值变量
+			valMap := make(map[string]string)
+			for val, valQuery := range param {
+				if valQuery.ParamType == "threshold" {
+					valMap[val] = getString(valQuery.Query)
+				}
+			}
+			// 替换阈值变量
+			for key, val := range valMap {
+				curPromql = strings.Replace(curPromql, fmt.Sprintf("$%s", key), val, -1)
+			}
+			// 得到参数变量的所有组合
+			paramPermutation, err := arw.getParamPermutation(param, ParamKeys, varToLabel, query.PromQl, readerClient)
+			if err != nil {
+				logger.Errorf("rule_eval:%s, paramPermutation error:%v", arw.Key(), err)
+				continue
+			}
+
+			keyToPromql := make(map[string]string)
+			for paramPermutationKeys, _ := range paramPermutation {
+				realPromql := curPromql
+				split := strings.Split(paramPermutationKeys, JoinMark)
+				for j := range ParamKeys {
+					realPromql = fillVar(realPromql, ParamKeys[j], split[j])
+				}
+				keyToPromql[paramPermutationKeys] = realPromql
+			}
+
+			// 并发查询
+			wg := sync.WaitGroup{}
+			semaphore := make(chan struct{}, 200)
+			for key, promql := range keyToPromql {
+				wg.Add(1)
+				semaphore <- struct{}{}
+				go func(key, promql string) {
+					defer func() {
+						<-semaphore
+						wg.Done()
+					}()
+					value, _, err := readerClient.Query(context.Background(), promql, time.Now())
+					if err != nil {
+						logger.Errorf("rule_eval:%s, promql:%s, error:%v", arw.Key(), promql, err)
+						return
+					}
+					logger.Infof("rule_eval:%s, promql:%s, value:%+v", arw.Key(), promql, value)
+
+					points := models.ConvertAnomalyPoints(value)
+					if len(points) == 0 {
+						anomalyPointsMap.Delete(key)
+						return
+					}
+					for i := 0; i < len(points); i++ {
+						points[i].Severity = query.Severity
+						points[i].Query = promql
+						points[i].ValuesUnit = map[string]unit.FormattedValue{
+							"v": unit.ValueFormatter(query.Unit, 2, points[i].Value),
+						}
+					}
+					anomalyPointsMap.Store(key, points)
+				}(key, promql)
+			}
+			wg.Wait()
+		}
+		curNode = curNode.ChildVarConfigs
+	}
+	anomalyPoints := make([]models.AnomalyPoint, 0)
+	anomalyPointsMap.Range(func(key, value any) bool {
+		if points, ok := value.([]models.AnomalyPoint); ok {
+			anomalyPoints = append(anomalyPoints, points...)
+		}
+		return true
+	})
+	return anomalyPoints
+}
+
+// 判断 query 中是否有会导致标签丢失的聚合函数
+func hasLabelLossAggregator(query models.PromQuery) bool {
+	noLabelAggregators := []string{
+		"sum", "min", "max", "avg",
+		"stddev", "stdvar",
+		"count", "quantile",
+		"group",
+	}
+	promql := strings.ToLower(query.PromQl)
+
+	for _, fn := range noLabelAggregators {
+		// 检查是否包含这些聚合函数，需要确保函数名后面跟着左括号
+		if strings.Contains(promql, fn+"(") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// 判断 query 中是否有 != =~ !~
+func notExactMatch(query models.PromQuery) bool {
+	promql := strings.ToLower(query.PromQl)
+	if strings.Contains(promql, "!=") || strings.Contains(promql, "=~") || strings.Contains(promql, "!~") {
+		return true
+	}
+	return false
+}
+
+// ExtractVarMapping 从 promql 中提取变量映射关系，为了在 query 之后可以将标签正确的放回 promql
+// 输入: sum(rate(mem_used_percent{host="$my_host"})) by (instance) + avg(node_load1{region="$region"}) > $val
+// 输出: map[string]string{"my_host":"host", "region":"region"}
+func ExtractVarMapping(promql string) map[string]string {
+	varMapping := make(map[string]string)
+
+	// 遍历所有花括号对
+	for {
+		start := strings.Index(promql, "{")
+		if start == -1 {
+			break
+		}
+
+		end := strings.Index(promql, "}")
+		if end == -1 {
+			break
+		}
+
+		// 提取标签键值对
+		labels := promql[start+1 : end]
+		pairs := strings.Split(labels, ",")
+
+		for _, pair := range pairs {
+			// 分割键值对
+			kv := strings.Split(pair, "=")
+			if len(kv) != 2 {
+				continue
+			}
+
+			key := strings.TrimSpace(kv[0])
+			value := strings.Trim(strings.TrimSpace(kv[1]), "\"")
+			value = strings.Trim(value, "'")
+
+			// 检查值是否为变量(以$开头)
+			if strings.HasPrefix(value, "$") {
+				varName := value[1:] // 去掉$前缀
+				varMapping[varName] = key
+			}
+		}
+
+		// 继续处理剩余部分
+		promql = promql[end+1:]
+	}
+
+	return varMapping
+}
+
+func fillVar(curRealQuery string, paramKey string, val string) string {
+	curRealQuery = strings.Replace(curRealQuery, fmt.Sprintf("'$%s'", paramKey), fmt.Sprintf("'%s'", val), -1)
+	curRealQuery = strings.Replace(curRealQuery, fmt.Sprintf("\"$%s\"", paramKey), fmt.Sprintf("\"%s\"", val), -1)
+	return curRealQuery
+}
+
+func (arw *AlertRuleWorker) GetAnomalyPoint(rule *models.AlertRule, dsId int64) ([]models.AnomalyPoint, []models.AnomalyPoint) {
+	// 获取查询和规则判断条件
+	points := []models.AnomalyPoint{}
+	recoverPoints := []models.AnomalyPoint{}
+	ruleConfig := strings.TrimSpace(rule.RuleConfig)
+	if ruleConfig == "" {
+		logger.Warningf("rule_eval:%d promql is blank", rule.Id)
+		arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_RULE_CONFIG, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
+		return points, recoverPoints
+	}
+
+	var ruleQuery models.RuleQuery
+	err := json.Unmarshal([]byte(ruleConfig), &ruleQuery)
+	if err != nil {
+		logger.Warningf("rule_eval:%d promql parse error:%s", rule.Id, err.Error())
+		arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_RULE_CONFIG, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
+		return points, recoverPoints
+	}
+
+	arw.Inhibit = ruleQuery.Inhibit
+	if len(ruleQuery.Queries) > 0 {
+		seriesStore := make(map[uint64]models.DataResp)
+		seriesTagIndexes := make(map[string]map[uint64][]uint64, 0)
+		for _, query := range ruleQuery.Queries {
+			seriesTagIndex := make(map[uint64][]uint64)
+
+			plug, exists := dscache.DsCache.Get(rule.Cate, dsId)
+			if !exists {
+				logger.Warningf("rule_eval rid:%d datasource:%d not exists", rule.Id, dsId)
+				arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_CLIENT, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
+				continue
+			}
+
+			series, err := plug.QueryData(context.Background(), query)
+			arw.Processor.Stats.CounterQueryDataTotal.WithLabelValues(fmt.Sprintf("%d", arw.DatasourceId)).Inc()
+			if err != nil {
+				logger.Warningf("rule_eval rid:%d query data error: %v", rule.Id, err)
+				arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), GET_CLIENT, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
+				continue
+			}
+
+			//  此条日志很重要，是告警判断的现场值
+			logger.Infof("rule_eval rid:%d req:%+v resp:%v", rule.Id, query, series)
+			for i := 0; i < len(series); i++ {
+				serieHash := hash.GetHash(series[i].Metric, series[i].Ref)
+				tagHash := hash.GetTagHash(series[i].Metric)
+				seriesStore[serieHash] = series[i]
+
+				// 将曲线按照相同的 tag 分组
+				if _, exists := seriesTagIndex[tagHash]; !exists {
+					seriesTagIndex[tagHash] = make([]uint64, 0)
+				}
+				seriesTagIndex[tagHash] = append(seriesTagIndex[tagHash], serieHash)
+			}
+			ref, err := GetQueryRef(query)
+			if err != nil {
+				logger.Warningf("rule_eval rid:%d query:%+v get ref error:%s", rule.Id, query, err.Error())
+				continue
+			}
+			seriesTagIndexes[ref] = seriesTagIndex
+		}
+
+		unitMap := make(map[string]string)
+		for _, query := range ruleQuery.Queries {
+			ref, unit, err := GetQueryRefAndUnit(query)
+			if err != nil {
+				continue
+			}
+			unitMap[ref] = unit
+		}
+
+		if !ruleQuery.ExpTriggerDisable {
+			for _, trigger := range ruleQuery.Triggers {
+				seriesTagIndex := ProcessJoins(rule.Id, trigger, seriesTagIndexes, seriesStore)
+				for _, seriesHash := range seriesTagIndex {
+					valuesUnitMap := make(map[string]unit.FormattedValue)
+
+					sort.Slice(seriesHash, func(i, j int) bool {
+						return seriesHash[i] < seriesHash[j]
+					})
+
+					m := make(map[string]interface{})
+					var ts int64
+					var sample models.DataResp
+					var value float64
+					for _, serieHash := range seriesHash {
+						series, exists := seriesStore[serieHash]
+						if !exists {
+							logger.Warningf("rule_eval rid:%d series:%+v not found", rule.Id, series)
+							continue
+						}
+						t, v, exists := series.Last()
+						if !exists {
+							logger.Warningf("rule_eval rid:%d series:%+v value not found", rule.Id, series)
+							continue
+						}
+
+						if !strings.Contains(trigger.Exp, "$"+series.Ref) {
+							// 表达式中不包含该变量
+							continue
+						}
+
+						m["$"+series.Ref] = v
+						m["$"+series.Ref+"."+series.MetricName()] = v
+						for k, v := range series.Metric {
+							if k == "__name__" {
+								continue
+							}
+
+							if !strings.Contains(trigger.Exp, "$"+series.Ref+"."+string(k)) {
+								// 过滤掉表达式中不包含的标签
+								continue
+							}
+
+							m["$"+series.Ref+"."+string(k)] = string(v)
+						}
+
+						if u, exists := unitMap[series.Ref]; exists {
+							valuesUnitMap["$"+series.Ref+"."+series.MetricName()] = unit.ValueFormatter(u, 2, v)
+						}
+
+						ts = int64(t)
+						sample = series
+						value = v
+						logger.Infof("rule_eval rid:%d origin series labels:%+v", rule.Id, series.Metric)
+					}
+
+					isTriggered := parser.CalcWithRid(trigger.Exp, m, rule.Id)
+					//  此条日志很重要，是告警判断的现场值
+					logger.Infof("rule_eval rid:%d trigger:%+v exp:%s res:%v m:%v", rule.Id, trigger, trigger.Exp, isTriggered, m)
+
+					var values string
+					for k, v := range m {
+						if !strings.Contains(k, ".") {
+							continue
+						}
+
+						switch v.(type) {
+						case float64:
+							values += fmt.Sprintf("%s:%.3f ", k, v)
+						case string:
+							values += fmt.Sprintf("%s:%s ", k, v)
+						}
+					}
+
+					point := models.AnomalyPoint{
+						Key:           sample.MetricName(),
+						Labels:        sample.Metric,
+						Timestamp:     int64(ts),
+						Value:         value,
+						Values:        values,
+						Severity:      trigger.Severity,
+						Triggered:     isTriggered,
+						Query:         fmt.Sprintf("query:%+v trigger:%+v", ruleQuery.Queries, trigger),
+						RecoverConfig: trigger.RecoverConfig,
+						ValuesUnit:    valuesUnitMap,
+					}
+
+					if isTriggered {
+						points = append(points, point)
+					} else {
+						switch trigger.RecoverConfig.JudgeType {
+						case models.Origin:
+							// do nothing
+						case models.RecoverOnCondition:
+							fulfill := parser.CalcWithRid(trigger.RecoverConfig.RecoverExp, m, rule.Id)
+							if !fulfill {
+								continue
+							}
+						}
+						recoverPoints = append(recoverPoints, point)
+					}
+				}
+			}
+		}
+
+		if ruleQuery.NodataTrigger.Enable {
+
+			now := time.Now().Unix()
+
+			// 使用 arw.LastSeriesStore 检查上次查询结果
+			if len(arw.LastSeriesStore) > 0 {
+				// 遍历上次的曲线数据
+				for hash, lastSeries := range arw.LastSeriesStore {
+
+					if ruleQuery.NodataTrigger.ResolveAfterEnable {
+						lastTs, _, exists := lastSeries.Last()
+						if !exists {
+							continue
+						}
+
+						// 检查是否超过 resolve_after 时间
+						if now-int64(lastTs) > int64(ruleQuery.NodataTrigger.ResolveAfter) {
+							logger.Infof("rule_eval rid:%d series:%+v resolve after %d seconds now:%d lastTs:%d", rule.Id, lastSeries, ruleQuery.NodataTrigger.ResolveAfter, now, int64(lastTs))
+							delete(arw.LastSeriesStore, hash)
+							continue
+						}
+					}
+
+					// 检查是否在本次查询结果中存在
+					if _, exists := seriesStore[hash]; !exists {
+						// 生成无数据告警点
+						point := models.AnomalyPoint{
+							Key:         lastSeries.MetricName(),
+							Labels:      lastSeries.Metric,
+							Timestamp:   now,
+							Value:       0,
+							Values:      fmt.Sprintf("nodata since %v", time.Unix(now, 0).Format("2006-01-02 15:04:05")),
+							Severity:    ruleQuery.NodataTrigger.Severity,
+							Triggered:   true,
+							Query:       fmt.Sprintf("nodata check for %s", lastSeries.LabelsString()),
+							TriggerType: models.TriggerTypeNodata,
+						}
+						points = append(points, point)
+						logger.Infof("rule_eval rid:%d nodata point:%+v", rule.Id, point)
+					}
+				}
+
+			}
+
+			// 更新 arw.LastSeriesStore
+			for hash, series := range seriesStore {
+				arw.LastSeriesStore[hash] = series
+			}
+		}
+	}
+
+	return points, recoverPoints
+}

alert/eval/eval_test.go

@@ -340,7 +340,7 @@ func Test_removeVal(t *testing.T) {
 		{
 			name: "removeVal7",
 			args: args{
-				promql: "mem{test1=\"test1\",test2=\"test2\",test3=\"$test3\"} > $val",
+				promql: "mem{test1=\"test1\",test2=\"test2\",test3='$test3'} > $val",
 			},
 			want: "mem{test1=\"test1\",test2=\"test2\"} > $val",
 		},
@@ -361,16 +361,16 @@ func Test_removeVal(t *testing.T) {
 		{
 			name: "removeVal10",
 			args: args{
-				promql: "mem{test1=\"test1\",test2=\"$test2\"} > $val1 and mem{test3=\"test3\",test4=\"test4\"} > $val2",
+				promql: "mem{test1=\"test1\",test2='$test2'} > $val1 and mem{test3=\"test3\",test4=\"test4\"} > $val2",
 			},
 			want: "mem{test1=\"test1\"} > $val1 and mem{test3=\"test3\",test4=\"test4\"} > $val2",
 		},
 		{
 			name: "removeVal11",
 			args: args{
-				promql: "mem{test1=\"test1\",test2=\"test2\"} > $val1 and mem{test3=\"$test3\",test4=\"test4\"} > $val2",
+				promql: "mem{test1='test1',test2=\"test2\"} > $val1 and mem{test3=\"$test3\",test4=\"test4\"} > $val2",
 			},
-			want: "mem{test1=\"test1\",test2=\"test2\"} > $val1 and mem{test4=\"test4\"} > $val2",
+			want: "mem{test1='test1',test2=\"test2\"} > $val1 and mem{test4=\"test4\"} > $val2",
 		},
 		{
 			name: "removeVal12",
@@ -388,3 +388,71 @@ func Test_removeVal(t *testing.T) {
 		})
 	}
 }
+
+func TestExtractVarMapping(t *testing.T) {
+	tests := []struct {
+		name   string
+		promql string
+		want   map[string]string
+	}{
+		{
+			name:   "单个花括号单个变量",
+			promql: `mem_used_percent{host="$my_host"} > $val`,
+			want:   map[string]string{"my_host": "host"},
+		},
+		{
+			name:   "单个花括号多个变量",
+			promql: `mem_used_percent{host="$my_host",region="$region",env="prod"} > $val`,
+			want:   map[string]string{"my_host": "host", "region": "region"},
+		},
+		{
+			name:   "多个花括号多个变量",
+			promql: `sum(rate(mem_used_percent{host="$my_host"})) by (instance) + avg(node_load1{region="$region"}) > $val`,
+			want:   map[string]string{"my_host": "host", "region": "region"},
+		},
+		{
+			name:   "相同变量出现多次",
+			promql: `sum(rate(mem_used_percent{host="$my_host"})) + avg(node_load1{host="$my_host"}) > $val`,
+			want:   map[string]string{"my_host": "host"},
+		},
+		{
+			name:   "没有变量",
+			promql: `mem_used_percent{host="localhost",region="cn"} > 80`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "没有花括号",
+			promql: `80 > $val`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "格式不规范的标签",
+			promql: `mem_used_percent{host=$my_host,region = $region} > $val`,
+			want:   map[string]string{"my_host": "host", "region": "region"},
+		},
+		{
+			name:   "空花括号",
+			promql: `mem_used_percent{} > $val`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "不完整的花括号",
+			promql: `mem_used_percent{host="$my_host"`,
+			want:   map[string]string{},
+		},
+		{
+			name:   "复杂表达式",
+			promql: `sum(rate(http_requests_total{handler="$handler",code="$code"}[5m])) by (handler) / sum(rate(http_requests_total{handler="$handler"}[5m])) by (handler) * 100 > $threshold`,
+			want:   map[string]string{"handler": "handler", "code": "code"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ExtractVarMapping(tt.promql)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ExtractVarMapping() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

alert/process/process.go

@@ -2,6 +2,7 @@ package process
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"sort"
@@ -80,8 +81,8 @@ type Processor struct {
 	HandleRecoverEventHook HandleEventFunc
 	EventMuteHook          EventMuteHookFunc
 
-	ScheduleEntry cron.Entry
-	EvalStart     int64
+	ScheduleEntry    cron.Entry
+	PromEvalInterval int
 }
 
 func (p *Processor) Key() string {
@@ -215,7 +216,6 @@ func (p *Processor) BuildEvent(anomalyPoint models.AnomalyPoint, from string, no
 	event.Callbacks = p.rule.Callbacks
 	event.CallbacksJSON = p.rule.CallbacksJSON
 	event.Annotations = p.rule.Annotations
-	event.AnnotationsJSON = make(map[string]string)
 	event.RuleConfig = p.rule.RuleConfig
 	event.RuleConfigJson = p.rule.RuleConfigJson
 	event.Severity = anomalyPoint.Severity
@@ -224,6 +224,20 @@ func (p *Processor) BuildEvent(anomalyPoint models.AnomalyPoint, from string, no
 	event.RecoverConfig = anomalyPoint.RecoverConfig
 	event.RuleHash = ruleHash
 
+	if anomalyPoint.TriggerType == models.TriggerTypeNodata {
+		event.TriggerValue = "nodata"
+		ruleConfig := models.RuleQuery{}
+		json.Unmarshal([]byte(p.rule.RuleConfig), &ruleConfig)
+		ruleConfig.TriggerType = anomalyPoint.TriggerType
+		b, _ := json.Marshal(ruleConfig)
+		event.RuleConfig = string(b)
+	}
+
+	if err := json.Unmarshal([]byte(p.rule.Annotations), &event.AnnotationsJSON); err != nil {
+		event.AnnotationsJSON = make(map[string]string) // 解析失败时使用空 map
+		logger.Warningf("unmarshal annotations json failed: %v, rule: %d", err, p.rule.Id)
+	}
+
 	if p.target != "" {
 		if pt, exist := p.TargetCache.Get(p.target); exist {
 			pt.GroupNames = p.BusiGroupCache.GetNamesByBusiGroupIds(pt.GroupIds)
@@ -424,6 +438,7 @@ func (p *Processor) handleEvent(events []*models.AlertCurEvent) {
 			p.pendingsUseByRecover.Set(event.Hash, event)
 		}
 
+		event.PromEvalInterval = p.PromEvalInterval
 		if p.rule.PromForDuration == 0 {
 			fireEvents = append(fireEvents, event)
 			if severity > event.Severity {
@@ -442,7 +457,6 @@ func (p *Processor) handleEvent(events []*models.AlertCurEvent) {
 			preTriggerTime = event.TriggerTime
 		}
 
-		event.PromEvalInterval = int(p.ScheduleEntry.Schedule.Next(time.Unix(p.EvalStart, 0)).Unix() - p.EvalStart)
 		if event.LastEvalTime-preTriggerTime+int64(event.PromEvalInterval) >= int64(p.rule.PromForDuration) {
 			fireEvents = append(fireEvents, event)
 			if severity > event.Severity {

alert/sender/webhook.go

@@ -59,17 +59,21 @@ func sendWebhook(webhook *models.Webhook, event interface{}, stats *astats.Stats
 	if webhook != nil {
 		insecureSkipVerify = webhook.SkipVerify
 	}
-	client := http.Client{
-		Timeout: time.Duration(conf.Timeout) * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify},
-		},
+
+	if conf.Client == nil {
+		logger.Warningf("event_%s, event:%s, url: [%s], error: [%s]", channel, string(bs), conf.Url, "client is nil")
+		conf.Client = &http.Client{
+			Timeout: time.Duration(conf.Timeout) * time.Second,
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify},
+			},
+		}
 	}
 
 	stats.AlertNotifyTotal.WithLabelValues(channel).Inc()
 	var resp *http.Response
 	var body []byte
-	resp, err = client.Do(req)
+	resp, err = conf.Client.Do(req)
 
 	if err != nil {
 		stats.AlertNotifyErrorTotal.WithLabelValues(channel).Inc()
@@ -91,7 +95,7 @@ func sendWebhook(webhook *models.Webhook, event interface{}, stats *astats.Stats
 	return false, string(body), nil
 }
 
-func SingleSendWebhooks(ctx *ctx.Context, webhooks []*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
+func SingleSendWebhooks(ctx *ctx.Context, webhooks map[string]*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
 	for _, conf := range webhooks {
 		retryCount := 0
 		for retryCount < 3 {
@@ -106,7 +110,7 @@ func SingleSendWebhooks(ctx *ctx.Context, webhooks []*models.Webhook, event *mod
 	}
 }
 
-func BatchSendWebhooks(ctx *ctx.Context, webhooks []*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
+func BatchSendWebhooks(ctx *ctx.Context, webhooks map[string]*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
 	for _, conf := range webhooks {
 		logger.Infof("push event:%+v to queue:%v", event, conf)
 		PushEvent(ctx, conf, event, stats)

center/center.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/ccfos/nightingale/v6/dscache"
+
 	"github.com/ccfos/nightingale/v6/alert"
 	"github.com/ccfos/nightingale/v6/alert/astats"
 	"github.com/ccfos/nightingale/v6/alert/dispatch"
@@ -27,14 +29,13 @@ import (
 	"github.com/ccfos/nightingale/v6/pkg/httpx"
 	"github.com/ccfos/nightingale/v6/pkg/i18nx"
 	"github.com/ccfos/nightingale/v6/pkg/logx"
+	"github.com/ccfos/nightingale/v6/pkg/macros"
 	"github.com/ccfos/nightingale/v6/pkg/version"
 	"github.com/ccfos/nightingale/v6/prom"
 	"github.com/ccfos/nightingale/v6/pushgw/idents"
 	pushgwrt "github.com/ccfos/nightingale/v6/pushgw/router"
 	"github.com/ccfos/nightingale/v6/pushgw/writer"
 	"github.com/ccfos/nightingale/v6/storage"
-	"github.com/ccfos/nightingale/v6/tdengine"
-
 	"github.com/flashcatcloud/ibex/src/cmd/ibex"
 )
 
@@ -107,10 +108,11 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 
 	dispatch.InitRegisterQueryFunc(promClients)
 
-	tdengineClients := tdengine.NewTdengineClient(ctx, config.Alert.Heartbeat)
-
 	externalProcessors := process.NewExternalProcessors()
-	alert.Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, taskTplCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache)
+
+	macros.RegisterMacro(macros.MacroInVain)
+	dscache.Init(ctx, false)
+	alert.Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, taskTplCache, dsCache, ctx, promClients, userCache, userGroupCache)
 
 	writers := writer.NewWriters(config.Pushgw)
 
@@ -120,7 +122,7 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 
 	alertrtRouter := alertrt.New(config.HTTP, config.Alert, alertMuteCache, targetCache, busiGroupCache, alertStats, ctx, externalProcessors)
 	centerRouter := centerrt.New(config.HTTP, config.Center, config.Alert, config.Ibex,
-		cconf.Operations, dsCache, notifyConfigCache, promClients, tdengineClients,
+		cconf.Operations, dsCache, notifyConfigCache, promClients,
 		redis, sso, ctx, metas, idents, targetCache, userCache, userGroupCache)
 	pushgwRouter := pushgwrt.New(config.HTTP, config.Pushgw, config.Alert, targetCache, busiGroupCache, idents, metas, writers, ctx)
 

center/router/router.go

@@ -24,7 +24,6 @@ import (
 	"github.com/ccfos/nightingale/v6/prom"
 	"github.com/ccfos/nightingale/v6/pushgw/idents"
 	"github.com/ccfos/nightingale/v6/storage"
-	"github.com/ccfos/nightingale/v6/tdengine"
 
 	"github.com/gin-gonic/gin"
 	"github.com/rakyll/statik/fs"
@@ -42,7 +41,6 @@ type Router struct {
 	DatasourceCache   *memsto.DatasourceCacheType
 	NotifyConfigCache *memsto.NotifyConfigCacheType
 	PromClients       *prom.PromClientMap
-	TdendgineClients  *tdengine.TdengineClientMap
 	Redis             storage.Redis
 	MetaSet           *metas.Set
 	IdentSet          *idents.Set
@@ -57,7 +55,7 @@ type Router struct {
 
 func New(httpConfig httpx.Config, center cconf.Center, alert aconf.Alert, ibex conf.Ibex,
 	operations cconf.Operation, ds *memsto.DatasourceCacheType, ncc *memsto.NotifyConfigCacheType,
-	pc *prom.PromClientMap, tdendgineClients *tdengine.TdengineClientMap, redis storage.Redis,
+	pc *prom.PromClientMap, redis storage.Redis,
 	sso *sso.SsoClient, ctx *ctx.Context, metaSet *metas.Set, idents *idents.Set,
 	tc *memsto.TargetCacheType, uc *memsto.UserCacheType, ugc *memsto.UserGroupCacheType) *Router {
 	return &Router{
@@ -69,7 +67,6 @@ func New(httpConfig httpx.Config, center cconf.Center, alert aconf.Alert, ibex c
 		DatasourceCache:   ds,
 		NotifyConfigCache: ncc,
 		PromClients:       pc,
-		TdendgineClients:  tdendgineClients,
 		Redis:             redis,
 		MetaSet:           metaSet,
 		IdentSet:          idents,
@@ -187,12 +184,24 @@ func (rt *Router) Config(r *gin.Engine) {
 			pages.POST("/datasource/query", rt.datasourceQuery)
 
 			pages.POST("/ds-query", rt.QueryData)
-			pages.POST("/logs-query", rt.QueryLog)
+			pages.POST("/logs-query", rt.QueryLogV2)
 
 			pages.POST("/tdengine-databases", rt.tdengineDatabases)
 			pages.POST("/tdengine-tables", rt.tdengineTables)
 			pages.POST("/tdengine-columns", rt.tdengineColumns)
 
+			pages.POST("/log-query-batch", rt.QueryLogBatch)
+
+			// 数据库元数据接口
+			pages.POST("/db-databases", rt.ShowDatabases)
+			pages.POST("/db-tables", rt.ShowTables)
+			pages.POST("/db-desc-table", rt.DescribeTable)
+
+			// es 专用接口
+			pages.POST("/indices", rt.auth(), rt.user(), rt.QueryIndices)
+			pages.POST("/es-variable", rt.auth(), rt.user(), rt.QueryESVariable)
+			pages.POST("/fields", rt.auth(), rt.user(), rt.QueryFields)
+			pages.POST("/log-query", rt.auth(), rt.user(), rt.QueryLog)
 		} else {
 			pages.Any("/proxy/:id/*url", rt.auth(), rt.dsProxy)
 			pages.POST("/query-range-batch", rt.auth(), rt.promBatchQueryRange)
@@ -201,11 +210,24 @@ func (rt *Router) Config(r *gin.Engine) {
 			pages.POST("/datasource/query", rt.auth(), rt.user(), rt.datasourceQuery)
 
 			pages.POST("/ds-query", rt.auth(), rt.QueryData)
-			pages.POST("/logs-query", rt.auth(), rt.QueryLog)
+			pages.POST("/logs-query", rt.auth(), rt.QueryLogV2)
 
 			pages.POST("/tdengine-databases", rt.auth(), rt.tdengineDatabases)
 			pages.POST("/tdengine-tables", rt.auth(), rt.tdengineTables)
 			pages.POST("/tdengine-columns", rt.auth(), rt.tdengineColumns)
+
+			pages.POST("/log-query-batch", rt.auth(), rt.user(), rt.QueryLogBatch)
+
+			// 数据库元数据接口
+			pages.POST("/db-databases", rt.auth(), rt.user(), rt.ShowDatabases)
+			pages.POST("/db-tables", rt.auth(), rt.user(), rt.ShowTables)
+			pages.POST("/db-desc-table", rt.auth(), rt.user(), rt.DescribeTable)
+
+			// es 专用接口
+			pages.POST("/indices", rt.auth(), rt.user(), rt.QueryIndices)
+			pages.POST("/es-variable", rt.QueryESVariable)
+			pages.POST("/fields", rt.QueryFields)
+			pages.POST("/log-query", rt.QueryLog)
 		}
 
 		pages.GET("/sql-template", rt.QuerySqlTemplate)
@@ -322,6 +344,11 @@ func (rt *Router) Config(r *gin.Engine) {
 		pages.GET("/share-charts", rt.chartShareGets)
 		pages.POST("/share-charts", rt.auth(), rt.chartShareAdd)
 
+		pages.POST("/dashboard-annotations", rt.auth(), rt.user(), rt.perm("/dashboards/put"), rt.dashAnnotationAdd)
+		pages.GET("/dashboard-annotations", rt.dashAnnotationGets)
+		pages.PUT("/dashboard-annotation/:id", rt.auth(), rt.user(), rt.perm("/dashboards/put"), rt.dashAnnotationPut)
+		pages.DELETE("/dashboard-annotation/:id", rt.auth(), rt.user(), rt.perm("/dashboards/del"), rt.dashAnnotationDel)
+
 		// pages.GET("/alert-rules/builtin/alerts-cates", rt.auth(), rt.user(), rt.builtinAlertCateGets)
 		// pages.GET("/alert-rules/builtin/list", rt.auth(), rt.user(), rt.builtinAlertRules)
 		pages.GET("/alert-rules/callbacks", rt.auth(), rt.user(), rt.alertRuleCallbacks)
@@ -479,6 +506,7 @@ func (rt *Router) Config(r *gin.Engine) {
 		pages.PUT("/builtin-payloads", rt.auth(), rt.user(), rt.perm("/built-in-components/put"), rt.builtinPayloadsPut)
 		pages.DELETE("/builtin-payloads", rt.auth(), rt.user(), rt.perm("/built-in-components/del"), rt.builtinPayloadsDel)
 		pages.GET("/builtin-payload", rt.auth(), rt.user(), rt.builtinPayloadsGetByUUIDOrID)
+
 	}
 
 	r.GET("/api/n9e/versions", func(c *gin.Context) {

center/router/router_builtin_componet.go

@@ -36,8 +36,9 @@ func (rt *Router) builtinComponentsAdd(c *gin.Context) {
 
 func (rt *Router) builtinComponentsGets(c *gin.Context) {
 	query := ginx.QueryStr(c, "query", "")
+	disabled := ginx.QueryInt(c, "disabled", -1)
 
-	bc, err := models.BuiltinComponentGets(rt.Ctx, query)
+	bc, err := models.BuiltinComponentGets(rt.Ctx, query, disabled)
 	ginx.Dangerous(err)
 
 	ginx.NewRender(c).Data(bc, nil)

center/router/router_dash_annotation.go

@@ -0,0 +1,99 @@
+package router
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+)
+
+func checkAnnotationPermission(c *gin.Context, ctx *ctx.Context, dashboardId int64) {
+	dashboard, err := models.BoardGetByID(ctx, dashboardId)
+	if err != nil {
+		ginx.Bomb(http.StatusInternalServerError, "failed to get dashboard: %v", err)
+	}
+
+	if dashboard == nil {
+		ginx.Bomb(http.StatusNotFound, "dashboard not found")
+	}
+
+	bg := BusiGroup(ctx, dashboard.GroupId)
+	me := c.MustGet("user").(*models.User)
+	can, err := me.CanDoBusiGroup(ctx, bg, "rw")
+	ginx.Dangerous(err)
+
+	if !can {
+		ginx.Bomb(http.StatusForbidden, "forbidden")
+	}
+}
+
+func (rt *Router) dashAnnotationAdd(c *gin.Context) {
+	var f models.DashAnnotation
+	ginx.BindJSON(c, &f)
+
+	username := c.MustGet("username").(string)
+	now := time.Now().Unix()
+
+	checkAnnotationPermission(c, rt.Ctx, f.DashboardId)
+
+	f.CreateBy = username
+	f.CreateAt = now
+	f.UpdateBy = username
+	f.UpdateAt = now
+
+	ginx.NewRender(c).Data(f.Id, f.Add(rt.Ctx))
+}
+
+func (rt *Router) dashAnnotationGets(c *gin.Context) {
+	dashboardId := ginx.QueryInt64(c, "dashboard_id")
+	from := ginx.QueryInt64(c, "from")
+	to := ginx.QueryInt64(c, "to")
+	limit := ginx.QueryInt(c, "limit", 100)
+
+	lst, err := models.DashAnnotationGets(rt.Ctx, dashboardId, from, to, limit)
+	ginx.NewRender(c).Data(lst, err)
+}
+
+func (rt *Router) dashAnnotationPut(c *gin.Context) {
+	var f models.DashAnnotation
+	ginx.BindJSON(c, &f)
+
+	id := ginx.UrlParamInt64(c, "id")
+	annotation, err := getAnnotationById(rt.Ctx, id)
+	ginx.Dangerous(err)
+
+	checkAnnotationPermission(c, rt.Ctx, annotation.DashboardId)
+
+	f.Id = id
+	f.UpdateAt = time.Now().Unix()
+	f.UpdateBy = c.MustGet("username").(string)
+
+	ginx.NewRender(c).Message(f.Update(rt.Ctx))
+}
+
+func (rt *Router) dashAnnotationDel(c *gin.Context) {
+	id := ginx.UrlParamInt64(c, "id")
+
+	annotation, err := getAnnotationById(rt.Ctx, id)
+	ginx.Dangerous(err)
+	checkAnnotationPermission(c, rt.Ctx, annotation.DashboardId)
+
+	ginx.NewRender(c).Message(models.DashAnnotationDel(rt.Ctx, id))
+}
+
+// 可以提取获取注释的通用方法
+func getAnnotationById(ctx *ctx.Context, id int64) (*models.DashAnnotation, error) {
+	annotation, err := models.DashAnnotationGet(ctx, "id=?", id)
+	if err != nil {
+		return nil, err
+	}
+	if annotation == nil {
+		return nil, fmt.Errorf("annotation not found")
+	}
+	return annotation, nil
+}

center/router/router_datasource.go

@@ -93,10 +93,12 @@ func (rt *Router) datasourceUpsert(c *gin.Context) {
 	var count int64
 
 	if !req.ForceSave {
-		err = DatasourceCheck(req)
-		if err != nil {
-			Dangerous(c, err)
-			return
+		if req.PluginType == models.PROMETHEUS || req.PluginType == models.LOKI || req.PluginType == models.TDENGINE {
+			err = DatasourceCheck(req)
+			if err != nil {
+				Dangerous(c, err)
+				return
+			}
 		}
 	}
 
@@ -122,7 +124,7 @@ func (rt *Router) datasourceUpsert(c *gin.Context) {
 }
 
 func DatasourceCheck(ds models.Datasource) error {
-	if ds.PluginType != models.ELASTICSEARCH {
+	if ds.PluginType == models.PROMETHEUS || ds.PluginType == models.LOKI || ds.PluginType == models.TDENGINE {
 		if ds.HTTPJson.Url == "" {
 			return fmt.Errorf("url is empty")
 		}

center/router/router_datasource_db.go

@@ -0,0 +1,101 @@
+package router
+
+import (
+	"context"
+
+	"github.com/ccfos/nightingale/v6/dscache"
+	"github.com/ccfos/nightingale/v6/dskit/types"
+	"github.com/ccfos/nightingale/v6/models"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+	"github.com/toolkits/pkg/logger"
+)
+
+func (rt *Router) ShowDatabases(c *gin.Context) {
+	var f models.QueryParam
+	ginx.BindJSON(c, &f)
+
+	plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if !exists {
+		logger.Warningf("cluster:%d not exists", f.DatasourceId)
+		ginx.Bomb(200, "cluster not exists")
+	}
+
+	var databases []string
+	var err error
+	type DatabaseShower interface {
+		ShowDatabases(context.Context) ([]string, error)
+	}
+	switch plug.(type) {
+	case DatabaseShower:
+		databases, err = plug.(DatabaseShower).ShowDatabases(c.Request.Context())
+		ginx.Dangerous(err)
+	default:
+		ginx.Bomb(200, "datasource not exists")
+	}
+
+	if len(databases) == 0 {
+		databases = make([]string, 0)
+	}
+
+	ginx.NewRender(c).Data(databases, nil)
+}
+
+func (rt *Router) ShowTables(c *gin.Context) {
+	var f models.QueryParam
+	ginx.BindJSON(c, &f)
+
+	plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if !exists {
+		logger.Warningf("cluster:%d not exists", f.DatasourceId)
+		ginx.Bomb(200, "cluster not exists")
+	}
+
+	// 只接受一个入参
+	tables := make([]string, 0)
+	var err error
+	type TableShower interface {
+		ShowTables(ctx context.Context, database string) ([]string, error)
+	}
+	switch plug.(type) {
+	case TableShower:
+		if len(f.Querys) > 0 {
+			database, ok := f.Querys[0].(string)
+			if ok {
+				tables, err = plug.(TableShower).ShowTables(c.Request.Context(), database)
+			}
+		}
+	default:
+		ginx.Bomb(200, "datasource not exists")
+	}
+	ginx.NewRender(c).Data(tables, err)
+}
+
+func (rt *Router) DescribeTable(c *gin.Context) {
+	var f models.QueryParam
+	ginx.BindJSON(c, &f)
+
+	plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if !exists {
+		logger.Warningf("cluster:%d not exists", f.DatasourceId)
+		ginx.Bomb(200, "cluster not exists")
+	}
+	// 只接受一个入参
+	columns := make([]*types.ColumnProperty, 0)
+	var err error
+	type TableDescriber interface {
+		DescribeTable(context.Context, interface{}) ([]*types.ColumnProperty, error)
+	}
+	switch plug.(type) {
+	case TableDescriber:
+		client := plug.(TableDescriber)
+		if len(f.Querys) > 0 {
+			columns, err = client.DescribeTable(c.Request.Context(), f.Querys[0])
+		}
+	default:
+		ginx.Bomb(200, "datasource not exists")
+	}
+
+	ginx.NewRender(c).Data(columns, err)
+}

center/router/router_es.go

@@ -0,0 +1,77 @@
+package router
+
+import (
+	"github.com/ccfos/nightingale/v6/datasource/es"
+	"github.com/ccfos/nightingale/v6/dscache"
+
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+	"github.com/toolkits/pkg/logger"
+)
+
+type IndexReq struct {
+	Cate         string `json:"cate"`
+	DatasourceId int64  `json:"datasource_id"`
+	Index        string `json:"index"`
+}
+
+type FieldValueReq struct {
+	Cate         string   `json:"cate"`
+	DatasourceId int64    `json:"datasource_id"`
+	Index        string   `json:"index"`
+	Query        FieldObj `json:"query"`
+}
+
+type FieldObj struct {
+	Find  string `json:"find"`
+	Field string `json:"field"`
+	Query string `json:"query"`
+}
+
+func (rt *Router) QueryIndices(c *gin.Context) {
+	var f IndexReq
+	ginx.BindJSON(c, &f)
+
+	plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if !exists {
+		logger.Warningf("cluster:%d not exists", f.DatasourceId)
+		ginx.Bomb(200, "cluster not exists")
+	}
+
+	indices, err := plug.(*es.Elasticsearch).QueryIndices()
+	ginx.Dangerous(err)
+
+	ginx.NewRender(c).Data(indices, nil)
+}
+
+func (rt *Router) QueryFields(c *gin.Context) {
+	var f IndexReq
+	ginx.BindJSON(c, &f)
+
+	plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if !exists {
+		logger.Warningf("cluster:%d not exists", f.DatasourceId)
+		ginx.Bomb(200, "cluster not exists")
+	}
+
+	fields, err := plug.(*es.Elasticsearch).QueryFields([]string{f.Index})
+	ginx.Dangerous(err)
+
+	ginx.NewRender(c).Data(fields, nil)
+}
+
+func (rt *Router) QueryESVariable(c *gin.Context) {
+	var f FieldValueReq
+	ginx.BindJSON(c, &f)
+
+	plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if !exists {
+		logger.Warningf("cluster:%d not exists", f.DatasourceId)
+		ginx.Bomb(200, "cluster not exists")
+	}
+
+	fields, err := plug.(*es.Elasticsearch).QueryFieldValue([]string{f.Index}, f.Query.Field, f.Query.Query)
+	ginx.Dangerous(err)
+
+	ginx.NewRender(c).Data(fields, nil)
+}

center/router/router_mw.go

@@ -130,9 +130,9 @@ func (rt *Router) User() gin.HandlerFunc {
 
 func (rt *Router) user() gin.HandlerFunc {
 	return func(c *gin.Context) {
-		userid := c.MustGet("userid").(int64)
+		username := c.MustGet("username").(string)
 
-		user, err := models.UserGetById(rt.Ctx, userid)
+		user, err := models.UserGetByUsername(rt.Ctx, username)
 		if err != nil {
 			ginx.Bomb(http.StatusUnauthorized, "unauthorized")
 		}

center/router/router_query.go

@@ -0,0 +1,181 @@
+package router
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/ccfos/nightingale/v6/dscache"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+	"github.com/toolkits/pkg/logger"
+)
+
+func CheckDsPerm(c *gin.Context, dsId int64, cate string, q interface{}) bool {
+	// todo: 后续需要根据 cate 判断是否需要权限
+	return true
+}
+
+type QueryFrom struct {
+	Queries []Query `json:"queries"`
+	Exps    []Exp   `json:"exps"`
+}
+
+type Query struct {
+	Ref    string      `json:"ref"`
+	Did    int64       `json:"ds_id"`
+	DsCate string      `json:"ds_cate"`
+	Query  interface{} `json:"query"`
+}
+
+type Exp struct {
+	Exp string `json:"exp"`
+	Ref string `json:"ref"`
+}
+
+type LogResp struct {
+	Total int64         `json:"total"`
+	List  []interface{} `json:"list"`
+}
+
+func (rt *Router) QueryLogBatch(c *gin.Context) {
+	var f QueryFrom
+	ginx.BindJSON(c, &f)
+
+	var resp LogResp
+	var errMsg string
+	for _, q := range f.Queries {
+		if !rt.Center.AnonymousAccess.PromQuerier && !CheckDsPerm(c, q.Did, q.DsCate, q) {
+			ginx.Bomb(200, "no permission")
+		}
+
+		plug, exists := dscache.DsCache.Get(q.DsCate, q.Did)
+		if !exists {
+			logger.Warningf("cluster:%d not exists query:%+v", q.Did, q)
+			ginx.Bomb(200, "cluster not exists")
+		}
+
+		data, total, err := plug.QueryLog(c.Request.Context(), q.Query)
+		if err != nil {
+			errMsg += fmt.Sprintf("query data error: %v query:%v\n ", err, q)
+			logger.Warningf("query data error: %v query:%v", err, q)
+			continue
+		}
+
+		m := make(map[string]interface{})
+		m["ref"] = q.Ref
+		m["ds_id"] = q.Did
+		m["ds_cate"] = q.DsCate
+		m["data"] = data
+		resp.List = append(resp.List, m)
+		resp.Total += total
+	}
+
+	if errMsg != "" || len(resp.List) == 0 {
+		ginx.Bomb(200, errMsg)
+	}
+
+	ginx.NewRender(c).Data(resp, nil)
+}
+
+func (rt *Router) QueryData(c *gin.Context) {
+	var f models.QueryParam
+	ginx.BindJSON(c, &f)
+
+	var resp []models.DataResp
+	var err error
+	for _, q := range f.Querys {
+		if !rt.Center.AnonymousAccess.PromQuerier && !CheckDsPerm(c, f.DatasourceId, f.Cate, q) {
+			ginx.Bomb(403, "no permission")
+		}
+
+		plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+		if !exists {
+			logger.Warningf("cluster:%d not exists", f.DatasourceId)
+			ginx.Bomb(200, "cluster not exists")
+		}
+		var datas []models.DataResp
+		datas, err = plug.QueryData(c.Request.Context(), q)
+		if err != nil {
+			logger.Warningf("query data error: req:%+v err:%v", q, err)
+			ginx.Bomb(200, "err:%v", err)
+		}
+		logger.Debugf("query data: req:%+v resp:%+v", q, datas)
+		resp = append(resp, datas...)
+	}
+	// 面向API的统一处理
+	// 按照 .Metric 排序
+	// 确保仪表盘中相同图例的曲线颜色相同
+	if len(resp) > 1 {
+		sort.Slice(resp, func(i, j int) bool {
+			if resp[i].Metric != nil && resp[j].Metric != nil {
+				return resp[i].Metric.String() < resp[j].Metric.String()
+			}
+			return false
+		})
+	}
+
+	ginx.NewRender(c).Data(resp, err)
+}
+
+func (rt *Router) QueryLogV2(c *gin.Context) {
+	var f models.QueryParam
+	ginx.BindJSON(c, &f)
+
+	var resp LogResp
+	var errMsg string
+	for _, q := range f.Querys {
+		if !rt.Center.AnonymousAccess.PromQuerier && !CheckDsPerm(c, f.DatasourceId, f.Cate, q) {
+			ginx.Bomb(200, "no permission")
+		}
+
+		plug, exists := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+		if !exists {
+			logger.Warningf("cluster:%d not exists query:%+v", f.DatasourceId, f)
+			ginx.Bomb(200, "cluster not exists")
+		}
+
+		data, total, err := plug.QueryLog(c.Request.Context(), q)
+		if err != nil {
+			errMsg += fmt.Sprintf("query data error: %v query:%v\n ", err, q)
+			logger.Warningf("query data error: %v query:%v", err, q)
+			continue
+		}
+		resp.List = append(resp.List, data...)
+		resp.Total += total
+	}
+
+	if errMsg != "" || len(resp.List) == 0 {
+		ginx.Bomb(200, errMsg)
+	}
+
+	ginx.NewRender(c).Data(resp, nil)
+}
+
+func (rt *Router) QueryLog(c *gin.Context) {
+	var f models.QueryParam
+	ginx.BindJSON(c, &f)
+
+	var resp []interface{}
+	for _, q := range f.Querys {
+		if !rt.Center.AnonymousAccess.PromQuerier && !CheckDsPerm(c, f.DatasourceId, f.Cate, q) {
+			ginx.Bomb(200, "no permission")
+		}
+
+		plug, exists := dscache.DsCache.Get("elasticsearch", f.DatasourceId)
+		if !exists {
+			logger.Warningf("cluster:%d not exists", f.DatasourceId)
+			ginx.Bomb(200, "cluster not exists")
+		}
+
+		data, _, err := plug.QueryLog(c.Request.Context(), q)
+		if err != nil {
+			logger.Warningf("query data error: %v", err)
+			ginx.Bomb(200, "err:%v", err)
+			continue
+		}
+		resp = append(resp, data...)
+	}
+
+	ginx.NewRender(c).Data(resp, nil)
+}

center/router/router_target.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/storage"
 
 	"github.com/gin-gonic/gin"
@@ -272,11 +273,9 @@ func (rt *Router) validateTags(tags []string) error {
 }
 
 func (rt *Router) addTagsToTarget(target *models.Target, tags []string) error {
-	hostTagsMap := target.GetHostTagsMap()
 	for _, tag := range tags {
 		tagKey := strings.Split(tag, "=")[0]
-		if _, ok := hostTagsMap[tagKey]; ok ||
-			strings.Contains(target.Tags, tagKey+"=") {
+		if _, exist := target.TagsMap[tagKey]; exist {
 			return fmt.Errorf("duplicate tagkey(%s)", tagKey)
 		}
 	}
@@ -401,6 +400,22 @@ type targetBgidsForm struct {
 	Action  string   `json:"action"` // add del reset
 }
 
+func haveNeverGroupedIdent(ctx *ctx.Context, idents []string) (bool, error) {
+	for _, ident := range idents {
+		bgids, err := models.TargetGroupIdsGetByIdent(ctx, ident)
+		if err != nil {
+			return false, err
+		}
+		
+		if len(bgids) <= 0 {
+			return true, nil
+		}	
+	}
+
+	return false, nil
+}
+
+
 func (rt *Router) targetBindBgids(c *gin.Context) {
 	var f targetBgidsForm
 	var err error
@@ -443,11 +458,15 @@ func (rt *Router) targetBindBgids(c *gin.Context) {
 				ginx.Bomb(http.StatusForbidden, "No permission. You are not admin of BG(%s)", bg.Name)
 			}
 		}
+		isNeverGrouped, checkErr := haveNeverGroupedIdent(rt.Ctx, f.Idents)
+		ginx.Dangerous(checkErr)
 
-		can, err := user.CheckPerm(rt.Ctx, "/targets/bind")
-		ginx.Dangerous(err)
-		if !can {
-			ginx.Bomb(http.StatusForbidden, "No permission. Only admin can assign BG")
+		if isNeverGrouped {
+			can, err := user.CheckPerm(rt.Ctx, "/targets/bind")
+			ginx.Dangerous(err)
+			if !can {
+				ginx.Bomb(http.StatusForbidden, "No permission. Only admin can assign BG")
+			}
 		}
 	}
 

center/router/router_tdengine.go

@@ -1,14 +1,14 @@
 package router
 
 import (
-	"net/http"
-
+	"fmt"
 	"github.com/ccfos/nightingale/v6/center/cconf"
+	"github.com/ccfos/nightingale/v6/datasource/tdengine"
+	"github.com/ccfos/nightingale/v6/dscache"
 	"github.com/ccfos/nightingale/v6/models"
-
 	"github.com/gin-gonic/gin"
 	"github.com/toolkits/pkg/ginx"
-	"github.com/toolkits/pkg/logger"
+	"net/http"
 )
 
 type databasesQueryForm struct {
@@ -20,13 +20,13 @@ func (rt *Router) tdengineDatabases(c *gin.Context) {
 	var f databasesQueryForm
 	ginx.BindJSON(c, &f)
 
-	tdClient := rt.TdendgineClients.GetCli(f.DatasourceId)
-	if tdClient == nil {
+	datasource, hit := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if _, ok := datasource.(*tdengine.TDengine); !hit || !ok {
 		ginx.NewRender(c, http.StatusNotFound).Message("No such datasource")
 		return
 	}
 
-	databases, err := tdClient.GetDatabases()
+	databases, err := datasource.(*tdengine.TDengine).ShowDatabases(rt.Ctx.Ctx)
 	ginx.NewRender(c).Data(databases, err)
 }
 
@@ -37,18 +37,29 @@ type tablesQueryForm struct {
 	IsStable     bool   `json:"is_stable"`
 }
 
+type Column struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+	Size int    `json:"size"`
+}
+
 // get tdengine tables
 func (rt *Router) tdengineTables(c *gin.Context) {
 	var f tablesQueryForm
 	ginx.BindJSON(c, &f)
 
-	tdClient := rt.TdendgineClients.GetCli(f.DatasourceId)
-	if tdClient == nil {
+	datasource, hit := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if _, ok := datasource.(*tdengine.TDengine); !hit || !ok {
 		ginx.NewRender(c, http.StatusNotFound).Message("No such datasource")
 		return
 	}
 
-	tables, err := tdClient.GetTables(f.Database, f.IsStable)
+	database := fmt.Sprintf("%s.tables", f.Database)
+	if f.IsStable {
+		database = fmt.Sprintf("%s.stables", f.Database)
+	}
+
+	tables, err := datasource.(*tdengine.TDengine).ShowTables(rt.Ctx.Ctx, database)
 	ginx.NewRender(c).Data(tables, err)
 }
 
@@ -59,50 +70,31 @@ type columnsQueryForm struct {
 	Table        string `json:"table"`
 }
 
-// get tdengine columns
 func (rt *Router) tdengineColumns(c *gin.Context) {
 	var f columnsQueryForm
 	ginx.BindJSON(c, &f)
 
-	tdClient := rt.TdendgineClients.GetCli(f.DatasourceId)
-	if tdClient == nil {
+	datasource, hit := dscache.DsCache.Get(f.Cate, f.DatasourceId)
+	if _, ok := datasource.(*tdengine.TDengine); !hit || !ok {
 		ginx.NewRender(c, http.StatusNotFound).Message("No such datasource")
 		return
 	}
 
-	columns, err := tdClient.GetColumns(f.Database, f.Table)
-	ginx.NewRender(c).Data(columns, err)
-}
-
-func (rt *Router) QueryData(c *gin.Context) {
-	var f models.QueryParam
-	ginx.BindJSON(c, &f)
-
-	var resp []models.DataResp
-	var err error
-	tdClient := rt.TdendgineClients.GetCli(f.DatasourceId)
-	for _, q := range f.Querys {
-		datas, err := tdClient.Query(q)
-		ginx.Dangerous(err)
-		resp = append(resp, datas...)
+	query := map[string]string{
+		"database": f.Database,
+		"table":    f.Table,
 	}
 
-	ginx.NewRender(c).Data(resp, err)
-}
-
-func (rt *Router) QueryLog(c *gin.Context) {
-	var f models.QueryParam
-	ginx.BindJSON(c, &f)
-
-	tdClient := rt.TdendgineClients.GetCli(f.DatasourceId)
-	if len(f.Querys) == 0 {
-		ginx.Bomb(200, "querys is empty")
-		return
+	columns, err := datasource.(*tdengine.TDengine).DescribeTable(rt.Ctx.Ctx, query)
+	// 对齐前端，后续可以将 tdEngine 的查数据的接口都统一
+	tdColumns := make([]Column, len(columns))
+	for i, column := range columns {
+		tdColumns[i] = Column{
+			Name: column.Field,
+			Type: column.Type,
+		}
 	}
-
-	data, err := tdClient.QueryLog(f.Querys[0])
-	logger.Debugf("tdengine query:%s result: %+v", f.Querys[0], data)
-	ginx.NewRender(c).Data(data, err)
+	ginx.NewRender(c).Data(tdColumns, err)
 }
 
 // query sql template

cmd/edge/edge.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ccfos/nightingale/v6/dscache"
+
 	"github.com/ccfos/nightingale/v6/alert"
 	"github.com/ccfos/nightingale/v6/alert/astats"
 	"github.com/ccfos/nightingale/v6/alert/dispatch"
@@ -17,13 +19,12 @@ import (
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/httpx"
 	"github.com/ccfos/nightingale/v6/pkg/logx"
+	"github.com/ccfos/nightingale/v6/pkg/macros"
 	"github.com/ccfos/nightingale/v6/prom"
 	"github.com/ccfos/nightingale/v6/pushgw/idents"
 	pushgwrt "github.com/ccfos/nightingale/v6/pushgw/router"
 	"github.com/ccfos/nightingale/v6/pushgw/writer"
 	"github.com/ccfos/nightingale/v6/storage"
-	"github.com/ccfos/nightingale/v6/tdengine"
-
 	"github.com/flashcatcloud/ibex/src/cmd/ibex"
 )
 
@@ -61,6 +62,8 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 	r := httpx.GinEngine(config.Global.RunMode, config.HTTP, configCvalCache.PrintBodyPaths, configCvalCache.PrintAccessLog)
 
 	pushgwRouter.Config(r)
+	macros.RegisterMacro(macros.MacroInVain)
+	dscache.Init(ctx, false)
 
 	if !config.Alert.Disable {
 		configCache := memsto.NewConfigCache(ctx, syncStats, nil, "")
@@ -77,11 +80,10 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 
 		dispatch.InitRegisterQueryFunc(promClients)
 
-		tdengineClients := tdengine.NewTdengineClient(ctx, config.Alert.Heartbeat)
 		externalProcessors := process.NewExternalProcessors()
 
 		alert.Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache,
-			alertRuleCache, notifyConfigCache, taskTplsCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache)
+			alertRuleCache, notifyConfigCache, taskTplsCache, dsCache, ctx, promClients, userCache, userGroupCache)
 
 		alertrtRouter := alertrt.New(config.HTTP, config.Alert, alertMuteCache, targetCache, busiGroupCache, alertStats, ctx, externalProcessors)
 

datasource/ck/clickhouse.go

@@ -0,0 +1,241 @@
+package ck
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/ccfos/nightingale/v6/datasource"
+	ck "github.com/ccfos/nightingale/v6/dskit/clickhouse"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/macros"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/toolkits/pkg/logger"
+)
+
+const (
+	CKType = "ck"
+
+	TimeFieldFormatEpochMilli  = "epoch_millis"
+	TimeFieldFormatEpochSecond = "epoch_second"
+
+	DefaultLimit = 500
+)
+
+var (
+	ckPrivBanned = []string{
+		"INSERT",
+		"CREATE",
+		"DROP",
+		"DELETE",
+		"UPDATE",
+		"ALL",
+	}
+
+	ckBannedOp = map[string]struct{}{
+		"CREATE":   {},
+		"INSERT":   {},
+		"ALTER":    {},
+		"REVOKE":   {},
+		"DROP":     {},
+		"RENAME":   {},
+		"ATTACH":   {},
+		"DETACH":   {},
+		"OPTIMIZE": {},
+		"TRUNCATE": {},
+		"SET":      {},
+	}
+)
+
+func init() {
+	datasource.RegisterDatasource(CKType, new(Clickhouse))
+}
+
+type CKShard struct {
+	Addr        string `json:"ck.addr" mapstructure:"ck.addr"`
+	User        string `json:"ck.user" mapstructure:"ck.user"`
+	Password    string `json:"ck.password" mapstructure:"ck.password"`
+	Database    string `json:"ck.db" mapstructure:"ck.db"`
+	IsEncrypted bool   `json:"ck.is_encrypt" mapstructure:"ck.is_encrypt"`
+}
+
+type QueryParam struct {
+	Limit      int             `json:"limit" mapstructure:"limit"`
+	Sql        string          `json:"sql" mapstructure:"sql"`
+	Ref        string          `json:"ref" mapstructure:"ref"`
+	From       int64           `json:"from" mapstructure:"from"`
+	To         int64           `json:"to" mapstructure:"to"`
+	TimeField  string          `json:"time_field" mapstructure:"time_field"`
+	TimeFormat string          `json:"time_format" mapstructure:"time_format"`
+	Keys       datasource.Keys `json:"keys" mapstructure:"keys"`
+	Database   string          `json:"database" mapstructure:"database"`
+	Table      string          `json:"table" mapstructure:"table"`
+}
+
+type Clickhouse struct {
+	ck.Clickhouse `json:",inline" mapstructure:",squash"`
+}
+
+func (c *Clickhouse) Init(settings map[string]interface{}) (datasource.Datasource, error) {
+	newest := new(Clickhouse)
+	err := mapstructure.Decode(settings, newest)
+	return newest, err
+}
+
+func (c *Clickhouse) InitClient() error {
+	return c.InitCli()
+}
+
+func (c *Clickhouse) Validate(ctx context.Context) error {
+	if len(c.Nodes) == 0 {
+		return fmt.Errorf("ck shard is invalid, please check datasource setting")
+	}
+
+	addr := c.Nodes[0]
+	if len(strings.Trim(c.User, " ")) == 0 {
+		return fmt.Errorf("ck shard user is invalid, please check datasource setting")
+	}
+
+	if len(strings.Trim(addr, " ")) == 0 {
+		return fmt.Errorf("ck shard addr is invalid, please check datasource setting")
+	}
+
+	// if len(strings.Trim(shard.Password, " ")) == 0 {
+	// 	return fmt.Errorf("ck shard password is empty, please check datasource setting or set password for user")
+	// }
+
+	return nil
+}
+
+// Equal compares whether two objects are the same, used for caching
+func (c *Clickhouse) Equal(p datasource.Datasource) bool {
+
+	plg, ok := p.(*Clickhouse)
+	if !ok {
+		logger.Errorf("unexpected plugin type, expected is ck")
+		return false
+	}
+
+	// only compare first shard
+	if len(c.Nodes) == 0 {
+		logger.Errorf("ck shard is empty")
+		return false
+	}
+	addr := c.Nodes[0]
+
+	if len(plg.Nodes) == 0 {
+		logger.Errorf("new ck plugin obj shard is empty")
+		return false
+	}
+	newAddr := plg.Nodes[0]
+
+	if c.User != plg.User {
+		return false
+	}
+
+	if addr != newAddr {
+		return false
+	}
+
+	if c.Password != plg.Password {
+		return false
+	}
+
+	return true
+}
+
+func (c *Clickhouse) MakeLogQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	return nil, nil
+}
+
+func (c *Clickhouse) MakeTSQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	return nil, nil
+}
+
+func (c *Clickhouse) QueryMapData(ctx context.Context, query interface{}) ([]map[string]string, error) {
+	return nil, nil
+}
+
+func (c *Clickhouse) QueryData(ctx context.Context, query interface{}) ([]models.DataResp, error) {
+
+	ckQueryParam := new(ck.QueryParam)
+	if err := mapstructure.Decode(query, ckQueryParam); err != nil {
+		return nil, err
+	}
+
+	if strings.Contains(ckQueryParam.Sql, "$__") {
+		var err error
+		ckQueryParam.Sql, err = macros.Macro(ckQueryParam.Sql, ckQueryParam.From, ckQueryParam.To)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ckQueryParam.Keys.ValueKey == "" {
+		return nil, fmt.Errorf("valueKey is required")
+	}
+
+	rows, err := c.QueryTimeseries(ctx, ckQueryParam)
+	if err != nil {
+		logger.Warningf("query:%+v get data err:%v", ckQueryParam, err)
+		return nil, err
+	}
+
+	if err != nil {
+		logger.Warningf("query:%+v get data err:%v", ckQueryParam, err)
+		return []models.DataResp{}, err
+	}
+	data := make([]models.DataResp, 0)
+	for i := range rows {
+		data = append(data, models.DataResp{
+			Ref:    ckQueryParam.Ref,
+			Metric: rows[i].Metric,
+			Values: rows[i].Values,
+		})
+	}
+
+	return data, nil
+}
+
+func (c *Clickhouse) QueryLog(ctx context.Context, query interface{}) ([]interface{}, int64, error) {
+	ckQueryParam := new(QueryParam)
+	if err := mapstructure.Decode(query, ckQueryParam); err != nil {
+		return nil, 0, err
+	}
+
+	if strings.Contains(ckQueryParam.Sql, "$__") {
+		var err error
+		ckQueryParam.Sql, err = macros.Macro(ckQueryParam.Sql, ckQueryParam.From, ckQueryParam.To)
+		if err != nil {
+			return nil, 0, err
+		}
+	}
+
+	rows, err := c.Query(ctx, ckQueryParam)
+	if err != nil {
+		logger.Warningf("query:%+v get data err:%v", ckQueryParam, err)
+		return nil, 0, err
+	}
+
+	limit := getLimit(len(rows), ckQueryParam.Limit)
+
+	logs := make([]interface{}, 0)
+	for i := 0; i < limit; i++ {
+		logs = append(logs, rows[i])
+	}
+
+	return logs, int64(limit), nil
+}
+
+func getLimit(rowLen, pLimit int) int {
+	limit := DefaultLimit
+	if pLimit > 0 {
+		limit = pLimit
+	}
+	if rowLen > limit {
+		return limit
+	}
+
+	return rowLen
+}

datasource/commons/eslike/eslike.go

@@ -0,0 +1,603 @@
+package eslike
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/araddon/dateparse"
+	"github.com/bitly/go-simplejson"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/mitchellh/mapstructure"
+	"github.com/olivere/elastic/v7"
+	"github.com/prometheus/common/model"
+	"github.com/toolkits/pkg/logger"
+)
+
+type Query struct {
+	Ref        string     `json:"ref" mapstructure:"ref"`
+	Index      string     `json:"index" mapstructure:"index"`
+	Filter     string     `json:"filter" mapstructure:"filter"`
+	MetricAggr MetricAggr `json:"value" mapstructure:"value"`
+	GroupBy    []GroupBy  `json:"group_by" mapstructure:"group_by"`
+	DateField  string     `json:"date_field" mapstructure:"date_field"`
+	Interval   int64      `json:"interval" mapstructure:"interval"`
+	Start      int64      `json:"start" mapstructure:"start"`
+	End        int64      `json:"end" mapstructure:"end"`
+	P          int        `json:"page" mapstructure:"page"`           // 页码
+	Limit      int        `json:"limit" mapstructure:"limit"`         // 每页个数
+	Ascending  bool       `json:"ascending" mapstructure:"ascending"` // 按照DataField排序
+
+	Timeout  int `json:"timeout" mapstructure:"timeout"`
+	MaxShard int `json:"max_shard" mapstructure:"max_shard"`
+}
+
+type MetricAggr struct {
+	Field string `json:"field" mapstructure:"field"`
+	Func  string `json:"func" mapstructure:"func"`
+	Ref   string `json:"ref" mapstructure:"ref"` // 变量名，A B C
+}
+
+type GroupBy struct {
+	Cate        GroupByCate `json:"cate" mapstructure:"cate"` // 分组类型
+	Field       string      `json:"field" mapstructure:"field"`
+	MinDocCount int64       `json:"min_doc_count" mapstructure:"min_doc_count"`
+	Order       string      `json:"order" mapstructure:"order"`
+	OrderBy     string      `json:"order_by" mapstructure:"order_by"`
+	Size        int         `json:"size" mapstructure:"size"`
+
+	Params   []Param `json:"params" mapstructure:"params"`     // 类型是 filter 时使用
+	Interval int64   `json:"interval" mapstructure:"interval"` // 分组间隔
+}
+
+type SearchFunc func(ctx context.Context, indices []string, source interface{}, timeout int, maxShard int) (*elastic.SearchResult, error)
+type QueryFieldsFunc func(indices []string) ([]string, error)
+
+// 分组类型
+type GroupByCate string
+
+const (
+	Filters  GroupByCate = "filters"
+	Histgram GroupByCate = "histgram"
+	Terms    GroupByCate = "terms"
+)
+
+// 参数
+type Param struct {
+	Alias string `json:"alias,omitempty"` // 别名，a=b的形式，filter 特有
+	Query string `json:"query,omitempty"` // 查询条件，filter 特有
+}
+
+type MetricPtr struct {
+	Data map[string][][]float64
+}
+
+func IterGetMap(m, ret map[string]interface{}, prefixKey string) {
+	for k, v := range m {
+		switch v.(type) {
+		case map[string]interface{}:
+			var key string
+			if prefixKey != "" {
+				key = fmt.Sprintf("%s.%s", prefixKey, k)
+			} else {
+				key = k
+			}
+			IterGetMap(v.(map[string]interface{}), ret, key)
+		default:
+			ret[prefixKey+"."+k] = []interface{}{v}
+		}
+	}
+}
+
+func TransferData(metric, ref string, m map[string][][]float64) []models.DataResp {
+	var datas []models.DataResp
+
+	for k, v := range m {
+		data := models.DataResp{
+			Ref:    ref,
+			Metric: make(model.Metric),
+			Labels: k,
+			Values: v,
+		}
+
+		data.Metric["__name__"] = model.LabelValue(metric)
+		labels := strings.Split(k, "--")
+		for _, label := range labels {
+			arr := strings.SplitN(label, "=", 2)
+			if len(arr) == 2 {
+				data.Metric[model.LabelName(arr[0])] = model.LabelValue(arr[1])
+			}
+		}
+		datas = append(datas, data)
+	}
+
+	for i := 0; i < len(datas); i++ {
+		for k, v := range datas[i].Metric {
+			if k == "__name__" {
+				datas[i].Metric[k] = model.LabelValue(ref) + "_" + v
+			}
+		}
+	}
+
+	return datas
+}
+
+func GetQueryString(filter string, q *elastic.RangeQuery) *elastic.BoolQuery {
+	var queryString *elastic.BoolQuery
+	if filter != "" {
+		if strings.Contains(filter, ":") || strings.Contains(filter, "AND") || strings.Contains(filter, "OR") || strings.Contains(filter, "NOT") {
+			queryString = elastic.NewBoolQuery().Must(elastic.NewQueryStringQuery(filter)).Filter(q)
+		} else {
+			queryString = elastic.NewBoolQuery().Filter(elastic.NewMultiMatchQuery(filter).Lenient(true).Type("phrase")).Filter(q)
+		}
+	} else {
+		queryString = elastic.NewBoolQuery().Should(q)
+	}
+
+	return queryString
+}
+
+func getUnixTs(timeStr string) int64 {
+	ts, err := strconv.ParseInt(timeStr, 10, 64)
+	if err == nil {
+		return ts
+	}
+
+	parsedTime, err := dateparse.ParseAny(timeStr)
+	if err != nil {
+		logger.Error("failed to ParseAny: ", err)
+		return 0
+	}
+	return parsedTime.UnixMilli()
+}
+
+func GetBuckts(labelKey string, keys []string, arr []interface{}, metrics *MetricPtr, labels string, ts int64, f string) {
+	var err error
+	bucketsKey := ""
+	if len(keys) > 0 {
+		bucketsKey = keys[0]
+	}
+
+	newlabels := ""
+	for i := 0; i < len(arr); i++ {
+		tmp := arr[i].(map[string]interface{})
+		keyAsString, getTs := tmp["key_as_string"]
+		if getTs {
+			ts = getUnixTs(keyAsString.(string))
+		}
+		keyValue := tmp["key"]
+		switch keyValue.(type) {
+		case json.Number, string:
+			if !getTs {
+				if labels != "" {
+					newlabels = fmt.Sprintf("%s--%s=%v", labels, labelKey, keyValue)
+				} else {
+					newlabels = fmt.Sprintf("%s=%v", labelKey, keyValue)
+				}
+			}
+		default:
+			continue
+		}
+
+		var finalValue float64
+		if len(keys) == 0 { // 计算 doc_count 的情况
+			count := tmp["doc_count"]
+			finalValue, err = count.(json.Number).Float64()
+			if err != nil {
+				logger.Warningf("labelKey:%s get value error:%v", labelKey, err)
+			}
+			newValues := []float64{float64(ts / 1000), finalValue}
+			metrics.Data[newlabels] = append(metrics.Data[newlabels], newValues)
+			continue
+		}
+
+		innerBuckets, exists := tmp[bucketsKey]
+		if !exists {
+			continue
+		}
+
+		nextBucketsArr, exists := innerBuckets.(map[string]interface{})["buckets"]
+		if exists {
+			if len(keys[1:]) >= 1 {
+				GetBuckts(bucketsKey, keys[1:], nextBucketsArr.([]interface{}), metrics, newlabels, ts, f)
+			} else {
+				GetBuckts(bucketsKey, []string{}, nextBucketsArr.([]interface{}), metrics, newlabels, ts, f)
+			}
+		} else {
+
+			// doc_count
+			if f == "count" || f == "nodata" {
+				count := tmp["doc_count"]
+				finalValue, err = count.(json.Number).Float64()
+				if err != nil {
+					logger.Warningf("get %v value error:%v", count, err)
+				}
+			} else {
+				values, exists := innerBuckets.(map[string]interface{})["value"]
+				if exists {
+					switch values.(type) {
+					case json.Number:
+						value, err := values.(json.Number).Float64()
+						if err != nil {
+							logger.Warningf("labelKey:%s get value error:%v", labelKey, err)
+						}
+						finalValue = value
+					}
+				} else {
+					switch values.(type) {
+					case map[string]interface{}:
+						var err error
+						values := innerBuckets.(map[string]interface{})["values"]
+						for _, v := range values.(map[string]interface{}) {
+							finalValue, err = v.(json.Number).Float64()
+							if err != nil {
+								logger.Warningf("labelKey:%s get value error:%v", labelKey, err)
+							}
+						}
+					default:
+						values := innerBuckets.(map[string]interface{})["values"]
+						for _, v := range values.(map[string]interface{}) {
+							// Todo 修复 v is nil 导致 panic 情况
+							finalValue, err = v.(json.Number).Float64()
+							if err != nil {
+								logger.Warningf("labelKey:%s get value error:%v", labelKey, err)
+							}
+						}
+					}
+				}
+			}
+
+			if _, exists := metrics.Data[newlabels]; !exists {
+				metrics.Data[newlabels] = [][]float64{}
+			}
+
+			newValues := []float64{float64(ts / 1000), finalValue}
+			metrics.Data[newlabels] = append(metrics.Data[newlabels], newValues)
+		}
+	}
+}
+
+func MakeLogQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	param := new(Query)
+	if err := mapstructure.Decode(query, param); err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(eventTags); i++ {
+		eventTags[i] = strings.Replace(eventTags[i], "=", ":", 1)
+	}
+
+	if len(eventTags) > 0 {
+		if param.Filter == "" {
+			param.Filter = strings.Join(eventTags, " AND ")
+		} else {
+			param.Filter = param.Filter + " AND " + strings.Join(eventTags, " AND ")
+		}
+	}
+
+	param.Start = start
+	param.End = end
+
+	return param, nil
+}
+
+func MakeTSQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	param := new(Query)
+	if err := mapstructure.Decode(query, param); err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(eventTags); i++ {
+		eventTags[i] = strings.Replace(eventTags[i], "=", ":", 1)
+	}
+
+	if len(eventTags) > 0 {
+		if param.Filter == "" {
+			param.Filter = strings.Join(eventTags, " AND ")
+		} else {
+			param.Filter = param.Filter + " AND " + strings.Join(eventTags, " AND ")
+		}
+	}
+	param.Start = start
+	param.End = end
+
+	return param, nil
+}
+
+func QueryData(ctx context.Context, queryParam interface{}, cliTimeout int64, version string, search SearchFunc) ([]models.DataResp, error) {
+	param := new(Query)
+	if err := mapstructure.Decode(queryParam, param); err != nil {
+		return nil, err
+	}
+
+	if param.Timeout == 0 {
+		param.Timeout = int(cliTimeout) / 1000
+	}
+
+	if param.Interval == 0 {
+		param.Interval = 60
+	}
+
+	if param.MaxShard < 1 {
+		param.MaxShard = 5
+	}
+
+	if param.DateField == "" {
+		param.DateField = "@timestamp"
+	}
+
+	indexArr := strings.Split(param.Index, ",")
+	q := elastic.NewRangeQuery(param.DateField)
+	now := time.Now().Unix()
+	var start, end int64
+	if param.End != 0 && param.Start != 0 {
+		end = param.End - param.End%param.Interval
+		start = param.Start - param.Start%param.Interval
+	} else {
+		end = now
+		start = end - param.Interval
+	}
+
+	delay, ok := ctx.Value("delay").(int64)
+	if ok && delay != 0 {
+		end = end - delay
+		start = start - delay
+	}
+
+	q.Gte(time.Unix(start, 0).UnixMilli())
+	q.Lte(time.Unix(end, 0).UnixMilli())
+	q.Format("epoch_millis")
+
+	field := param.MetricAggr.Field
+	groupBys := param.GroupBy
+
+	queryString := GetQueryString(param.Filter, q)
+
+	var aggr elastic.Aggregation
+	switch param.MetricAggr.Func {
+	case "avg":
+		aggr = elastic.NewAvgAggregation().Field(field)
+	case "max":
+		aggr = elastic.NewMaxAggregation().Field(field)
+	case "min":
+		aggr = elastic.NewMinAggregation().Field(field)
+	case "sum":
+		aggr = elastic.NewSumAggregation().Field(field)
+	case "count":
+		aggr = elastic.NewValueCountAggregation().Field(field)
+	case "p90":
+		aggr = elastic.NewPercentilesAggregation().Percentiles(90).Field(field)
+	case "p95":
+		aggr = elastic.NewPercentilesAggregation().Percentiles(95).Field(field)
+	case "p99":
+		aggr = elastic.NewPercentilesAggregation().Percentiles(99).Field(field)
+	case "median":
+		aggr = elastic.NewPercentilesAggregation().Percentiles(50).Field(field)
+	default:
+		return nil, fmt.Errorf("func %s not support", param.MetricAggr.Func)
+	}
+
+	tsAggr := elastic.NewDateHistogramAggregation().
+		Field(param.DateField).
+		MinDocCount(1)
+
+	if strings.HasPrefix(version, "7") {
+		tsAggr.FixedInterval(fmt.Sprintf("%ds", param.Interval))
+	} else {
+		// 兼容 7.0 以下的版本
+		// OpenSearch 也使用这个字段
+		tsAggr.Interval(fmt.Sprintf("%ds", param.Interval))
+	}
+
+	// group by
+	var groupByAggregation elastic.Aggregation
+	if len(groupBys) > 0 {
+		groupBy := groupBys[0]
+
+		if groupBy.MinDocCount == 0 {
+			groupBy.MinDocCount = 1
+		}
+
+		if groupBy.Size == 0 {
+			groupBy.Size = 300
+		}
+
+		switch groupBy.Cate {
+		case Terms:
+			if param.MetricAggr.Func != "count" {
+				groupByAggregation = elastic.NewTermsAggregation().Field(groupBy.Field).SubAggregation(field, aggr).OrderByKeyDesc().Size(groupBy.Size).MinDocCount(int(groupBy.MinDocCount))
+			} else {
+				groupByAggregation = elastic.NewTermsAggregation().Field(groupBy.Field).OrderByKeyDesc().Size(groupBy.Size).MinDocCount(int(groupBy.MinDocCount))
+			}
+		case Histgram:
+			if param.MetricAggr.Func != "count" {
+				groupByAggregation = elastic.NewHistogramAggregation().Field(groupBy.Field).Interval(float64(groupBy.Interval)).SubAggregation(field, aggr)
+			} else {
+				groupByAggregation = elastic.NewHistogramAggregation().Field(groupBy.Field).Interval(float64(groupBy.Interval))
+			}
+		case Filters:
+			for _, filterParam := range groupBy.Params {
+				if param.MetricAggr.Func != "count" {
+					groupByAggregation = elastic.NewFilterAggregation().Filter(elastic.NewTermQuery(filterParam.Query, "true")).SubAggregation(field, aggr)
+				} else {
+					groupByAggregation = elastic.NewFilterAggregation().Filter(elastic.NewTermQuery(filterParam.Query, "true"))
+				}
+			}
+		}
+
+		for i := 1; i < len(groupBys); i++ {
+			groupBy := groupBys[i]
+
+			if groupBy.MinDocCount == 0 {
+				groupBy.MinDocCount = 1
+			}
+
+			if groupBy.Size == 0 {
+				groupBy.Size = 300
+			}
+
+			switch groupBy.Cate {
+			case Terms:
+				groupByAggregation = elastic.NewTermsAggregation().Field(groupBy.Field).SubAggregation(groupBys[i-1].Field, groupByAggregation).OrderByKeyDesc().Size(groupBy.Size).MinDocCount(int(groupBy.MinDocCount))
+			case Histgram:
+				groupByAggregation = elastic.NewHistogramAggregation().Field(groupBy.Field).Interval(float64(groupBy.Interval)).SubAggregation(groupBys[i-1].Field, groupByAggregation)
+			case Filters:
+				for _, filterParam := range groupBy.Params {
+					groupByAggregation = elastic.NewFilterAggregation().Filter(elastic.NewTermQuery(filterParam.Query, "true")).SubAggregation(groupBys[i-1].Field, groupByAggregation)
+				}
+			}
+		}
+
+		tsAggr.SubAggregation(groupBys[len(groupBys)-1].Field, groupByAggregation)
+	} else if param.MetricAggr.Func != "count" {
+		tsAggr.SubAggregation(field, aggr)
+	}
+
+	source, _ := queryString.Source()
+	b, _ := json.Marshal(source)
+	logger.Debugf("query_data q:%+v tsAggr:%+v query_string:%s", param, tsAggr, string(b))
+
+	searchSource := elastic.NewSearchSource().
+		Query(queryString).
+		Aggregation("ts", tsAggr)
+
+	searchSourceString, err := searchSource.Source()
+	if err != nil {
+		logger.Warningf("query_data searchSource:%s to string error:%v", searchSourceString, err)
+	}
+
+	jsonSearchSource, err := json.Marshal(searchSourceString)
+	if err != nil {
+		logger.Warningf("query_data searchSource:%s to json error:%v", searchSourceString, err)
+	}
+
+	result, err := search(ctx, indexArr, searchSource, param.Timeout, param.MaxShard)
+	if err != nil {
+		logger.Warningf("query_data searchSource:%s query_data error:%v", searchSourceString, err)
+		return nil, err
+	}
+
+	logger.Debugf("query_data searchSource:%s resp:%s", string(jsonSearchSource), string(result.Aggregations["ts"]))
+
+	js, err := simplejson.NewJson(result.Aggregations["ts"])
+	if err != nil {
+		return nil, err
+	}
+
+	bucketsData, err := js.Get("buckets").Array()
+	if err != nil {
+		return nil, err
+	}
+
+	var keys []string
+	for i := len(groupBys) - 1; i >= 0; i-- {
+		keys = append(keys, groupBys[i].Field)
+	}
+
+	if param.MetricAggr.Func != "count" {
+		keys = append(keys, field)
+	}
+
+	metrics := &MetricPtr{Data: make(map[string][][]float64)}
+
+	GetBuckts("", keys, bucketsData, metrics, "", 0, param.MetricAggr.Func)
+
+	return TransferData(fmt.Sprintf("%s_%s", field, param.MetricAggr.Func), param.Ref, metrics.Data), nil
+}
+
+func HitFilter(typ string) bool {
+	switch typ {
+	case "keyword", "date", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long":
+		return false
+	default:
+		return true
+	}
+}
+
+func QueryLog(ctx context.Context, queryParam interface{}, timeout int64, version string, maxShard int, search SearchFunc) ([]interface{}, int64, error) {
+	param := new(Query)
+	if err := mapstructure.Decode(queryParam, param); err != nil {
+		return nil, 0, err
+	}
+
+	if param.Timeout == 0 {
+		param.Timeout = int(timeout)
+	}
+
+	indexArr := strings.Split(param.Index, ",")
+
+	now := time.Now().Unix()
+	var start, end int64
+	if param.End != 0 && param.Start != 0 {
+		end = param.End - param.End%param.Interval
+		start = param.Start - param.Start%param.Interval
+	} else {
+		end = now
+		start = end - param.Interval
+	}
+
+	q := elastic.NewRangeQuery(param.DateField)
+	q.Gte(time.Unix(start, 0).UnixMilli())
+	q.Lte(time.Unix(end, 0).UnixMilli())
+	q.Format("epoch_millis")
+
+	queryString := GetQueryString(param.Filter, q)
+
+	if param.Limit <= 0 {
+		param.Limit = 10
+	}
+
+	if param.MaxShard < 1 {
+		param.MaxShard = maxShard
+	}
+
+	source := elastic.NewSearchSource().
+		TrackTotalHits(true).
+		Query(queryString).
+		From(param.P).
+		Size(param.Limit).
+		Sort(param.DateField, param.Ascending)
+
+	result, err := search(ctx, indexArr, source, param.Timeout, param.MaxShard)
+	if err != nil {
+		logger.Warningf("query data error:%v", err)
+		return nil, 0, err
+	}
+
+	total := result.TotalHits()
+
+	var ret []interface{}
+
+	b, _ := json.Marshal(source)
+	logger.Debugf("query data result query source:%s len:%d total:%d", string(b), len(result.Hits.Hits), total)
+
+	resultBytes, _ := json.Marshal(result)
+	logger.Debugf("query data result query source:%s result:%s", string(b), string(resultBytes))
+
+	if strings.HasPrefix(version, "6") {
+		for i := 0; i < len(result.Hits.Hits); i++ {
+			var x map[string]interface{}
+			err := json.Unmarshal(result.Hits.Hits[i].Source, &x)
+			if err != nil {
+				logger.Warningf("Unmarshal soruce error:%v", err)
+				continue
+			}
+
+			if result.Hits.Hits[i].Fields == nil {
+				result.Hits.Hits[i].Fields = make(map[string]interface{})
+			}
+
+			IterGetMap(x, result.Hits.Hits[i].Fields, "")
+			ret = append(ret, result.Hits.Hits[i])
+		}
+	} else {
+		for _, hit := range result.Hits.Hits {
+			ret = append(ret, hit)
+		}
+	}
+
+	return ret, total, nil
+}

datasource/datasource.go

@@ -0,0 +1,114 @@
+package datasource
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/ccfos/nightingale/v6/models"
+)
+
+type DatasourceType struct {
+	Id             int64  `json:"id"`
+	Category       string `json:"category"`
+	PluginType     string `json:"type"`
+	PluginTypeName string `json:"type_name"`
+}
+
+type Keys struct {
+	ValueKey   string `json:"valueKey" mapstructure:"valueKey"` // 多个用空格分隔
+	LabelKey   string `json:"labelKey" mapstructure:"labelKey"` // 多个用空格分隔
+	TimeKey    string `json:"timeKey" mapstructure:"timeKey"`
+	TimeFormat string `json:"timeFormat" mapstructure:"timeFormat"`
+}
+
+var DatasourceTypes map[int64]DatasourceType
+
+func init() {
+	DatasourceTypes = make(map[int64]DatasourceType)
+	DatasourceTypes[1] = DatasourceType{
+		Id:             1,
+		Category:       "timeseries",
+		PluginType:     "prometheus",
+		PluginTypeName: "Prometheus Like",
+	}
+
+	DatasourceTypes[2] = DatasourceType{
+		Id:             2,
+		Category:       "logging",
+		PluginType:     "elasticsearch",
+		PluginTypeName: "Elasticsearch",
+	}
+
+	DatasourceTypes[3] = DatasourceType{
+		Id:             3,
+		Category:       "logging",
+		PluginType:     "aliyun-sls",
+		PluginTypeName: "SLS",
+	}
+
+	DatasourceTypes[4] = DatasourceType{
+		Id:             4,
+		Category:       "timeseries",
+		PluginType:     "ck",
+		PluginTypeName: "ClickHouse",
+	}
+}
+
+type NewDatasrouceFn func(settings map[string]interface{}) (Datasource, error)
+
+var datasourceRegister = map[string]NewDatasrouceFn{}
+
+type Datasource interface {
+	Init(settings map[string]interface{}) (Datasource, error) // 初始化配置
+	InitClient() error                                        // 初始化客户端
+	Validate(ctx context.Context) error                       // 参数验证
+	Equal(p Datasource) bool                                  // 验证是否相等
+	MakeLogQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error)
+	MakeTSQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error)
+	QueryData(ctx context.Context, query interface{}) ([]models.DataResp, error)
+	QueryLog(ctx context.Context, query interface{}) ([]interface{}, int64, error)
+
+	// 在生成告警事件时，会调用该方法，用于获取额外的数据
+	QueryMapData(ctx context.Context, query interface{}) ([]map[string]string, error)
+}
+
+func RegisterDatasource(typ string, p Datasource) {
+	if _, found := datasourceRegister[typ]; found {
+		return
+	}
+	datasourceRegister[typ] = p.Init
+}
+
+func GetDatasourceByType(typ string, settings map[string]interface{}) (Datasource, error) {
+	typ = strings.ReplaceAll(typ, ".logging", "")
+	fn, found := datasourceRegister[typ]
+	if !found {
+		return nil, fmt.Errorf("plugin type %s not found", typ)
+	}
+
+	plug, err := fn(settings)
+	if err != nil {
+		return nil, err
+	}
+
+	return plug, nil
+}
+
+type DatasourceInfo struct {
+	Id             int64                  `json:"id"`
+	Name           string                 `json:"name"`
+	Description    string                 `json:"description"`
+	ClusterName    string                 `json:"cluster_name"`
+	Category       string                 `json:"category"`
+	PluginId       int64                  `json:"plugin_id"`
+	Type           string                 `json:"plugin_type"`
+	PluginTypeName string                 `json:"plugin_type_name"`
+	Settings       map[string]interface{} `json:"settings"`
+	HTTPJson       models.HTTP            `json:"http"`
+	AuthJson       models.Auth            `json:"auth"`
+	Status         string                 `json:"status"`
+	CreatedAt      int64                  `json:"created_at"`
+	UpdatedAt      int64                  `json:"updated_at"`
+	IsDefault      bool                   `json:"is_default"`
+}

datasource/es/es.go

@@ -0,0 +1,405 @@
+package es
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/datasource"
+	"github.com/ccfos/nightingale/v6/datasource/commons/eslike"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/tlsx"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/olivere/elastic/v7"
+	"github.com/toolkits/pkg/logger"
+)
+
+const (
+	ESType = "elasticsearch"
+)
+
+type Elasticsearch struct {
+	Addr        string            `json:"es.addr" mapstructure:"es.addr"`
+	Nodes       []string          `json:"es.nodes" mapstructure:"es.nodes"`
+	Timeout     int64             `json:"es.timeout" mapstructure:"es.timeout"` // millis
+	Basic       BasicAuth         `json:"es.basic" mapstructure:"es.basic"`
+	TLS         TLS               `json:"es.tls" mapstructure:"es.tls"`
+	Version     string            `json:"es.version" mapstructure:"es.version"`
+	Headers     map[string]string `json:"es.headers" mapstructure:"es.headers"`
+	MinInterval int               `json:"es.min_interval" mapstructure:"es.min_interval"` // seconds
+	MaxShard    int               `json:"es.max_shard" mapstructure:"es.max_shard"`
+	ClusterName string            `json:"es.cluster_name" mapstructure:"es.cluster_name"`
+	EnableWrite bool              `json:"es.enable_write" mapstructure:"es.enable_write"` // 允许写操作
+	Client      *elastic.Client   `json:"es.client" mapstructure:"es.client"`
+}
+
+type TLS struct {
+	SkipTlsVerify bool `json:"es.tls.skip_tls_verify" mapstructure:"es.tls.skip_tls_verify"`
+}
+
+type BasicAuth struct {
+	Enable   bool   `json:"es.auth.enable" mapstructure:"es.auth.enable"`
+	Username string `json:"es.user" mapstructure:"es.user"`
+	Password string `json:"es.password" mapstructure:"es.password"`
+}
+
+func init() {
+	datasource.RegisterDatasource(ESType, new(Elasticsearch))
+}
+
+func (e *Elasticsearch) Init(settings map[string]interface{}) (datasource.Datasource, error) {
+	newest := new(Elasticsearch)
+	err := mapstructure.Decode(settings, newest)
+	return newest, err
+}
+
+func (e *Elasticsearch) InitClient() error {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout: time.Duration(e.Timeout) * time.Millisecond,
+		}).DialContext,
+		ResponseHeaderTimeout: time.Duration(e.Timeout) * time.Millisecond,
+	}
+
+	if len(e.Nodes) > 0 {
+		e.Addr = e.Nodes[0]
+	}
+
+	if strings.Contains(e.Addr, "https") {
+		tlsConfig := tlsx.ClientConfig{
+			InsecureSkipVerify: e.TLS.SkipTlsVerify,
+			UseTLS:             true,
+		}
+		cfg, err := tlsConfig.TLSConfig()
+		if err != nil {
+			return err
+		}
+		transport.TLSClientConfig = cfg
+	}
+
+	var err error
+	options := []elastic.ClientOptionFunc{
+		elastic.SetURL(e.Nodes...),
+	}
+
+	if e.Basic.Username != "" {
+		options = append(options, elastic.SetBasicAuth(e.Basic.Username, e.Basic.Password))
+	}
+
+	headers := http.Header{}
+	for k, v := range e.Headers {
+		headers[k] = []string{v}
+	}
+
+	options = append(options, elastic.SetHeaders(headers))
+	options = append(options, elastic.SetHttpClient(&http.Client{Transport: transport}))
+	options = append(options, elastic.SetSniff(false))
+	options = append(options, elastic.SetHealthcheck(false))
+
+	e.Client, err = elastic.NewClient(options...)
+	return err
+}
+
+func (e *Elasticsearch) Equal(other datasource.Datasource) bool {
+	sort.Strings(e.Nodes)
+	sort.Strings(other.(*Elasticsearch).Nodes)
+
+	if strings.Join(e.Nodes, ",") != strings.Join(other.(*Elasticsearch).Nodes, ",") {
+		return false
+	}
+
+	if e.Basic.Username != other.(*Elasticsearch).Basic.Username {
+		return false
+	}
+
+	if e.Basic.Password != other.(*Elasticsearch).Basic.Password {
+		return false
+	}
+
+	if e.TLS.SkipTlsVerify != other.(*Elasticsearch).TLS.SkipTlsVerify {
+		return false
+	}
+	if e.EnableWrite != other.(*Elasticsearch).EnableWrite {
+		return false
+	}
+
+	if !reflect.DeepEqual(e.Headers, other.(*Elasticsearch).Headers) {
+		return false
+	}
+
+	return true
+}
+
+func (e *Elasticsearch) Validate(ctx context.Context) (err error) {
+	if len(e.Nodes) == 0 {
+		return fmt.Errorf("need a valid addr")
+	}
+
+	for _, addr := range e.Nodes {
+		_, err = url.Parse(addr)
+		if err != nil {
+			return fmt.Errorf("parse addr error: %v", err)
+		}
+	}
+
+	if e.Basic.Enable && (len(e.Basic.Username) == 0 || len(e.Basic.Password) == 0) {
+		return fmt.Errorf("need a valid user, password")
+	}
+
+	if e.MaxShard == 0 {
+		e.MaxShard = 5
+	}
+
+	if e.MinInterval < 10 {
+		e.MinInterval = 10
+	}
+
+	if e.Timeout == 0 {
+		e.Timeout = 60000
+	}
+
+	if !strings.HasPrefix(e.Version, "6") && !strings.HasPrefix(e.Version, "7") {
+		return fmt.Errorf("version must be 6.0+ or 7.0+")
+	}
+
+	return nil
+}
+
+func (e *Elasticsearch) MakeLogQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	return eslike.MakeLogQuery(ctx, query, eventTags, start, end)
+}
+
+func (e *Elasticsearch) MakeTSQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	return eslike.MakeTSQuery(ctx, query, eventTags, start, end)
+}
+
+func (e *Elasticsearch) QueryData(ctx context.Context, queryParam interface{}) ([]models.DataResp, error) {
+
+	search := func(ctx context.Context, indices []string, source interface{}, timeout int, maxShard int) (*elastic.SearchResult, error) {
+		return e.Client.Search().
+			Index(indices...).
+			Source(source).
+			Timeout(fmt.Sprintf("%ds", timeout)).
+			MaxConcurrentShardRequests(maxShard).
+			Do(ctx)
+	}
+
+	return eslike.QueryData(ctx, queryParam, e.Timeout, e.Version, search)
+}
+
+func (e *Elasticsearch) QueryIndices() ([]string, error) {
+	result, err := e.Client.IndexNames()
+
+	return result, err
+}
+
+func (e *Elasticsearch) QueryFields(indexs []string) ([]string, error) {
+	var fields []string
+	result, err := elastic.NewGetFieldMappingService(e.Client).Index(indexs...).Do(context.Background())
+	if err != nil {
+		return fields, err
+	}
+
+	fieldMap := make(map[string]struct{})
+	for _, indexMap := range result {
+		if m, exists := indexMap.(map[string]interface{})["mappings"]; exists {
+			for k, v := range m.(map[string]interface{}) {
+				// 兼容 es6 版本
+				if k == "doc" && strings.HasPrefix(e.Version, "6") {
+					// if k == "doc" {
+					for kk, vv := range v.(map[string]interface{}) {
+						typ := getFieldType(kk, vv.(map[string]interface{}))
+						if eslike.HitFilter(typ) {
+							continue
+						}
+
+						if _, exsits := fieldMap[kk]; !exsits {
+							fieldMap[kk] = struct{}{}
+							fields = append(fields, kk)
+						}
+					}
+				} else {
+					// es7 版本
+					typ := getFieldType(k, v.(map[string]interface{}))
+					if eslike.HitFilter(typ) {
+						continue
+					}
+
+					if _, exsits := fieldMap[k]; !exsits {
+						fieldMap[k] = struct{}{}
+						fields = append(fields, k)
+					}
+				}
+
+			}
+		}
+	}
+
+	sort.Strings(fields)
+	return fields, nil
+}
+
+func (e *Elasticsearch) QueryLog(ctx context.Context, queryParam interface{}) ([]interface{}, int64, error) {
+
+	search := func(ctx context.Context, indices []string, source interface{}, timeout int, maxShard int) (*elastic.SearchResult, error) {
+		// 应该是之前为了获取 fields 字段，做的这个兼容
+		// fields, err := e.QueryFields(indices)
+		// if err != nil {
+		// 	logger.Warningf("query data error:%v", err)
+		// 	return nil, err
+		// }
+
+		// if source != nil && strings.HasPrefix(e.Version, "7") {
+		// 	source = source.(*elastic.SearchSource).DocvalueFields(fields...)
+		// }
+
+		return e.Client.Search().
+			Index(indices...).
+			MaxConcurrentShardRequests(maxShard).
+			Source(source).
+			Timeout(fmt.Sprintf("%ds", timeout)).
+			Do(ctx)
+	}
+
+	return eslike.QueryLog(ctx, queryParam, e.Timeout, e.Version, e.MaxShard, search)
+}
+
+func (e *Elasticsearch) QueryFieldValue(indexs []string, field string, query string) ([]string, error) {
+	var values []string
+	search := e.Client.Search().
+		Index(indexs...).
+		Size(0)
+
+	if query != "" {
+		search = search.Query(elastic.NewBoolQuery().Must(elastic.NewQueryStringQuery(query)))
+	}
+	search = search.Aggregation("distinct", elastic.NewTermsAggregation().Field(field).Size(10000))
+
+	result, err := search.Do(context.Background())
+	if err != nil {
+		return values, err
+	}
+
+	agg, found := result.Aggregations.Terms("distinct")
+	if !found {
+		return values, nil
+	}
+
+	for _, bucket := range agg.Buckets {
+		values = append(values, bucket.Key.(string))
+	}
+
+	return values, nil
+}
+
+func (e *Elasticsearch) Test(ctx context.Context) (err error) {
+	err = e.Validate(ctx)
+	if err != nil {
+		return err
+	}
+
+	if e.Addr == "" {
+		return fmt.Errorf("addr is invalid")
+	}
+
+	if e.Version == "7.10+" {
+		options := []elastic.ClientOptionFunc{
+			elastic.SetURL(e.Addr),
+		}
+
+		if e.Basic.Enable {
+			options = append(options, elastic.SetBasicAuth(e.Basic.Username, e.Basic.Password))
+		}
+
+		client, err := elastic.NewClient(options...)
+		if err != nil {
+			return fmt.Errorf("config is invalid:%v", err)
+		}
+
+		_, err = client.ElasticsearchVersion(e.Addr)
+		if err != nil {
+			return fmt.Errorf("config is invalid:%v", err)
+		}
+	} else {
+		return fmt.Errorf("version must be 7.10+")
+	}
+
+	return nil
+}
+
+func getFieldType(key string, m map[string]interface{}) string {
+	if innerMap, exists := m["mapping"]; exists {
+		if innerM, exists := innerMap.(map[string]interface{})[key]; exists {
+			if typ, exists := innerM.(map[string]interface{})["type"]; exists {
+				return typ.(string)
+			}
+		} else {
+			arr := strings.Split(key, ".")
+			if innerM, exists := innerMap.(map[string]interface{})[arr[len(arr)-1]]; exists {
+				if typ, exists := innerM.(map[string]interface{})["type"]; exists {
+					return typ.(string)
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func (e *Elasticsearch) QueryMapData(ctx context.Context, query interface{}) ([]map[string]string, error) {
+	search := func(ctx context.Context, indices []string, source interface{}, timeout int, maxShard int) (*elastic.SearchResult, error) {
+
+		return e.Client.Search().
+			Index(indices...).
+			Source(source).
+			Timeout(fmt.Sprintf("%ds", timeout)).
+			Do(ctx)
+	}
+
+	param := new(eslike.Query)
+	if err := mapstructure.Decode(query, param); err != nil {
+		return nil, err
+	}
+	// 扩大查询范围, 解决上一次查询消耗时间太多，导致本次查询时间范围起止时间，滞后问题
+	param.Interval += 30
+
+	res, _, err := eslike.QueryLog(ctx, param, e.Timeout, e.Version, e.MaxShard, search)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []map[string]string
+	for _, item := range res {
+		logger.Debugf("query:%v item:%v", query, item)
+		if itemMap, ok := item.(*elastic.SearchHit); ok {
+			mItem := make(map[string]string)
+			// 遍历 fields 字段的每个键值对
+			sourceMap := make(map[string]interface{})
+			err := json.Unmarshal(itemMap.Source, &sourceMap)
+			if err != nil {
+				logger.Warningf("unmarshal source%s error:%v", string(itemMap.Source), err)
+				continue
+			}
+
+			for k, v := range sourceMap {
+				mItem[k] = fmt.Sprintf("%v", v)
+			}
+
+			// 将处理好的 map 添加到 m 切片中
+			result = append(result, mItem)
+
+			// 只取第一条数据
+			break
+		}
+	}
+
+	return result, nil
+}

datasource/prom/prom.go

@@ -0,0 +1,15 @@
+package prom
+
+type Prometheus struct {
+	PrometheusAddr  string `json:"prometheus.addr"`
+	PrometheusBasic struct {
+		PrometheusUser string `json:"prometheus.user"`
+		PrometheusPass string `json:"prometheus.password"`
+	} `json:"prometheus.basic"`
+	Headers           map[string]string `json:"prometheus.headers"`
+	PrometheusTimeout int64             `json:"prometheus.timeout"`
+	ClusterName       string            `json:"prometheus.cluster_name"`
+	WriteAddr         string            `json:"prometheus.write_addr"`
+	TsdbType          string            `json:"prometheus.tsdb_type"`
+	InternalAddr      string            `json:"prometheus.internal_addr"`
+}

datasource/tdengine/tdengine.go

@@ -1,24 +1,32 @@
 package tdengine
 
 import (
-	"encoding/base64"
+	"context"
 	"encoding/json"
 	"fmt"
-	"net"
-	"net/http"
 	"reflect"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/ccfos/nightingale/v6/alert/aconf"
-	"github.com/ccfos/nightingale/v6/models"
-	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/prometheus/common/model"
 	"github.com/toolkits/pkg/logger"
+
+	"github.com/ccfos/nightingale/v6/datasource"
+	td "github.com/ccfos/nightingale/v6/dskit/tdengine"
+	"github.com/ccfos/nightingale/v6/models"
+
+	"github.com/mitchellh/mapstructure"
 )
 
+const (
+	TDEngineType = "tdengine"
+)
+
+type TDengine struct {
+	td.Tdengine `json:",inline" mapstructure:",squash"`
+}
+
 type TdengineQuery struct {
 	From     string `json:"from"`
 	Interval int64  `json:"interval"`
@@ -34,162 +42,94 @@ type Keys struct {
 	TimeFormat string `json:"timeFormat"`
 }
 
-type TdengineClientMap struct {
-	sync.RWMutex
-	ctx           *ctx.Context
-	heartbeat     aconf.HeartbeatConfig
-	ReaderClients map[int64]*tdengineClient
+func init() {
+	datasource.RegisterDatasource(TDEngineType, new(TDengine))
 }
 
-func (pc *TdengineClientMap) Set(datasourceId int64, r *tdengineClient) {
-	if r == nil {
-		return
-	}
-	pc.Lock()
-	defer pc.Unlock()
-	pc.ReaderClients[datasourceId] = r
+func (td *TDengine) Init(settings map[string]interface{}) (datasource.Datasource, error) {
+	newest := new(TDengine)
+	err := mapstructure.Decode(settings, newest)
+	return newest, err
 }
 
-func (pc *TdengineClientMap) GetDatasourceIds() []int64 {
-	pc.RLock()
-	defer pc.RUnlock()
-	var datasourceIds []int64
-	for k := range pc.ReaderClients {
-		datasourceIds = append(datasourceIds, k)
+func (td *TDengine) InitClient() error {
+	td.InitCli()
+	return nil
+}
+
+func (td *TDengine) Equal(other datasource.Datasource) bool {
+	otherTD, ok := other.(*TDengine)
+	if !ok {
+		return false
 	}
 
-	return datasourceIds
-}
-
-func (pc *TdengineClientMap) GetCli(datasourceId int64) *tdengineClient {
-	pc.RLock()
-	defer pc.RUnlock()
-	c := pc.ReaderClients[datasourceId]
-	return c
-}
-
-func (pc *TdengineClientMap) IsNil(datasourceId int64) bool {
-	pc.RLock()
-	defer pc.RUnlock()
-
-	c, exists := pc.ReaderClients[datasourceId]
-	if !exists {
-		return true
+	if td.Addr != otherTD.Addr {
+		return false
 	}
 
-	return c == nil
-}
-
-func (pc *TdengineClientMap) Reset() {
-	pc.Lock()
-	defer pc.Unlock()
-
-	pc.ReaderClients = make(map[int64]*tdengineClient)
-}
-
-func (pc *TdengineClientMap) Del(datasourceId int64) {
-	pc.Lock()
-	defer pc.Unlock()
-	delete(pc.ReaderClients, datasourceId)
-}
-
-type tdengineClient struct {
-	url    string
-	client *http.Client
-	header map[string][]string
-}
-
-func newTdengine(po TdengineOption) *tdengineClient {
-	tc := &tdengineClient{
-		url: po.Url,
-	}
-	tc.client = &http.Client{
-		Transport: &http.Transport{
-			Proxy: http.ProxyFromEnvironment,
-			DialContext: (&net.Dialer{
-				Timeout:   30 * time.Second,
-				KeepAlive: 30 * time.Second,
-			}).DialContext,
-			IdleConnTimeout:       90 * time.Second,
-			TLSHandshakeTimeout:   10 * time.Second,
-			ExpectContinueTimeout: 1 * time.Second,
-			DisableCompression:    true,
-		},
-	}
-
-	tc.header = map[string][]string{
-		"Connection": {"keep-alive"},
-	}
-
-	for _, v := range po.Headers {
-		kv := strings.Split(v, ":")
-		if len(kv) != 2 {
-			continue
+	if td.Basic != nil && otherTD.Basic != nil {
+		if td.Basic.User != otherTD.Basic.User {
+			return false
 		}
-		tc.header[kv[0]] = []string{kv[1]}
-	}
 
-	if po.BasicAuthUser != "" {
-		basic := base64.StdEncoding.EncodeToString([]byte(po.BasicAuthUser + ":" + po.BasicAuthPass))
-		tc.header["Authorization"] = []string{fmt.Sprintf("Basic %s", basic)}
-	}
-
-	return tc
-}
-
-type APIResponse struct {
-	Code       int             `json:"code"`
-	ColumnMeta [][]interface{} `json:"column_meta"`
-	Data       [][]interface{} `json:"data"`
-	Rows       int             `json:"rows"`
-}
-
-func (tc *tdengineClient) QueryTable(query string) (APIResponse, error) {
-	var apiResp APIResponse
-	req, err := http.NewRequest("POST", tc.url+"/rest/sql", strings.NewReader(query))
-	if err != nil {
-		return apiResp, err
-	}
-
-	for k, v := range tc.header {
-		req.Header[k] = v
-	}
-
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := tc.client.Do(req)
-	if err != nil {
-		return apiResp, err
-	}
-	defer resp.Body.Close()
-
-	// 限制响应体大小为10MB
-	maxSize := int64(10 * 1024 * 1024) // 10MB
-	limitedReader := http.MaxBytesReader(nil, resp.Body, maxSize)
-
-	if resp.StatusCode != http.StatusOK {
-		return apiResp, fmt.Errorf("HTTP error, status: %s", resp.Status)
-	}
-
-	err = json.NewDecoder(limitedReader).Decode(&apiResp)
-	if err != nil {
-		if strings.Contains(err.Error(), "http: request body too large") {
-			return apiResp, fmt.Errorf("response body exceeds 10MB limit")
+		if td.Basic.Password != otherTD.Basic.Password {
+			return false
 		}
-		return apiResp, err
 	}
 
-	return apiResp, nil
+	if td.Token != otherTD.Token {
+		return false
+	}
+
+	if td.Timeout != otherTD.Timeout {
+		return false
+	}
+
+	if td.DialTimeout != otherTD.DialTimeout {
+		return false
+	}
+
+	if td.MaxIdleConnsPerHost != otherTD.MaxIdleConnsPerHost {
+		return false
+	}
+
+	if len(td.Headers) != len(otherTD.Headers) {
+		return false
+	}
+
+	for k, v := range td.Headers {
+		if otherV, ok := otherTD.Headers[k]; !ok || v != otherV {
+			return false
+		}
+	}
+	return true
 }
 
-func (tc *tdengineClient) QueryLog(query interface{}) (APIResponse, error) {
-	b, err := json.Marshal(query)
+func (td *TDengine) Validate(ctx context.Context) (err error) {
+	return nil
+}
+
+func (td *TDengine) MakeLogQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	return nil, nil
+}
+
+func (td *TDengine) MakeTSQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
+	return nil, nil
+}
+
+func (td *TDengine) QueryData(ctx context.Context, queryParam interface{}) ([]models.DataResp, error) {
+	return td.Query(queryParam, 0)
+}
+
+func (td *TDengine) QueryLog(ctx context.Context, queryParam interface{}) ([]interface{}, int64, error) {
+	b, err := json.Marshal(queryParam)
 	if err != nil {
-		return APIResponse{}, err
+		return nil, 0, err
 	}
 	var q TdengineQuery
 	err = json.Unmarshal(b, &q)
 	if err != nil {
-		return APIResponse{}, err
+		return nil, 0, err
 	}
 
 	if q.Interval == 0 {
@@ -218,135 +158,19 @@ func (tc *tdengineClient) QueryLog(query interface{}) (APIResponse, error) {
 		q.Query = q.Query + " limit 200"
 	}
 
-	data, err := tc.QueryTable(q.Query)
+	data, err := td.QueryTable(q.Query)
 	if err != nil {
-		return data, err
+		return nil, 0, err
 	}
 
-	return TimeFormat(data, q.Keys.TimeFormat), err
+	return ConvertToTable(data), int64(len(data.Data)), nil
 }
 
-func TimeFormat(src APIResponse, timeFormat string) APIResponse {
-	if timeFormat == "" {
-		return src
-	}
-
-	tsIdx := -1
-	for colIndex, colData := range src.ColumnMeta {
-		//  类型参考 https://docs.taosdata.com/taos-sql/data-type/
-		// 处理v2版本数字类型和v3版本字符串类型
-		switch t := colData[1].(type) {
-		case float64:
-			// v2版本数字类型映射
-			if int(t) == 9 { // TIMESTAMP type in v2
-				tsIdx = colIndex
-				break
-			}
-		case string:
-			// v3版本直接使用字符串类型
-			if t == "TIMESTAMP" {
-				tsIdx = colIndex
-				break
-			}
-		default:
-			logger.Warningf("unexpected column type: %v", colData[1])
-			continue
-		}
-	}
-
-	if tsIdx == -1 {
-		return src
-	}
-
-	for i := range src.Data {
-		var t time.Time
-		var err error
-
-		switch tsVal := src.Data[i][tsIdx].(type) {
-		case string:
-			// 尝试解析不同格式的时间字符串
-			t, err = parseTimeString(tsVal)
-			if err != nil {
-				logger.Warningf("parse timestamp string failed: %v, value: %v", err, tsVal)
-				continue
-			}
-		default:
-			logger.Warningf("unexpected timestamp type: %T, value: %v", tsVal, tsVal)
-			continue
-		}
-
-		src.Data[i][tsIdx] = t.In(time.Local).Format(timeFormat)
-	}
-	return src
+func (td *TDengine) QueryMapData(ctx context.Context, query interface{}) ([]map[string]string, error) {
+	return nil, nil
 }
 
-func parseTimeString(ts string) (time.Time, error) {
-	// 尝试不同的时间格式
-	formats := []string{
-		// 标准格式
-		time.Layout,      // "01/02 03:04:05PM '06 -0700"
-		time.ANSIC,       // "Mon Jan _2 15:04:05 2006"
-		time.UnixDate,    // "Mon Jan _2 15:04:05 MST 2006"
-		time.RubyDate,    // "Mon Jan 02 15:04:05 -0700 2006"
-		time.RFC822,      // "02 Jan 06 15:04 MST"
-		time.RFC822Z,     // "02 Jan 06 15:04 -0700"
-		time.RFC850,      // "Monday, 02-Jan-06 15:04:05 MST"
-		time.RFC1123,     // "Mon, 02 Jan 2006 15:04:05 MST"
-		time.RFC1123Z,    // "Mon, 02 Jan 2006 15:04:05 -0700"
-		time.RFC3339,     // "2006-01-02T15:04:05Z07:00"
-		time.RFC3339Nano, // "2006-01-02T15:04:05.999999999Z07:00"
-		time.Kitchen,     // "3:04PM"
-
-		// 实用时间戳格式
-		time.Stamp,      // "Jan _2 15:04:05"
-		time.StampMilli, // "Jan _2 15:04:05.000"
-		time.StampMicro, // "Jan _2 15:04:05.000000"
-		time.StampNano,  // "Jan _2 15:04:05.000000000"
-		time.DateTime,   // "2006-01-02 15:04:05"
-		time.DateOnly,   // "2006-01-02"
-		time.TimeOnly,   // "15:04:05"
-
-		// 常用自定义格式
-		"2006-01-02T15:04:05", // 无时区的ISO格式
-		"2006-01-02T15:04:05.000Z",
-		"2006-01-02T15:04:05Z",
-		"2006-01-02 15:04:05.999999999", // 纳秒
-		"2006-01-02 15:04:05.999999",    // 微秒
-		"2006-01-02 15:04:05.999",       // 毫秒
-		"2006/01/02",
-		"20060102",
-		"01/02/2006",
-		"2006年01月02日",
-		"2006年01月02日 15:04:05",
-	}
-
-	var lastErr error
-	for _, format := range formats {
-		t, err := time.Parse(format, ts)
-		if err == nil {
-			return t, nil
-		}
-		lastErr = err
-	}
-
-	// 尝试解析 Unix 时间戳
-	if timestamp, err := strconv.ParseInt(ts, 10, 64); err == nil {
-		switch len(ts) {
-		case 10: // 秒
-			return time.Unix(timestamp, 0), nil
-		case 13: // 毫秒
-			return time.Unix(timestamp/1000, (timestamp%1000)*1000000), nil
-		case 16: // 微秒
-			return time.Unix(timestamp/1000000, (timestamp%1000000)*1000), nil
-		case 19: // 纳秒
-			return time.Unix(timestamp/1000000000, timestamp%1000000000), nil
-		}
-	}
-
-	return time.Time{}, fmt.Errorf("failed to parse time with any format: %v", lastErr)
-}
-
-func (tc *tdengineClient) Query(query interface{}, delay ...int) ([]models.DataResp, error) {
+func (td *TDengine) Query(query interface{}, delay ...int) ([]models.DataResp, error) {
 	b, err := json.Marshal(query)
 	if err != nil {
 		return nil, err
@@ -384,7 +208,7 @@ func (tc *tdengineClient) Query(query interface{}, delay ...int) ([]models.DataR
 		q.Query = strings.ReplaceAll(q.Query, key, val)
 	}
 
-	data, err := tc.QueryTable(q.Query)
+	data, err := td.QueryTable(q.Query)
 	if err != nil {
 		return nil, err
 	}
@@ -393,154 +217,7 @@ func (tc *tdengineClient) Query(query interface{}, delay ...int) ([]models.DataR
 	return ConvertToTStData(data, q.Keys, q.Ref)
 }
 
-// get tdendgine databases
-func (tc *tdengineClient) GetDatabases() ([]string, error) {
-	var databases []string
-	data, err := tc.QueryTable("show databases")
-	if err != nil {
-		return databases, err
-	}
-
-	for _, row := range data.Data {
-		databases = append(databases, row[0].(string))
-	}
-	return databases, nil
-}
-
-// get tdendgine tables by database
-func (tc *tdengineClient) GetTables(database string, isStable bool) ([]string, error) {
-	var tables []string
-	sql := fmt.Sprintf("show %s.tables", database)
-	if isStable {
-		sql = fmt.Sprintf("show %s.stables", database)
-	}
-
-	data, err := tc.QueryTable(sql)
-	if err != nil {
-		return tables, err
-	}
-
-	for _, row := range data.Data {
-		tables = append(tables, row[0].(string))
-	}
-	return tables, nil
-}
-
-type Column struct {
-	Name string `json:"name"`
-	Type string `json:"type"`
-	Size int    `json:"size"`
-}
-
-func (tc *tdengineClient) GetColumns(database, table string) ([]Column, error) {
-	var columns []Column
-	sql := fmt.Sprintf("select * from %s.%s limit 1", database, table)
-	data, err := tc.QueryTable(sql)
-	if err != nil {
-		return columns, err
-	}
-	for _, row := range data.ColumnMeta {
-		var colType string
-		switch t := row[1].(type) {
-		case float64:
-			// v2版本数字类型映射
-			switch int(t) {
-			case 1:
-				colType = "BOOL"
-			case 2:
-				colType = "TINYINT"
-			case 3:
-				colType = "SMALLINT"
-			case 4:
-				colType = "INT"
-			case 5:
-				colType = "BIGINT"
-			case 6:
-				colType = "FLOAT"
-			case 7:
-				colType = "DOUBLE"
-			case 8:
-				colType = "BINARY"
-			case 9:
-				colType = "TIMESTAMP"
-			case 10:
-				colType = "NCHAR"
-			default:
-				colType = "UNKNOWN"
-			}
-		case string:
-			// v3版本直接使用字符串类型
-			colType = t
-		default:
-			logger.Warningf("unexpected column type format: %v", row[1])
-			colType = "UNKNOWN"
-		}
-
-		column := Column{
-			Name: row[0].(string),
-			Type: colType,
-			Size: int(row[2].(float64)),
-		}
-		columns = append(columns, column)
-	}
-
-	return columns, nil
-}
-
-// {
-// 	"code": 0,
-// 	"column_meta": [
-// 	  ["ts", "TIMESTAMP", 8],
-// 	  ["count", "BIGINT", 8],
-// 	  ["endpoint", "VARCHAR", 45],
-// 	  ["status_code", "INT", 4],
-// 	  ["client_ip", "VARCHAR", 40],
-// 	  ["request_method", "VARCHAR", 15],
-// 	  ["request_uri", "VARCHAR", 128]
-// 	],
-// 	"data": [
-// 	  [
-// 		"2022-06-29T05:50:55.401Z",
-// 		2,
-// 		"LAPTOP-NNKFTLTG:6041",
-// 		200,
-// 		"172.23.208.1",
-// 		"POST",
-// 		"/rest/sql"
-// 	  ],
-// 	  [
-// 		"2022-06-29T05:52:16.603Z",
-// 		1,
-// 		"LAPTOP-NNKFTLTG:6041",
-// 		200,
-// 		"172.23.208.1",
-// 		"POST",
-// 		"/rest/sql"
-// 	  ]
-// 	],
-// 	"rows": 2
-//   }
-
-// {
-//     "dat": [
-//         {
-//             "ref": "",
-//             "metric": {
-//                 "__name__": "count",
-//                 "host":"host1"
-//             },
-//             "values": [
-//                 [
-//                     1693219500,
-//                     12
-//                 ]
-//             ]
-//         }
-//     ],
-//     "err": ""
-// }
-
-func ConvertToTStData(src APIResponse, key Keys, ref string) ([]models.DataResp, error) {
+func ConvertToTStData(src td.APIResponse, key Keys, ref string) ([]models.DataResp, error) {
 	metricIdxMap := make(map[string]int)
 	labelIdxMap := make(map[string]int)
 
@@ -702,3 +379,82 @@ func interfaceToFloat64(input interface{}) (float64, error) {
 		return 0, fmt.Errorf("unsupported type: %T", input)
 	}
 }
+
+func parseTimeString(ts string) (time.Time, error) {
+	// 尝试不同的时间格式
+	formats := []string{
+		// 标准格式
+		time.Layout,      // "01/02 03:04:05PM '06 -0700"
+		time.ANSIC,       // "Mon Jan _2 15:04:05 2006"
+		time.UnixDate,    // "Mon Jan _2 15:04:05 MST 2006"
+		time.RubyDate,    // "Mon Jan 02 15:04:05 -0700 2006"
+		time.RFC822,      // "02 Jan 06 15:04 MST"
+		time.RFC822Z,     // "02 Jan 06 15:04 -0700"
+		time.RFC850,      // "Monday, 02-Jan-06 15:04:05 MST"
+		time.RFC1123,     // "Mon, 02 Jan 2006 15:04:05 MST"
+		time.RFC1123Z,    // "Mon, 02 Jan 2006 15:04:05 -0700"
+		time.RFC3339,     // "2006-01-02T15:04:05Z07:00"
+		time.RFC3339Nano, // "2006-01-02T15:04:05.999999999Z07:00"
+		time.Kitchen,     // "3:04PM"
+
+		// 实用时间戳格式
+		time.Stamp,      // "Jan _2 15:04:05"
+		time.StampMilli, // "Jan _2 15:04:05.000"
+		time.StampMicro, // "Jan _2 15:04:05.000000"
+		time.StampNano,  // "Jan _2 15:04:05.000000000"
+		time.DateTime,   // "2006-01-02 15:04:05"
+		time.DateOnly,   // "2006-01-02"
+		time.TimeOnly,   // "15:04:05"
+
+		// 常用自定义格式
+		"2006-01-02T15:04:05", // 无时区的ISO格式
+		"2006-01-02T15:04:05.000Z",
+		"2006-01-02T15:04:05Z",
+		"2006-01-02 15:04:05.999999999", // 纳秒
+		"2006-01-02 15:04:05.999999",    // 微秒
+		"2006-01-02 15:04:05.999",       // 毫秒
+		"2006/01/02",
+		"20060102",
+		"01/02/2006",
+		"2006年01月02日",
+		"2006年01月02日 15:04:05",
+	}
+
+	var lastErr error
+	for _, format := range formats {
+		t, err := time.Parse(format, ts)
+		if err == nil {
+			return t, nil
+		}
+		lastErr = err
+	}
+
+	// 尝试解析 Unix 时间戳
+	if timestamp, err := strconv.ParseInt(ts, 10, 64); err == nil {
+		switch len(ts) {
+		case 10: // 秒
+			return time.Unix(timestamp, 0), nil
+		case 13: // 毫秒
+			return time.Unix(timestamp/1000, (timestamp%1000)*1000000), nil
+		case 16: // 微秒
+			return time.Unix(timestamp/1000000, (timestamp%1000000)*1000), nil
+		case 19: // 纳秒
+			return time.Unix(timestamp/1000000000, timestamp%1000000000), nil
+		}
+	}
+
+	return time.Time{}, fmt.Errorf("failed to parse time with any format: %v", lastErr)
+}
+
+func ConvertToTable(src td.APIResponse) []interface{} {
+	var resp []interface{}
+
+	for i := range src.Data {
+		cur := make(map[string]interface{})
+		for j := range src.Data[i] {
+			cur[src.ColumnMeta[j][0].(string)] = src.Data[i][j]
+		}
+		resp = append(resp, cur)
+	}
+	return resp
+}

docker/compose-bridge/etc-nightingale/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/compose-host-network-metric-log/etc-nightingale/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/compose-host-network/etc-nightingale/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/compose-postgres/initsql_for_postgres/a-n9e-for-Postgres.sql

@@ -790,6 +790,7 @@ CREATE TABLE es_index_pattern (
     time_field varchar(128) not null default '@timestamp',
     allow_hide_system_indices smallint not null default 0,
     fields_format varchar(4096) not null default '',
+    cross_cluster_enabled int not null default 0,
     create_at bigint default '0',
     create_by varchar(64) default '',
     update_at bigint default '0',
@@ -859,6 +860,7 @@ CREATE TABLE builtin_components (
   ident VARCHAR(191) NOT NULL,
   logo VARCHAR(191) NOT NULL,
   readme TEXT NOT NULL,
+  disabled INT NOT NULL DEFAULT 0,
   created_at BIGINT NOT NULL DEFAULT 0,
   created_by VARCHAR(191) NOT NULL DEFAULT '',
   updated_at BIGINT NOT NULL DEFAULT 0,
@@ -885,4 +887,20 @@ CREATE TABLE builtin_payloads (
 CREATE INDEX idx_component ON builtin_payloads (component);
 CREATE INDEX idx_builtin_payloads_name ON builtin_payloads (name);
 CREATE INDEX idx_cate ON builtin_payloads (cate);
-CREATE INDEX idx_type ON builtin_payloads (type);
+CREATE INDEX idx_type ON builtin_payloads (type);
+
+
+CREATE TABLE dash_annotation (
+    id bigserial PRIMARY KEY,
+    dashboard_id bigint not null,
+    panel_id varchar(191) not null,
+    tags text,
+    description text,
+    config text,
+    time_start bigint not null default 0,
+    time_end bigint not null default 0,
+    create_at bigint not null default 0,
+    create_by varchar(64) not null default '',
+    update_at bigint not null default 0,
+    update_by varchar(64) not null default ''
+);

docker/compose-postgres/n9eetc_pg/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

docker/initsql/a-n9e.sql

@@ -531,11 +531,11 @@ CREATE TABLE `builtin_components` (
   `created_by` varchar(191) NOT NULL DEFAULT '' COMMENT '''creator''',
   `updated_at` bigint NOT NULL DEFAULT 0 COMMENT '''update time''',
   `updated_by` varchar(191) NOT NULL DEFAULT '' COMMENT '''updater''',
+  `disabled` int NOT NULL DEFAULT 0 COMMENT '''is disabled or not''',
   PRIMARY KEY (`id`),
-  UNIQUE KEY `idx_ident` (`ident`) USING BTREE
+  UNIQUE KEY `idx_ident` (`ident`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
-
 CREATE TABLE `builtin_payloads` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '''unique identifier''',
   `component_id` bigint NOT NULL DEFAULT 0 COMMENT '''component_id of payload''',
@@ -693,6 +693,7 @@ CREATE TABLE `es_index_pattern` (
     `time_field` varchar(128) not null default '@timestamp',
     `allow_hide_system_indices` tinyint(1) not null default 0,
     `fields_format` varchar(4096) not null default '',
+    `cross_cluster_enabled` int not null default 0,
     `create_at` bigint default '0',
     `create_by` varchar(64) default '',
     `update_at` bigint default '0',
@@ -745,6 +746,24 @@ CREATE TABLE `target_busi_group` (
   UNIQUE KEY `idx_target_group` (`target_ident`,`group_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
+
+CREATE TABLE `dash_annotation` (
+    `id` bigint unsigned not null auto_increment,
+    `dashboard_id` bigint not null comment 'dashboard id',
+    `panel_id` varchar(191) not null comment 'panel id',
+    `tags` text comment 'tags array json string',
+    `description` text comment 'annotation description',
+    `config` text comment 'annotation config',
+    `time_start` bigint not null default 0 comment 'start timestamp',
+    `time_end` bigint not null default 0 comment 'end timestamp',
+    `create_at` bigint not null default 0 comment 'create time',
+    `create_by` varchar(64) not null default '' comment 'creator',
+    `update_at` bigint not null default 0 comment 'update time',
+    `update_by` varchar(64) not null default '' comment 'updater',
+    PRIMARY KEY (`id`),
+    KEY `idx_dashboard_id` (`dashboard_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 CREATE TABLE `task_meta`
 (
     `id`          bigint unsigned NOT NULL AUTO_INCREMENT,

docker/migratesql/migrate.sql

@@ -120,4 +120,31 @@ CREATE TABLE `target_busi_group` (
 
 /* v7.7.2 2024-12-02 */
 ALTER TABLE alert_subscribe MODIFY COLUMN rule_ids varchar(1024);
-ALTER TABLE alert_subscribe MODIFY COLUMN busi_groups varchar(4096);
+ALTER TABLE alert_subscribe MODIFY COLUMN busi_groups varchar(4096);
+
+/* v8.0.0-beta.1 2024-12-13 */
+ALTER TABLE `alert_rule` ADD COLUMN `cron_pattern` VARCHAR(64);
+ALTER TABLE `builtin_components` MODIFY COLUMN `logo` mediumtext COMMENT '''logo of component''';
+
+/* v8.0.0-beta.2 2024-12-26 */
+ALTER TABLE `es_index_pattern` ADD COLUMN `cross_cluster_enabled` int not null default 0;
+
+/* v8.0.0-beta.3 2024-01-03 */
+ALTER TABLE `builtin_components` ADD COLUMN `disabled` INT NOT NULL DEFAULT 0 COMMENT 'is disabled or not';
+        
+CREATE TABLE `dash_annotation` (
+    `id` bigint unsigned not null auto_increment,
+    `dashboard_id` bigint not null comment 'dashboard id',
+    `panel_id` varchar(191) not null comment 'panel id',
+    `tags` text comment 'tags array json string',
+    `description` text comment 'annotation description',
+    `config` text comment 'annotation config',
+    `time_start` bigint not null default 0 comment 'start timestamp',
+    `time_end` bigint not null default 0 comment 'end timestamp',
+    `create_at` bigint not null default 0 comment 'create time',
+    `create_by` varchar(64) not null default '' comment 'creator',
+    `update_at` bigint not null default 0 comment 'update time',
+    `update_by` varchar(64) not null default '' comment 'updater',
+    PRIMARY KEY (`id`),
+    KEY `idx_dashboard_id` (`dashboard_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

docker/sqlite.sql

@@ -17,6 +17,8 @@ CREATE TABLE `users` (
     `update_by` varchar(64) not null default ''
 );
 
+CREATE UNIQUE INDEX idx_users_username ON `users` (username);
+
 insert into `users`(id, username, nickname, password, roles, create_at, create_by, update_at, update_by) values(1, 'root', '超管', 'root.2020', 'Admin', strftime('%s', 'now'), 'system', strftime('%s', 'now'), 'system');
 
 CREATE TABLE `user_group` (
@@ -182,8 +184,9 @@ CREATE TABLE `board` (
     `create_by` varchar(64) not null default '',
     `update_at` bigint not null default 0,
     `update_by` varchar(64) not null default '',
-    unique (`group_id`, `name`)
+    `public_cate` bigint not null default 0
 );
+CREATE UNIQUE INDEX idx_board_group_id_name ON `board` (group_id, name);
 CREATE INDEX `idx_board_ident` ON `board` (`ident` asc);
 
 -- for dashboard new version
@@ -192,6 +195,15 @@ CREATE TABLE `board_payload` (
     `payload` mediumtext not null
 );
 
+CREATE TABLE `chart` (
+    `id` integer primary key autoincrement,
+    `group_id` integer not null,
+    `configs` text,
+    `weight` integer not null default 0
+);
+
+CREATE INDEX idx_chart_group_id ON `chart` (group_id);
+
 CREATE TABLE `chart_share` (
     `id` integer primary key autoincrement,
     `cluster` varchar(128) not null,
@@ -238,7 +250,9 @@ CREATE TABLE `alert_rule` (
     `create_at` bigint not null default 0,
     `create_by` varchar(64) not null default '',
     `update_at` bigint not null default 0,
-    `update_by` varchar(64) not null default ''
+    `update_by` varchar(64) not null default '',
+    `cron_pattern` varchar(64),
+    `datasource_queries` text
 );
 CREATE INDEX `idx_alert_rule_group_id` ON `alert_rule` (`group_id` asc);
 CREATE INDEX `idx_alert_rule_update_at` ON `alert_rule` (`update_at` asc);
@@ -308,11 +322,18 @@ CREATE TABLE `target` (
     `tags` varchar(512) not null default '',
     `host_ip` varchar(15) default '',
     `agent_version` varchar(255) default '',
+    `host_tags` text,
     `engine_name` varchar(255) default '',
+    `os` varchar(31) default '',
     `update_at` bigint not null default 0
 );
-CREATE INDEX `idx_target_group_id` ON `target` (`group_id` asc);
 
+CREATE INDEX `idx_target_group_id` ON `target` (`group_id` asc);
+CREATE UNIQUE INDEX idx_target_ident ON `target` (ident);
+CREATE INDEX idx_host_ip ON `target` (host_ip);
+CREATE INDEX idx_agent_version ON `target` (agent_version);
+CREATE INDEX idx_engine_name ON `target` (engine_name);
+CREATE INDEX idx_os ON `target` (os);
 
 CREATE TABLE `metric_view` (
     `id` integer primary key autoincrement,
@@ -337,12 +358,14 @@ CREATE TABLE `recording_rule` (
     `disabled` tinyint(1) not null default 0,
     `prom_ql` varchar(8192) not null,
     `prom_eval_interval` int not null,
+    `cron_pattern` varchar(255) default '',
     `append_tags` varchar(255) default '',
     `query_configs` text not null,
     `create_at` bigint default '0',
     `create_by` varchar(64) default '',
     `update_at` bigint default '0',
-    `update_by` varchar(64) default ''
+    `update_by` varchar(64) default '',
+    `datasource_queries` text
 );
 CREATE INDEX `idx_recording_rule_group_id` ON `recording_rule` (`group_id` asc);
 CREATE INDEX `idx_recording_rule_update_at` ON `recording_rule` (`update_at` asc);
@@ -430,6 +453,7 @@ CREATE TABLE `alert_his_event` (
     `trigger_value` varchar(2048) not null,
     `recover_time` bigint not null default 0,
     `last_eval_time` bigint not null default 0,
+    `original_tags` varchar(8192),
     `tags` varchar(1024) not null default '',
     `annotations` text not null,
     `rule_config` text not null
@@ -459,6 +483,8 @@ CREATE INDEX `idx_builtin_components_ident` ON `builtin_components` (`ident` asc
 
 CREATE TABLE `builtin_payloads` (
   `id` integer primary key autoincrement,
+  `component_id` integer not null default 0,
+  `uuid` integer not null,
   `type` varchar(191) not null,
   `component` varchar(191) not null,
   `cate` varchar(191) not null,
@@ -474,6 +500,20 @@ CREATE INDEX `idx_builtin_payloads_component` ON `builtin_payloads` (`component`
 CREATE INDEX `idx_builtin_payloads_name` ON `builtin_payloads` (`name` asc);
 CREATE INDEX `idx_builtin_payloads_cate` ON `builtin_payloads` (`cate` asc);
 CREATE INDEX `idx_builtin_payloads_type` ON `builtin_payloads` (`type` asc);
+CREATE INDEX idx_uuid ON `builtin_payloads` (uuid);
+
+
+CREATE TABLE `notification_record` (
+    `id` integer primary key autoincrement,
+    `event_id` integer not null,
+    `sub_id` integer,
+    `channel` varchar(255) not null,
+    `status` integer,
+    `target` varchar(1024) not null,
+    `details` varchar(2048) default '',
+    `created_at` integer not null
+);
+CREATE INDEX idx_evt ON notification_record (event_id);
 
 CREATE TABLE `task_tpl` (
     `id`        integer primary key autoincrement,
@@ -553,6 +593,8 @@ CREATE TABLE `datasource`
     `updated_by` varchar(64) not null default ''
 );
 
+CREATE UNIQUE INDEX idx_datasource_name ON datasource (name);
+
 CREATE TABLE `builtin_cate` (
     `id` integer primary key autoincrement,
     `name` varchar(191) not null,
@@ -570,6 +612,8 @@ CREATE TABLE `notify_tpl` (
     `update_by` varchar(64) not null default ''
 );
 
+CREATE UNIQUE INDEX idx_notify_tpl_channel ON notify_tpl (channel);
+
 CREATE TABLE `sso_config` (
     `id` integer primary key autoincrement,
     `name` varchar(191) not null unique,
@@ -577,6 +621,8 @@ CREATE TABLE `sso_config` (
     `update_at` bigint not null default 0
 );
 
+CREATE UNIQUE INDEX idx_sso_config_name ON sso_config (name);
+
 CREATE TABLE `es_index_pattern` (
     `id` integer primary key autoincrement,
     `datasource_id` bigint not null default 0,
@@ -584,6 +630,7 @@ CREATE TABLE `es_index_pattern` (
     `time_field` varchar(128) not null default '@timestamp',
     `allow_hide_system_indices` tinyint(1) not null default 0,
     `fields_format` varchar(4096) not null default '',
+    `cross_cluster_enabled` int not null default 0,
     `create_at` bigint default '0',
     `create_by` varchar(64) default '',
     `update_at` bigint default '0',
@@ -591,6 +638,8 @@ CREATE TABLE `es_index_pattern` (
     unique (`datasource_id`, `name`)
 );
 
+CREATE UNIQUE INDEX idx_es_index_pattern_datasource_id_name ON es_index_pattern (datasource_id, name);
+
 CREATE TABLE `builtin_metrics` (
     `id` integer primary key autoincrement,
     `collector` varchar(191) NOT NULL,
@@ -603,13 +652,15 @@ CREATE TABLE `builtin_metrics` (
     `created_at` bigint NOT NULL DEFAULT 0,
     `created_by` varchar(191) NOT NULL DEFAULT '',
     `updated_at` bigint NOT NULL DEFAULT 0,
-    `updated_by` varchar(191) NOT NULL DEFAULT ''
+    `updated_by` varchar(191) NOT NULL DEFAULT '',
+    `uuid integer` not null default 0
 );
--- CREATE UNIQUE INDEX `idx_builtin_metrics_collector_typ_name` ON `builtin_metrics` (`lang`,`collector`, `typ`, `name` asc);
--- CREATE INDEX `idx_builtin_metrics_collector` ON `builtin_metrics` (`collector` asc);
--- CREATE INDEX `idx_builtin_metrics_typ` ON `builtin_metrics` (`typ` asc);
--- CREATE INDEX `idx_builtin_metrics_name` ON `builtin_metrics` (`name` asc);
--- CREATE INDEX `idx_builtin_metrics_lang` ON `builtin_metrics` (`lang` asc);
+
+CREATE UNIQUE INDEX idx_collector_typ_name ON builtin_metrics (lang, collector, typ, name);
+CREATE INDEX idx_collector ON builtin_metrics (collector);
+CREATE INDEX idx_typ ON builtin_metrics (typ);
+CREATE INDEX idx_builtinmetric_name ON builtin_metrics (name);
+CREATE INDEX idx_lang ON builtin_metrics (lang);
 
 
 CREATE TABLE `metric_filter` (
@@ -624,6 +675,30 @@ CREATE TABLE `metric_filter` (
 );
 CREATE INDEX `idx_metric_filter_name` ON `metric_filter` (`name` asc);
 
+CREATE TABLE `target_busi_group` (
+    `id` integer primary key autoincrement,
+    `target_ident` varchar(191) not null,
+    `group_id` integer not null,
+    `update_at` integer not null
+);
+
+CREATE UNIQUE INDEX idx_target_busi_group ON target_busi_group (target_ident, group_id);
+
+
+CREATE TABLE `dash_annotation` (
+    `id` integer primary key autoincrement,
+    `dashboard_id` bigint not null,
+    `panel_id` varchar(191) not null,
+    `tags` text,
+    `description` text,
+    `config` text,
+    `time_start` bigint not null default 0,
+    `time_end` bigint not null default 0,
+    `create_at` bigint not null default 0,
+    `create_by` varchar(64) not null default '',
+    `update_at` bigint not null default 0,
+    `update_by` varchar(64) not null default ''
+);
 
 CREATE TABLE `task_meta`
 (

dscache/cache.go

@@ -0,0 +1,59 @@
+package dscache
+
+import (
+	"sync"
+
+	"github.com/ccfos/nightingale/v6/datasource"
+	"github.com/toolkits/pkg/logger"
+)
+
+type Cache struct {
+	datas map[string]map[int64]datasource.Datasource
+	mutex *sync.RWMutex
+}
+
+var DsCache = Cache{
+	datas: make(map[string]map[int64]datasource.Datasource),
+	mutex: new(sync.RWMutex),
+}
+
+func (cs *Cache) Put(cate string, dsId int64, ds datasource.Datasource) {
+	cs.mutex.Lock()
+	if _, found := cs.datas[cate]; !found {
+		cs.datas[cate] = make(map[int64]datasource.Datasource)
+	}
+
+	if _, found := cs.datas[cate][dsId]; found {
+		if cs.datas[cate][dsId].Equal(ds) {
+			cs.mutex.Unlock()
+			return
+		}
+	}
+	cs.mutex.Unlock()
+
+	// InitClient() 在用户配置错误或远端不可用时, 会非常耗时, mutex被长期持有, 导致Get()会超时
+	err := ds.InitClient()
+	if err != nil {
+		logger.Errorf("init plugin:%s %d %+v client fail: %v", cate, dsId, ds, err)
+		return
+	}
+
+	logger.Debugf("init plugin:%s %d %+v client success", cate, dsId, ds)
+	cs.mutex.Lock()
+	cs.datas[cate][dsId] = ds
+	cs.mutex.Unlock()
+}
+
+func (cs *Cache) Get(cate string, dsId int64) (datasource.Datasource, bool) {
+	cs.mutex.RLock()
+	defer cs.mutex.RUnlock()
+	if _, found := cs.datas[cate]; !found {
+		return nil, false
+	}
+
+	if _, found := cs.datas[cate][dsId]; !found {
+		return nil, false
+	}
+
+	return cs.datas[cate][dsId], true
+}

dscache/sync.go

@@ -0,0 +1,190 @@
+package dscache
+
+import (
+	"context"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/datasource"
+	_ "github.com/ccfos/nightingale/v6/datasource/ck"
+	"github.com/ccfos/nightingale/v6/datasource/es"
+	"github.com/ccfos/nightingale/v6/dskit/tdengine"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
+
+	"github.com/toolkits/pkg/logger"
+)
+
+var FromAPIHook func()
+
+func Init(ctx *ctx.Context, fromAPI bool) {
+	go getDatasourcesFromDBLoop(ctx, fromAPI)
+}
+
+type ListInput struct {
+	Page       int    `json:"p"`
+	Limit      int    `json:"limit"`
+	Category   string `json:"category"`
+	PluginType string `json:"plugin_type"` // promethues
+	Status     string `json:"status"`
+}
+
+type DSReply struct {
+	RequestID string `json:"request_id"`
+	Data      struct {
+		Items []datasource.DatasourceInfo `json:"items"`
+	} `json:"data"`
+}
+
+type DSReplyEncrypt struct {
+	RequestID string `json:"request_id"`
+	Data      string `json:"data"`
+}
+
+var PromDefaultDatasourceId int64
+
+func getDatasourcesFromDBLoop(ctx *ctx.Context, fromAPI bool) {
+	for {
+		if !fromAPI {
+			items, err := models.GetDatasources(ctx)
+			if err != nil {
+				logger.Errorf("get datasource from database fail: %v", err)
+				//stat.CounterExternalErrorTotal.WithLabelValues("db", "get_cluster").Inc()
+				time.Sleep(time.Second * 2)
+				continue
+			}
+			var dss []datasource.DatasourceInfo
+			for _, item := range items {
+				if item.PluginType == "prometheus" && item.IsDefault {
+					atomic.StoreInt64(&PromDefaultDatasourceId, item.Id)
+				}
+
+				logger.Debugf("get datasource: %+v", item)
+				ds := datasource.DatasourceInfo{
+					Id:             item.Id,
+					Name:           item.Name,
+					Description:    item.Description,
+					Category:       item.Category,
+					PluginId:       item.PluginId,
+					Type:           item.PluginType,
+					PluginTypeName: item.PluginTypeName,
+					Settings:       item.SettingsJson,
+					HTTPJson:       item.HTTPJson,
+					AuthJson:       item.AuthJson,
+					Status:         item.Status,
+					IsDefault:      item.IsDefault,
+				}
+
+				if item.PluginType == "elasticsearch" {
+					esN9eToDatasourceInfo(&ds, item)
+				} else if item.PluginType == "opensearch" {
+					osN9eToDatasourceInfo(&ds, item)
+				} else if item.PluginType == "tdengine" {
+					tdN9eToDatasourceInfo(&ds, item)
+				} else {
+					ds.Settings = make(map[string]interface{})
+					for k, v := range item.SettingsJson {
+						ds.Settings[k] = v
+					}
+				}
+				dss = append(dss, ds)
+			}
+			PutDatasources(dss)
+		} else {
+			FromAPIHook()
+		}
+
+		time.Sleep(time.Second * 2)
+	}
+}
+
+func tdN9eToDatasourceInfo(ds *datasource.DatasourceInfo, item models.Datasource) {
+	ds.Settings = make(map[string]interface{})
+	ds.Settings["tdengine.cluster_name"] = item.Name
+	ds.Settings["tdengine.addr"] = item.HTTPJson.Url
+	ds.Settings["tdengine.timeout"] = item.HTTPJson.Timeout
+	ds.Settings["tdengine.dial_timeout"] = item.HTTPJson.DialTimeout
+	ds.Settings["tdengine.max_idle_conns_per_host"] = item.HTTPJson.MaxIdleConnsPerHost
+	ds.Settings["tdengine.headers"] = item.HTTPJson.Headers
+	ds.Settings["tdengine.basic"] = tdengine.TDengineBasicAuth{
+		User:     item.AuthJson.BasicAuthUser,
+		Password: item.AuthJson.BasicAuthPassword,
+	}
+}
+
+func esN9eToDatasourceInfo(ds *datasource.DatasourceInfo, item models.Datasource) {
+	ds.Settings = make(map[string]interface{})
+	ds.Settings["es.nodes"] = []string{item.HTTPJson.Url}
+	if len(item.HTTPJson.Urls) > 0 {
+		ds.Settings["es.nodes"] = item.HTTPJson.Urls
+	}
+	ds.Settings["es.timeout"] = item.HTTPJson.Timeout
+	ds.Settings["es.basic"] = es.BasicAuth{
+		Username: item.AuthJson.BasicAuthUser,
+		Password: item.AuthJson.BasicAuthPassword,
+	}
+	ds.Settings["es.tls"] = es.TLS{
+		SkipTlsVerify: item.HTTPJson.TLS.SkipTlsVerify,
+	}
+	ds.Settings["es.version"] = item.SettingsJson["version"]
+	ds.Settings["es.headers"] = item.HTTPJson.Headers
+	ds.Settings["es.min_interval"] = item.SettingsJson["min_interval"]
+	ds.Settings["es.max_shard"] = item.SettingsJson["max_shard"]
+	ds.Settings["es.enable_write"] = item.SettingsJson["enable_write"]
+}
+
+// for opensearch
+func osN9eToDatasourceInfo(ds *datasource.DatasourceInfo, item models.Datasource) {
+	ds.Settings = make(map[string]interface{})
+	ds.Settings["os.nodes"] = []string{item.HTTPJson.Url}
+	ds.Settings["os.timeout"] = item.HTTPJson.Timeout
+	ds.Settings["os.basic"] = es.BasicAuth{
+		Username: item.AuthJson.BasicAuthUser,
+		Password: item.AuthJson.BasicAuthPassword,
+	}
+	ds.Settings["os.tls"] = es.TLS{
+		SkipTlsVerify: item.HTTPJson.TLS.SkipTlsVerify,
+	}
+	ds.Settings["os.version"] = item.SettingsJson["version"]
+	ds.Settings["os.headers"] = item.HTTPJson.Headers
+	ds.Settings["os.min_interval"] = item.SettingsJson["min_interval"]
+	ds.Settings["os.max_shard"] = item.SettingsJson["max_shard"]
+}
+
+func PutDatasources(items []datasource.DatasourceInfo) {
+	ids := make([]int64, 0)
+	for _, item := range items {
+		if item.Type == "prometheus" {
+			continue
+		}
+
+		if item.Type == "loki" {
+			continue
+		}
+
+		if item.Name == "" {
+			logger.Warningf("cluster name is empty, ignore %+v", item)
+			continue
+		}
+		typ := strings.ReplaceAll(item.Type, ".logging", "")
+
+		ds, err := datasource.GetDatasourceByType(typ, item.Settings)
+		if err != nil {
+			logger.Warningf("get plugin:%+v fail: %v", item, err)
+			continue
+		}
+
+		err = ds.Validate(context.Background())
+		if err != nil {
+			logger.Warningf("get plugin:%+v fail: %v", item, err)
+			continue
+		}
+		ids = append(ids, item.Id)
+
+		// 异步初始化 client 不然数据源同步的会很慢
+		go DsCache.Put(typ, item.Id, ds)
+	}
+
+	logger.Debugf("get plugin by type success Ids:%v", ids)
+}

dskit/clickhouse/clickhouse.go

@@ -0,0 +1,303 @@
+package clickhouse
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/dskit/sqlbase"
+	"github.com/ccfos/nightingale/v6/dskit/types"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/mitchellh/mapstructure"
+	"github.com/toolkits/pkg/net/httplib"
+	ckDriver "gorm.io/driver/clickhouse"
+	"gorm.io/gorm"
+)
+
+const (
+	ckDataSource = "clickhouse://%s:%s@%s?read_timeout=10s"
+
+	DefaultLimit = 500
+)
+
+type Clickhouse struct {
+	Nodes        []string `json:"ck.nodes" mapstructure:"ck.nodes"`
+	User         string   `json:"ck.user" mapstructure:"ck.user"`
+	Password     string   `json:"ck.password" mapstructure:"ck.password"`
+	Timeout      int      `json:"ck.timeout" mapstructure:"ck.timeout"`
+	MaxQueryRows int      `json:"ck.max_query_rows" mapstructure:"ck.max_query_rows"`
+
+	Client       *gorm.DB `json:"-"`
+	ClientByHTTP *sql.DB  `json:"-"`
+}
+
+func (c *Clickhouse) InitCli() error {
+	if c.MaxQueryRows == 0 {
+		c.MaxQueryRows = DefaultLimit
+	}
+
+	if len(c.Nodes) == 0 {
+		return fmt.Errorf("not found ck shard, please check datasource config")
+	}
+	addr := c.Nodes[0]
+	url := addr
+	if !strings.HasPrefix(url, "http://") {
+		url = "http://" + url
+	}
+	resp, err := httplib.Get(url).SetTimeout(time.Second * 1).Response()
+	// 忽略HTTP Code错误, 因为可能不是HTTP协议
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	// HTTP 协议
+	if resp.StatusCode == 200 {
+		jsonBytes, _ := io.ReadAll(resp.Body)
+		if len(jsonBytes) > 0 && strings.Contains(strings.ToLower(string(jsonBytes)), "ok.") {
+			ckconn := clickhouse.OpenDB(&clickhouse.Options{
+				Addr: []string{addr},
+				Auth: clickhouse.Auth{
+					Username: c.User,
+					Password: c.Password,
+				},
+				Settings: clickhouse.Settings{
+					"max_execution_time": 60,
+				},
+				DialTimeout: 10 * time.Second,
+				Protocol:    clickhouse.HTTP,
+			})
+			if ckconn == nil {
+				return errors.New("db conn failed")
+			}
+			c.ClientByHTTP = ckconn
+			return nil
+		}
+	}
+
+	db, err := gorm.Open(
+		ckDriver.New(
+			ckDriver.Config{
+				DSN: fmt.Sprintf(ckDataSource,
+					c.User, c.Password, addr),
+				DisableDatetimePrecision:  true,
+				DontSupportRenameColumn:   true,
+				SkipInitializeWithVersion: false,
+			}),
+	)
+	if err != nil {
+		return err
+	}
+
+	c.Client = db
+
+	return nil
+}
+
+const (
+	ShowDatabases = "SHOW DATABASES"
+	ShowTables    = "SELECT name FROM system.tables WHERE database = '%s'"
+	DescTable     = "SELECT name,type FROM system.columns WHERE database='%s' AND table = '%s';"
+)
+
+func (c *Clickhouse) QueryRows(ctx context.Context, query string) (*sql.Rows, error) {
+	var (
+		rows *sql.Rows
+		err  error
+	)
+
+	if c.ClientByHTTP != nil {
+		rows, err = c.ClientByHTTP.Query(query)
+		if err != nil {
+			return nil, err
+		}
+	} else if c.Client != nil {
+		rows, err = c.Client.Raw(query).Rows()
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, fmt.Errorf("clickhouse client is nil")
+	}
+
+	return rows, nil
+}
+
+// ShowDatabases lists all databases in Clickhouse
+func (c *Clickhouse) ShowDatabases(ctx context.Context) ([]string, error) {
+	var (
+		res []string
+	)
+
+	rows, err := c.QueryRows(ctx, ShowDatabases)
+	if err != nil {
+		return nil, err
+	}
+
+	for rows.Next() {
+		var r string
+		if err := rows.Scan(&r); err != nil {
+			return nil, err
+		}
+		res = append(res, r)
+	}
+
+	return res, nil
+}
+
+// ShowTables lists all tables in a given database
+func (c *Clickhouse) ShowTables(ctx context.Context, database string) ([]string, error) {
+	var (
+		res []string
+	)
+
+	showTables := fmt.Sprintf(ShowTables, database)
+	rows, err := c.QueryRows(ctx, showTables)
+	if err != nil {
+		return nil, err
+	}
+
+	for rows.Next() {
+		var r string
+		if err := rows.Scan(&r); err != nil {
+			return nil, err
+		}
+		res = append(res, r)
+	}
+
+	return res, nil
+}
+
+// DescribeTable describes the schema of a specified table in Clickhouse
+func (c *Clickhouse) DescribeTable(ctx context.Context, query interface{}) ([]*types.ColumnProperty, error) {
+	var (
+		ret []*types.ColumnProperty
+	)
+
+	ckQueryParam := new(QueryParam)
+	if err := mapstructure.Decode(query, ckQueryParam); err != nil {
+		return nil, err
+	}
+	descTable := fmt.Sprintf(DescTable, ckQueryParam.Database, ckQueryParam.Table)
+
+	rows, err := c.QueryRows(ctx, descTable)
+	if err != nil {
+		return nil, err
+	}
+
+	for rows.Next() {
+		var column types.ColumnProperty
+		if err := rows.Scan(&column.Field, &column.Type); err != nil {
+			return nil, err
+		}
+		ret = append(ret, &column)
+	}
+
+	return ret, nil
+}
+
+func (c *Clickhouse) ExecQueryBySqlDB(ctx context.Context, sql string) ([]map[string]interface{}, error) {
+	rows, err := c.QueryRows(ctx, sql)
+
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	columns, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+
+	var results []map[string]interface{}
+
+	for rows.Next() {
+		columnValues := make([]interface{}, len(columns))
+		columnPointers := make([]interface{}, len(columns))
+		for i := range columnValues {
+			columnPointers[i] = &columnValues[i]
+		}
+
+		if err := rows.Scan(columnPointers...); err != nil {
+			continue
+		}
+
+		rowMap := make(map[string]interface{})
+		for i, colName := range columns {
+			val := columnValues[i]
+			bytes, ok := val.([]byte)
+			if ok {
+				rowMap[colName] = string(bytes)
+			} else {
+				rowMap[colName] = val
+			}
+		}
+		results = append(results, rowMap)
+	}
+	return results, nil
+}
+
+func (c *Clickhouse) Query(ctx context.Context, query interface{}) ([]map[string]interface{}, error) {
+
+	ckQuery := new(QueryParam)
+	if err := mapstructure.Decode(query, ckQuery); err != nil {
+		return nil, err
+	}
+
+	// 校验SQL的合法性, 过滤掉 write请求
+	sqlItem := strings.Split(strings.ToUpper(ckQuery.Sql), " ")
+	for _, item := range sqlItem {
+		if _, ok := ckBannedOp[item]; ok {
+			return nil, fmt.Errorf("operation %s is forbid, only read db, please check your sql", item)
+		}
+	}
+
+	// 检查匹配数据长度，防止数据量过大
+	err := c.CheckMaxQueryRows(ctx, ckQuery.Sql)
+	if err != nil {
+		return nil, err
+	}
+
+	dbRows := make([]map[string]interface{}, 0)
+
+	if c.ClientByHTTP != nil {
+		dbRows, err = c.ExecQueryBySqlDB(ctx, ckQuery.Sql)
+	} else {
+		err = c.Client.Raw(ckQuery.Sql).Find(&dbRows).Error
+	}
+	if err != nil {
+		return nil, fmt.Errorf("fetch data failed, sql is %s, err is %s", ckQuery.Sql, err.Error())
+	}
+
+	return dbRows, nil
+}
+
+func (c *Clickhouse) CheckMaxQueryRows(ctx context.Context, sql string) error {
+
+	subSql := strings.ReplaceAll(sql, ";", "")
+	subSql = fmt.Sprintf("SELECT COUNT(*) as count FROM (%s) AS subquery;", subSql)
+
+	dbRows, err := c.ExecQueryBySqlDB(ctx, subSql)
+	if err != nil {
+		return fmt.Errorf("fetch data failed, sql is %s, err is %s", subSql, err.Error())
+	}
+
+	if len(dbRows) > 0 {
+		if count, exists := dbRows[0]["count"]; exists {
+			v, err := sqlbase.ParseFloat64Value(count)
+			if err != nil {
+				return err
+			}
+
+			if v > float64(c.MaxQueryRows) {
+				return fmt.Errorf("query result rows count %d exceeds the maximum limit %d", int(v), c.MaxQueryRows)
+			}
+		}
+	}
+
+	return nil
+}

dskit/clickhouse/clickhouse_test.go

@@ -0,0 +1,43 @@
+package clickhouse
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/dskit/types"
+)
+
+func Test_Timeseries(t *testing.T) {
+	ck := &Clickhouse{
+		Nodes:    []string{"127.0.0.1:8123"},
+		User:     "default",
+		Password: "123456",
+	}
+
+	err := ck.InitCli()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data, err := ck.QueryTimeseries(context.TODO(), &QueryParam{
+		Sql:        `select * from default.student limit 20`,
+		From:       time.Now().Unix() - 300,
+		To:         time.Now().Unix(),
+		TimeField:  "created_at",
+		TimeFormat: "datetime",
+		Keys: types.Keys{
+			LabelKey: "age",
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	bs, err := json.Marshal(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(string(bs))
+}

dskit/clickhouse/timeseries.go

@@ -0,0 +1,58 @@
+package clickhouse
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ccfos/nightingale/v6/dskit/sqlbase"
+	"github.com/ccfos/nightingale/v6/dskit/types"
+)
+
+const (
+	TimeFieldFormatEpochMilli  = "epoch_millis"
+	TimeFieldFormatEpochSecond = "epoch_second"
+)
+
+// 时序数据相关的API
+type QueryParam struct {
+	Limit      int        `json:"limit" mapstructure:"limit"`
+	Sql        string     `json:"sql" mapstructure:"sql"`
+	Ref        string     `json:"ref" mapstructure:"ref"`
+	From       int64      `json:"from" mapstructure:"from"`
+	To         int64      `json:"to" mapstructure:"to"`
+	TimeField  string     `json:"time_field" mapstructure:"time_field"`
+	TimeFormat string     `json:"time_format" mapstructure:"time_format"`
+	Keys       types.Keys `json:"keys" mapstructure:"keys"`
+	Database   string     `json:"database" mapstructure:"database"`
+	Table      string     `json:"table" mapstructure:"table"`
+}
+
+var (
+	ckBannedOp = map[string]struct{}{
+		"CREATE":   {},
+		"INSERT":   {},
+		"ALTER":    {},
+		"REVOKE":   {},
+		"DROP":     {},
+		"RENAME":   {},
+		"ATTACH":   {},
+		"DETACH":   {},
+		"OPTIMIZE": {},
+		"TRUNCATE": {},
+		"SET":      {},
+	}
+)
+
+func (c *Clickhouse) QueryTimeseries(ctx context.Context, query *QueryParam) ([]types.MetricValues, error) {
+	if query.Keys.ValueKey == "" {
+		return nil, fmt.Errorf("valueKey is required")
+	}
+
+	rows, err := c.Query(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+
+	// 构造成时续数据
+	return sqlbase.FormatMetricValues(query.Keys, rows, true), nil
+}

dskit/sqlbase/base.go

@@ -0,0 +1,236 @@
+// @Author: Ciusyan 5/19/24
+
+package sqlbase
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/dskit/types"
+
+	"gorm.io/gorm"
+)
+
+// NewDB creates a new Gorm DB instance based on the provided gorm.Dialector and configures the connection pool
+func NewDB(ctx context.Context, dialector gorm.Dialector, maxIdleConns, maxOpenConns int, connMaxLifetime time.Duration) (*gorm.DB, error) {
+	// Create a new Gorm DB instance
+	db, err := gorm.Open(dialector, &gorm.Config{})
+	if err != nil {
+		return nil, err
+	}
+
+	// Configure the connection pool
+	sqlDB, err := db.DB()
+	if err != nil {
+		return nil, err
+	}
+
+	sqlDB.SetMaxIdleConns(maxIdleConns)
+	sqlDB.SetMaxOpenConns(maxOpenConns)
+	sqlDB.SetConnMaxLifetime(connMaxLifetime)
+
+	return db.WithContext(ctx), sqlDB.Ping()
+}
+
+// ShowTables retrieves a list of all tables in the specified database
+func ShowTables(ctx context.Context, db *gorm.DB, query string) ([]string, error) {
+	var tables []string
+
+	rows, err := db.WithContext(ctx).Raw(query).Rows()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var table string
+		if err := rows.Scan(&table); err != nil {
+			return nil, err
+		}
+		tables = append(tables, table)
+	}
+
+	return tables, nil
+}
+
+// ShowDatabases retrieves a list of all databases in the connected database server
+func ShowDatabases(ctx context.Context, db *gorm.DB, query string) ([]string, error) {
+	var databases []string
+
+	rows, err := db.WithContext(ctx).Raw(query).Rows()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var database string
+		if err := rows.Scan(&database); err != nil {
+			return nil, err
+		}
+		databases = append(databases, database)
+	}
+
+	return databases, nil
+}
+
+// DescTable describes the schema of a specified table in MySQL or PostgreSQL
+func DescTable(ctx context.Context, db *gorm.DB, query string) ([]*types.ColumnProperty, error) {
+	rows, err := db.WithContext(ctx).Raw(query).Rows()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var columns []*types.ColumnProperty
+	for rows.Next() {
+		var (
+			field        string
+			typ          string
+			null         string
+			key          sql.NullString
+			defaultValue sql.NullString
+			extra        sql.NullString
+		)
+
+		switch db.Dialector.Name() {
+		case "mysql":
+			if err := rows.Scan(&field, &typ, &null, &key, &defaultValue, &extra); err != nil {
+				continue
+			}
+		case "postgres", "sqlserver":
+			if err := rows.Scan(&field, &typ, &null, &defaultValue); err != nil {
+				continue
+			}
+		case "oracle":
+			if err := rows.Scan(&field, &typ, &null); err != nil {
+				continue
+			}
+		}
+
+		// Convert the database-specific type to internal type
+		type2, indexable := convertDBType(db.Dialector.Name(), typ)
+		columns = append(columns, &types.ColumnProperty{
+			Field:     field,
+			Type:      typ,
+			Type2:     type2,
+			Indexable: indexable,
+		})
+	}
+	return columns, nil
+}
+
+// ExecQuery executes the specified query and returns the result rows
+func ExecQuery(ctx context.Context, db *gorm.DB, sql string) ([]map[string]interface{}, error) {
+	rows, err := db.WithContext(ctx).Raw(sql).Rows()
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	columns, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+
+	var results []map[string]interface{}
+
+	for rows.Next() {
+		columnValues := make([]interface{}, len(columns))
+		columnPointers := make([]interface{}, len(columns))
+		for i := range columnValues {
+			columnPointers[i] = &columnValues[i]
+		}
+
+		if err := rows.Scan(columnPointers...); err != nil {
+			continue
+		}
+
+		rowMap := make(map[string]interface{})
+		for i, colName := range columns {
+			val := columnValues[i]
+			bytes, ok := val.([]byte)
+			if ok {
+				rowMap[colName] = string(bytes)
+			} else {
+				rowMap[colName] = val
+			}
+		}
+		results = append(results, rowMap)
+	}
+	return results, nil
+}
+
+// SelectRows selects rows from a specified table based on a given query
+func SelectRows(ctx context.Context, db *gorm.DB, table, query string) ([]map[string]interface{}, error) {
+	sql := fmt.Sprintf("SELECT * FROM %s", table)
+	if query != "" {
+		sql += " WHERE " + query
+	}
+
+	return ExecQuery(ctx, db, sql)
+}
+
+// convertDBType converts MySQL or PostgreSQL data types to custom internal types and determines if they are indexable
+func convertDBType(dialect, dbType string) (string, bool) {
+	typ := strings.ToLower(dbType)
+
+	// Common type conversions
+	switch {
+	case strings.HasPrefix(typ, "int"), strings.HasPrefix(typ, "tinyint"),
+		strings.HasPrefix(typ, "smallint"), strings.HasPrefix(typ, "mediumint"),
+		strings.HasPrefix(typ, "bigint"), strings.HasPrefix(typ, "serial"),
+		strings.HasPrefix(typ, "bigserial"):
+		return types.LogExtractValueTypeLong, true
+
+	case strings.HasPrefix(typ, "varchar"), strings.HasPrefix(typ, "text"),
+		strings.HasPrefix(typ, "char"), strings.HasPrefix(typ, "tinytext"),
+		strings.HasPrefix(typ, "mediumtext"), strings.HasPrefix(typ, "longtext"),
+		strings.HasPrefix(typ, "character varying"), strings.HasPrefix(typ, "nvarchar"),
+		strings.HasPrefix(typ, "nchar"):
+		return types.LogExtractValueTypeText, true
+
+	case strings.HasPrefix(typ, "float"), strings.HasPrefix(typ, "double"),
+		strings.HasPrefix(typ, "decimal"), strings.HasPrefix(typ, "numeric"),
+		strings.HasPrefix(typ, "real"), strings.HasPrefix(typ, "double precision"):
+		return types.LogExtractValueTypeFloat, true
+
+	case strings.HasPrefix(typ, "date"), strings.HasPrefix(typ, "datetime"),
+		strings.HasPrefix(typ, "timestamp"), strings.HasPrefix(typ, "timestamptz"),
+		strings.HasPrefix(typ, "time"), strings.HasPrefix(typ, "smalldatetime"):
+		return types.LogExtractValueTypeDate, false
+
+	case strings.HasPrefix(typ, "boolean"), strings.HasPrefix(typ, "bit"):
+		return types.LogExtractValueTypeBool, false
+	}
+
+	// Specific type conversions for MySQL
+	if dialect == "mysql" {
+		switch {
+		default:
+			return typ, false
+		}
+	}
+
+	// Specific type conversions for PostgreSQL
+	if dialect == "postgres" {
+		switch {
+		default:
+			return typ, false
+		}
+	}
+
+	if dialect == "oracle" {
+		switch {
+		default:
+			return typ, false
+		}
+	}
+
+	// Can continue to add specific 'dialect' type ...
+
+	return typ, false
+}

dskit/sqlbase/timeseries.go

@@ -0,0 +1,250 @@
+// @Author: Ciusyan 5/20/24
+
+package sqlbase
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/dskit/types"
+
+	"github.com/prometheus/common/model"
+	"gorm.io/gorm"
+)
+
+type QueryParam struct {
+	Sql  string     `json:"sql"`
+	Keys types.Keys `json:"keys" mapstructure:"keys"`
+}
+
+var (
+	BannedOp = map[string]struct{}{
+		"CREATE":   {},
+		"INSERT":   {},
+		"UPDATE":   {},
+		"DELETE":   {},
+		"ALTER":    {},
+		"REVOKE":   {},
+		"DROP":     {},
+		"RENAME":   {},
+		"TRUNCATE": {},
+		"SET":      {},
+	}
+)
+
+// Query executes a given SQL query and returns the results
+func Query(ctx context.Context, db *gorm.DB, query *QueryParam) ([]map[string]interface{}, error) {
+	// Validate SQL to prevent write operations if needed
+	sqlItem := strings.Split(strings.ToUpper(query.Sql), " ")
+	for _, item := range sqlItem {
+		if _, ok := BannedOp[item]; ok {
+			return nil, fmt.Errorf("operation %s is forbidden, only read operations are allowed, please check your SQL", item)
+		}
+	}
+
+	return ExecQuery(ctx, db, query.Sql)
+}
+
+// QueryTimeseries executes a time series data query using the given parameters
+func QueryTimeseries(ctx context.Context, db *gorm.DB, query *QueryParam, ignoreDefault ...bool) ([]types.MetricValues, error) {
+	rows, err := Query(ctx, db, query)
+	if err != nil {
+		return nil, err
+	}
+
+	return FormatMetricValues(query.Keys, rows, ignoreDefault...), nil
+}
+
+func FormatMetricValues(keys types.Keys, rows []map[string]interface{}, ignoreDefault ...bool) []types.MetricValues {
+	ignore := false
+	if len(ignoreDefault) > 0 {
+		ignore = ignoreDefault[0]
+	}
+
+	keyMap := make(map[string]string)
+	for _, valueMetric := range strings.Split(keys.ValueKey, " ") {
+		keyMap[valueMetric] = "value"
+	}
+
+	for _, labelMetric := range strings.Split(keys.LabelKey, " ") {
+		keyMap[labelMetric] = "label"
+	}
+
+	if keys.TimeKey == "" {
+		keys.TimeKey = "time"
+	}
+
+	if len(keys.TimeKey) > 0 {
+		keyMap[keys.TimeKey] = "time"
+	}
+
+	var dataResps []types.MetricValues
+	dataMap := make(map[string]*types.MetricValues)
+
+	for _, row := range rows {
+		labels := make(map[string]string)
+		metricValue := make(map[string]float64)
+		metricTs := make(map[string]float64)
+
+		// Process each column based on its designated role (value, label, time)
+		for k, v := range row {
+			switch keyMap[k] {
+			case "value":
+				val, err := ParseFloat64Value(v)
+				if err != nil {
+					continue
+				}
+				metricValue[k] = val
+			case "label":
+				labels[k] = fmt.Sprintf("%v", v)
+			case "time":
+				ts, err := ParseTime(v, keys.TimeFormat)
+				if err != nil {
+					continue
+				}
+				metricTs[k] = float64(ts.Unix())
+			default:
+				// Default to labels for any unrecognized columns
+				if !ignore {
+					labels[k] = fmt.Sprintf("%v", v)
+				}
+			}
+		}
+
+		// Compile and store the metric values
+		for metricName, value := range metricValue {
+			metrics := make(model.Metric)
+			var labelsStr []string
+
+			for k1, v1 := range labels {
+				metrics[model.LabelName(k1)] = model.LabelValue(v1)
+				labelsStr = append(labelsStr, fmt.Sprintf("%s=%s", k1, v1))
+			}
+			metrics["__name__"] = model.LabelValue(metricName)
+			labelsStr = append(labelsStr, fmt.Sprintf("__name__=%s", metricName))
+
+			// Hash the labels to use as a key
+			sort.Strings(labelsStr)
+			labelsStrHash := fmt.Sprintf("%x", md5.Sum([]byte(strings.Join(labelsStr, ","))))
+
+			// Append new values to the existing metric, if present
+			ts, exists := metricTs[keys.TimeKey]
+			if !exists {
+				ts = float64(time.Now().Unix()) // Default to current time if not specified
+			}
+
+			valuePair := []float64{ts, value}
+			if existing, ok := dataMap[labelsStrHash]; ok {
+				existing.Values = append(existing.Values, valuePair)
+			} else {
+				dataResp := types.MetricValues{
+					Metric: metrics,
+					Values: [][]float64{valuePair},
+				}
+				dataMap[labelsStrHash] = &dataResp
+			}
+		}
+	}
+
+	// Convert the map to a slice for the response
+	for _, v := range dataMap {
+		sort.Slice(v.Values, func(i, j int) bool { return v.Values[i][0] < v.Values[j][0] }) // Sort by timestamp
+		dataResps = append(dataResps, *v)
+	}
+
+	return dataResps
+}
+
+// parseFloat64Value attempts to convert an interface{} to float64 using reflection
+func ParseFloat64Value(val interface{}) (float64, error) {
+	v := reflect.ValueOf(val)
+	switch v.Kind() {
+	case reflect.Float64, reflect.Float32:
+		return v.Float(), nil
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return float64(v.Int()), nil
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return float64(v.Uint()), nil
+	case reflect.String:
+		return strconv.ParseFloat(v.String(), 64)
+	case reflect.Slice:
+		if v.Type().Elem().Kind() == reflect.Uint8 {
+			return strconv.ParseFloat(string(v.Bytes()), 64)
+		}
+	case reflect.Interface:
+		return ParseFloat64Value(v.Interface())
+	case reflect.Ptr:
+		if !v.IsNil() {
+			return ParseFloat64Value(v.Elem().Interface())
+		}
+	case reflect.Struct:
+		if num, ok := val.(json.Number); ok {
+			return num.Float64()
+		}
+	}
+	return 0, fmt.Errorf("cannot convert type %T to float64", val)
+}
+
+// ParseTime attempts to parse a time value from an interface{} using a specified format
+func ParseTime(val interface{}, format string) (time.Time, error) {
+	v := reflect.ValueOf(val)
+	switch v.Kind() {
+	case reflect.String:
+		str := v.String()
+		return parseTimeFromString(str, format)
+	case reflect.Slice:
+		if v.Type().Elem().Kind() == reflect.Uint8 {
+			str := string(v.Bytes())
+			return parseTimeFromString(str, format)
+		}
+	case reflect.Int, reflect.Int64:
+		return time.Unix(v.Int(), 0), nil
+	case reflect.Float64:
+		return time.Unix(int64(v.Float()), 0), nil
+	case reflect.Interface:
+		return ParseTime(v.Interface(), format)
+	case reflect.Ptr:
+		if !v.IsNil() {
+			return ParseTime(v.Elem().Interface(), format)
+		}
+	case reflect.Struct:
+		if t, ok := val.(time.Time); ok {
+			return t, nil
+		}
+	}
+	return time.Time{}, fmt.Errorf("invalid time value type: %v", val)
+}
+
+func parseTimeFromString(str, format string) (time.Time, error) {
+	// If a custom time format is provided, use it to parse the string
+	if format != "" {
+		parsedTime, err := time.Parse(format, str)
+		if err == nil {
+			return parsedTime, nil
+		}
+		return time.Time{}, fmt.Errorf("failed to parse time '%s' with format '%s': %v", str, format, err)
+	}
+
+	// Try to parse the string as RFC3339, RFC3339Nano, or Unix timestamp
+	if parsedTime, err := time.Parse(time.RFC3339, str); err == nil {
+		return parsedTime, nil
+	}
+	if parsedTime, err := time.Parse(time.RFC3339Nano, str); err == nil {
+		return parsedTime, nil
+	}
+	if timestamp, err := strconv.ParseInt(str, 10, 64); err == nil {
+		return time.Unix(timestamp, 0), nil
+	}
+	if timestamp, err := strconv.ParseFloat(str, 64); err == nil {
+		return time.Unix(int64(timestamp), 0), nil
+	}
+
+	return time.Time{}, fmt.Errorf("failed to parse time '%s'", str)
+}

dskit/sqlbase/timeseries_test.go

@@ -0,0 +1,149 @@
+// @Author: Ciusyan 5/17/24
+
+package sqlbase
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/dskit/types"
+)
+
+func TestFormatMetricValues(t *testing.T) {
+	tests := []struct {
+		name string
+		keys types.Keys
+		rows []map[string]interface{}
+		want []types.MetricValues
+	}{
+		{
+			name: "cases1",
+			keys: types.Keys{
+				ValueKey:   "grade a_grade",
+				LabelKey:   "id student_name",
+				TimeKey:    "update_time",
+				TimeFormat: "2006-01-02 15:04:05",
+			},
+			rows: []map[string]interface{}{
+				{
+					"id":           "10007",
+					"grade":        20003,
+					"student_name": "邵子韬",
+					"a_grade":      69,
+					"update_time":  "2024-05-14 10:00:00",
+				},
+				{
+					"id":           "10007",
+					"grade":        20003,
+					"student_name": "邵子韬",
+					"a_grade":      69,
+					"update_time":  "2024-05-14 10:05:00",
+				},
+				{
+					"id":           "10007",
+					"grade":        20003,
+					"student_name": "邵子韬",
+					"a_grade":      69,
+					"update_time":  "2024-05-14 10:10:00",
+				},
+				{
+					"id":           "10008",
+					"grade":        20004,
+					"student_name": "Ciusyan",
+					"a_grade":      100,
+					"update_time":  "2024-05-14 12:00:00",
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := FormatMetricValues(tt.keys, tt.rows)
+			for _, g := range got {
+				t.Log(g)
+			}
+		})
+	}
+}
+
+func TestParseFloat64Value(t *testing.T) {
+
+	ptr := func(val float64) *float64 {
+		return &val
+	}
+
+	tests := []struct {
+		name    string
+		input   interface{}
+		want    float64
+		wantErr bool
+	}{
+		{"float64", 1.23, 1.23, false},
+		{"float32", float32(1.23), float64(float32(1.23)), false},
+		{"int", 123, 123, false},
+		{"int64", int64(123), 123, false},
+		{"uint", uint(123), 123, false},
+		{"uint64", uint64(123), 123, false},
+		{"string", "1.23", 1.23, false},
+		{"[]byte", []byte("1.23"), 1.23, false},
+		{"json.Number", json.Number("1.23"), 1.23, false},
+		{"interface", interface{}(1.23), 1.23, false},
+		{"pointer", ptr(1.23), 1.23, false},
+		{"invalid string", "abc", 0, true},
+		{"invalid type", struct{}{}, 0, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseFloat64Value(tt.input)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseFloat64Value() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("parseFloat64Value() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseTime(t *testing.T) {
+
+	ptrTime := func(t time.Time) *time.Time {
+		return &t
+	}
+
+	tests := []struct {
+		name    string
+		input   interface{}
+		format  string
+		want    time.Time
+		wantErr bool
+	}{
+		{"RFC3339", "2024-05-14T12:34:56Z", "", time.Date(2024, 5, 14, 12, 34, 56, 0, time.UTC), false},
+		{"RFC3339Nano", "2024-05-14T12:34:56.789Z", "", time.Date(2024, 5, 14, 12, 34, 56, 789000000, time.UTC), false},
+		{"Unix timestamp int", int64(1715642135), "", time.Unix(1715642135, 0), false},
+		{"Unix timestamp float64", 1715642135.0, "", time.Unix(int64(1715642135), 0), false},
+		{"custom format", "14/05/2024", "02/01/2006", time.Date(2024, 5, 14, 0, 0, 0, 0, time.UTC), false},
+		{"slice", []byte("2024-05-14T12:34:56Z"), "", time.Date(2024, 5, 14, 12, 34, 56, 0, time.UTC), false},
+		{"interface", interface{}("2024-05-14T12:34:56Z"), "", time.Date(2024, 5, 14, 12, 34, 56, 0, time.UTC), false},
+		{"pointer", ptrTime(time.Date(2024, 5, 14, 12, 34, 56, 0, time.UTC)), "", time.Date(2024, 5, 14, 12, 34, 56, 0, time.UTC), false},
+		{"invalid format", "14-05-2024", "02/01/2006", time.Time{}, true},
+		{"invalid type", struct{}{}, "", time.Time{}, true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseTime(tt.input, tt.format)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseTime() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !got.Equal(tt.want) {
+				t.Errorf("ParseTime() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

dskit/tdengine/tdengine.go

@@ -0,0 +1,207 @@
+package tdengine
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/dskit/types"
+	"github.com/ccfos/nightingale/v6/pkg/tlsx"
+
+	"github.com/toolkits/pkg/logger"
+)
+
+type Tdengine struct {
+	Addr                string             `json:"tdengine.addr" mapstructure:"tdengine.addr"`
+	Basic               *TDengineBasicAuth `json:"tdengine.basic" mapstructure:"tdengine.basic"`
+	Token               string             `json:"tdengine.token" mapstructure:"tdengine.token"`
+	Timeout             int64              `json:"tdengine.timeout" mapstructure:"tdengine.timeout"`
+	DialTimeout         int64              `json:"tdengine.dial_timeout" mapstructure:"tdengine.dial_timeout"`
+	MaxIdleConnsPerHost int                `json:"tdengine.max_idle_conns_per_host" mapstructure:"tdengine.max_idle_conns_per_host"`
+	Headers             map[string]string  `json:"tdengine.headers" mapstructure:"tdengine.headers"`
+	SkipTlsVerify       bool               `json:"tdengine.skip_tls_verify" mapstructure:"tdengine.skip_tls_verify"`
+
+	tlsx.ClientConfig
+
+	header map[string][]string `json:"-"`
+	client *http.Client        `json:"-"`
+}
+
+type TDengineBasicAuth struct {
+	User      string `json:"tdengine.user" mapstructure:"tdengine.user"`
+	Password  string `json:"tdengine.password" mapstructure:"tdengine.password"`
+	IsEncrypt bool   `json:"tdengine.is_encrypt" mapstructure:"tdengine.is_encrypt"`
+}
+
+type APIResponse struct {
+	Code       int             `json:"code"`
+	ColumnMeta [][]interface{} `json:"column_meta"`
+	Data       [][]interface{} `json:"data"`
+	Rows       int             `json:"rows"`
+}
+
+type QueryParam struct {
+	Database string `json:"database"`
+	Table    string `json:"table"`
+}
+
+func (tc *Tdengine) InitCli() {
+
+	tc.client = &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			DisableCompression:    true,
+		},
+	}
+
+	tc.header = map[string][]string{
+		"Connection": {"keep-alive"},
+	}
+
+	for k, v := range tc.Headers {
+		kv := strings.Split(v, ":")
+		if len(kv) != 2 {
+			continue
+		}
+		tc.header[k] = []string{v}
+	}
+
+	if tc.Basic != nil {
+		basic := base64.StdEncoding.EncodeToString([]byte(tc.Basic.User + ":" + tc.Basic.Password))
+		tc.header["Authorization"] = []string{fmt.Sprintf("Basic %s", basic)}
+	}
+}
+
+func (tc *Tdengine) QueryTable(query string) (APIResponse, error) {
+	var apiResp APIResponse
+	req, err := http.NewRequest("POST", tc.Addr+"/rest/sql", strings.NewReader(query))
+	if err != nil {
+		return apiResp, err
+	}
+
+	for k, v := range tc.header {
+		req.Header[k] = v
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	resp, err := tc.client.Do(req)
+	if err != nil {
+		return apiResp, err
+	}
+	defer resp.Body.Close()
+
+	// 限制响应体大小为10MB
+	maxSize := int64(10 * 1024 * 1024) // 10MB
+	limitedReader := http.MaxBytesReader(nil, resp.Body, maxSize)
+
+	if resp.StatusCode != http.StatusOK {
+		return apiResp, fmt.Errorf("HTTP error, status: %s", resp.Status)
+	}
+
+	err = json.NewDecoder(limitedReader).Decode(&apiResp)
+	if err != nil {
+		if strings.Contains(err.Error(), "http: request body too large") {
+			return apiResp, fmt.Errorf("response body exceeds 10MB limit")
+		}
+		return apiResp, err
+	}
+
+	return apiResp, nil
+}
+
+func (tc *Tdengine) ShowDatabases(context.Context) ([]string, error) {
+	var databases []string
+	data, err := tc.QueryTable("show databases")
+	if err != nil {
+		return databases, err
+	}
+
+	for _, row := range data.Data {
+		databases = append(databases, row[0].(string))
+	}
+	return databases, nil
+}
+
+func (tc *Tdengine) ShowTables(ctx context.Context, database string) ([]string, error) {
+	var tables []string
+	sql := fmt.Sprintf("show %s", database)
+	data, err := tc.QueryTable(sql)
+	if err != nil {
+		return tables, err
+	}
+
+	for _, row := range data.Data {
+		tables = append(tables, row[0].(string))
+	}
+	return tables, nil
+}
+
+func (tc *Tdengine) DescribeTable(ctx context.Context, query interface{}) ([]*types.ColumnProperty, error) {
+	var columns []*types.ColumnProperty
+	queryMap, ok := query.(map[string]string)
+	if !ok {
+		return nil, fmt.Errorf("invalid query")
+	}
+	sql := fmt.Sprintf("select * from %s.%s limit 1", queryMap["database"], queryMap["table"])
+	data, err := tc.QueryTable(sql)
+	if err != nil {
+		return columns, err
+	}
+	for _, row := range data.ColumnMeta {
+		var colType string
+		switch t := row[1].(type) {
+		case float64:
+			// v2版本数字类型映射
+			switch int(t) {
+			case 1:
+				colType = "BOOL"
+			case 2:
+				colType = "TINYINT"
+			case 3:
+				colType = "SMALLINT"
+			case 4:
+				colType = "INT"
+			case 5:
+				colType = "BIGINT"
+			case 6:
+				colType = "FLOAT"
+			case 7:
+				colType = "DOUBLE"
+			case 8:
+				colType = "BINARY"
+			case 9:
+				colType = "TIMESTAMP"
+			case 10:
+				colType = "NCHAR"
+			default:
+				colType = "UNKNOWN"
+			}
+		case string:
+			// v3版本直接使用字符串类型
+			colType = t
+		default:
+			logger.Warningf("unexpected column type format: %v", row[1])
+			colType = "UNKNOWN"
+		}
+
+		column := &types.ColumnProperty{
+			Field: row[0].(string),
+			Type:  colType,
+		}
+		columns = append(columns, column)
+	}
+
+	return columns, nil
+}

dskit/types/timeseries.go

@@ -0,0 +1,51 @@
+package types
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+
+	"github.com/prometheus/common/model"
+)
+
+// 时序数据
+type MetricValues struct {
+	Metric model.Metric `json:"metric"`
+	Values [][]float64  `json:"values"`
+}
+
+type HistogramValues struct {
+	Total  int64       `json:"total"`
+	Values [][]float64 `json:"values"`
+}
+
+// 瞬时值
+type AggregateValues struct {
+	Labels map[string]string  `json:"labels"`
+	Values map[string]float64 `json:"values"`
+}
+
+// string
+func (m *MetricValues) String() string {
+	var buf bytes.Buffer
+	buf.WriteString(fmt.Sprintf("Metric: %+v ", m.Metric))
+	buf.WriteString("Values: ")
+	for _, v := range m.Values {
+		buf.WriteString("  [")
+		for i, ts := range v {
+			if i > 0 {
+				buf.WriteString(", ")
+			}
+			buf.WriteString(strconv.FormatFloat(ts, 'f', -1, 64))
+		}
+		buf.WriteString("] ")
+	}
+	return buf.String()
+}
+
+type Keys struct {
+	ValueKey   string `json:"valueKey" mapstructure:"valueKey"` // 多个用空格分隔
+	LabelKey   string `json:"labelKey" mapstructure:"labelKey"` // 多个用空格分隔
+	TimeKey    string `json:"timeKey" mapstructure:"timeKey"`
+	TimeFormat string `json:"timeFormat" mapstructure:"timeFormat"` // not used anymore
+}

dskit/types/types.go

@@ -0,0 +1,19 @@
+package types
+
+const (
+	LogExtractValueTypeLong   = "long"
+	LogExtractValueTypeFloat  = "float"
+	LogExtractValueTypeText   = "text"
+	LogExtractValueTypeDate   = "date"
+	LogExtractValueTypeBool   = "bool"
+	LogExtractValueTypeObject = "object"
+	LogExtractValueTypeArray  = "array"
+	LogExtractValueTypeJSON   = "json"
+)
+
+type ColumnProperty struct {
+	Field     string `json:"field"`
+	Type      string `json:"type"`
+	Type2     string `json:"type2,omitempty"` // field_property.Type
+	Indexable bool   `json:"indexable"`       // 是否可以索引
+}

etc/config.toml

@@ -50,7 +50,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
@@ -73,14 +73,14 @@ DefaultRoles = ["Standard"]
 OpenRSA = false
 
 [DB]
+# mysql postgres sqlite
+DBType = "sqlite"
 # postgres: host=%s port=%s user=%s dbname=%s password=%s sslmode=%s
 # postgres: DSN="host=127.0.0.1 port=5432 user=root dbname=n9e_v6 password=1234 sslmode=disable"
-# sqlite: DSN="/path/to/filename.db"
-DSN = "root:1234@tcp(127.0.0.1:3306)/n9e_v6?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"
+# mysql: DSN="root:1234@tcp(localhost:3306)/n9e_v6?charset=utf8mb4&parseTime=True&loc=Local"
+DSN = "n9e.db"
 # enable debug mode or not
 Debug = false
-# mysql postgres sqlite
-DBType = "mysql"
 # unit: s
 MaxLifetime = 7200
 # max open connections
@@ -98,8 +98,8 @@ Address = "127.0.0.1:6379"
 # DB = 0
 # UseTLS = false
 # TLSMinVersion = "1.2"
-# standalone cluster sentinel
-RedisType = "standalone"
+# standalone cluster sentinel miniredis
+RedisType = "miniredis"
 # Mastername for sentinel type
 # MasterName = "mymaster"
 # SentinelUsername = ""
@@ -138,6 +138,9 @@ ForceUseServerTS = true
 # [Pushgw.WriterOpt]
 # QueueMaxSize = 1000000
 # QueuePopSize = 1000
+# AllQueueMaxSize = 1000000
+# fresh time, unit ms
+# AllQueueMaxSizeInterval = 200
 
 [[Pushgw.Writers]] 
 # Url = "http://127.0.0.1:8480/insert/0/prometheus/api/v1/write"

etc/edge/edge.toml

@@ -54,7 +54,7 @@ Enable = true
 # user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 
 [HTTP.APIForService]
-Enable = true 
+Enable = false
 [HTTP.APIForService.BasicAuth]
 user001 = "ccc26da7b9aba533cbb263a36c07dcc5"
 

go.mod

@@ -3,7 +3,11 @@ module github.com/ccfos/nightingale/v6
 go 1.22
 
 require (
-	github.com/BurntSushi/toml v0.3.1
+	github.com/BurntSushi/toml v1.4.0
+	github.com/ClickHouse/clickhouse-go/v2 v2.23.2
+	github.com/VictoriaMetrics/metricsql v0.81.1
+	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
+	github.com/bitly/go-simplejson v0.5.1
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -11,47 +15,85 @@ require (
 	github.com/flashcatcloud/ibex v1.3.5
 	github.com/gin-contrib/pprof v1.4.0
 	github.com/gin-gonic/gin v1.9.1
+	github.com/glebarez/sqlite v1.11.0
 	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/jinzhu/copier v0.4.0
 	github.com/json-iterator/go v1.1.12
 	github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
 	github.com/mailru/easyjson v0.7.7
 	github.com/mattn/go-isatty v0.0.19
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mojocn/base64Captcha v1.3.6
+	github.com/olivere/elastic/v7 v7.0.32
 	github.com/pelletier/go-toml/v2 v2.0.8
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.16.0
-	github.com/prometheus/common v0.44.0
+	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/common v0.60.1
 	github.com/prometheus/prometheus v0.47.1
 	github.com/rakyll/statik v0.1.7
 	github.com/redis/go-redis/v9 v9.0.2
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/stretchr/testify v1.8.4
-	github.com/tidwall/gjson v1.14.0
+	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.14.2
 	github.com/toolkits/pkg v1.3.8
-	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
-	golang.org/x/oauth2 v0.10.0
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/oauth2 v0.23.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/yaml.v2 v2.4.0
+	gorm.io/driver/clickhouse v0.6.1
 	gorm.io/driver/mysql v1.4.4
-	gorm.io/driver/postgres v1.4.5
+	gorm.io/driver/postgres v1.5.11
 	gorm.io/driver/sqlite v1.5.5
-	gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde
+	gorm.io/gorm v1.25.10
 )
 
-require github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+require (
+	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
+)
+
+require (
+	github.com/VictoriaMetrics/metrics v1.34.0 // indirect
+	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/glebarez/go-sqlite v1.21.2 // indirect
+	github.com/jackc/pgx/v5 v5.7.1 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/valyala/fastrand v1.1.0 // indirect
+	github.com/valyala/histogram v1.2.0 // indirect
+	github.com/yuin/gopher-lua v1.1.1 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/sqlite v1.23.1 // indirect
+)
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
+	github.com/alicebob/miniredis/v2 v2.33.0
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -69,26 +111,19 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
 	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
-	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.13.0 // indirect
-	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.3.1 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.12.0 // indirect
-	github.com/jackc/pgx/v4 v4.17.2 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/procfs v0.11.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -97,14 +132,15 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/automaxprocs v1.5.2 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/image v0.18.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace golang.org/x/exp v0.0.0-20231006140011-7918f672742d => golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1

go.sum

@@ -9,16 +9,32 @@ github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft
 github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
-github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
+github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
+github.com/ClickHouse/clickhouse-go/v2 v2.23.2 h1:+DAKPMnxLS7pduQZsrJc8OhdLS2L9MfDEJ2TS+hpYDM=
+github.com/ClickHouse/clickhouse-go/v2 v2.23.2/go.mod h1:aNap51J1OM3yxQJRgM+AlP/MPkGBCL8A74uQThoQhR0=
+github.com/VictoriaMetrics/metrics v1.34.0 h1:0i8k/gdOJdSoZB4Z9pikVnVQXfhcIvnG7M7h2WaQW2w=
+github.com/VictoriaMetrics/metrics v1.34.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
+github.com/VictoriaMetrics/metricsql v0.81.1 h1:1gpqI3Mwru1tCM8nZiKxBG0P+DNkjlRwLhRPII3cuho=
+github.com/VictoriaMetrics/metricsql v0.81.1/go.mod h1:1g4hdCwlbJZ851PU9VN65xy9Rdlzupo6fx3SNZ8Z64U=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.33.0 h1:uvTF0EDeu9RLnUEG27Db5I68ESoIxTiXbNUiji6lZrA=
+github.com/alicebob/miniredis/v2 v2.33.0/go.mod h1:MhP4a3EU7aENRi9aO+tHfTBZicLqQevyi/DJpoj6mi0=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/aws/aws-sdk-go v1.44.302 h1:ST3ko6GrJKn3Xi+nAvxjG3uk/V1pW8KC52WLeIxqqNk=
 github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
+github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
 github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ=
 github.com/bsm/ginkgo/v2 v2.5.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
 github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8=
@@ -26,18 +42,13 @@ github.com/bsm/gomega v1.20.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATs
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
 github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
-github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
-github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk=
 github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -49,6 +60,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/expr-lang/expr v1.16.1 h1:Na8CUcMdyGbnNpShY7kzcHCU7WqxuL+hnxgHZ4vaz/A=
 github.com/expr-lang/expr v1.16.1/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
@@ -57,6 +70,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flashcatcloud/ibex v1.3.5 h1:8GOOf5+aJT0TP/MC6izz7CO5JKJSdKVFBwL0vQp93Nc=
 github.com/flashcatcloud/ibex v1.3.5/go.mod h1:T8hbMUySK2q6cXUaYp0AUVeKkU9Od2LjzwmB5lmTRBM=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
 github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
@@ -67,14 +82,20 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
 github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
+github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
+github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
+github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
 github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
 github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -91,12 +112,9 @@ github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg
 github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
-github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
@@ -105,71 +123,33 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA=
+github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww=
 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
-github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
-github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
-github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
-github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
-github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
-github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
-github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
-github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
-github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
-github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys=
-github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI=
-github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
-github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
-github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
-github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
-github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc=
-github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
-github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
-github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
-github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
-github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
-github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y=
-github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
-github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
-github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
-github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
-github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
-github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
-github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w=
-github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
-github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
-github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
-github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
-github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
-github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E=
-github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw=
-github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
+github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -187,22 +167,21 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
-github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
 github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
+github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7 h1:SWlt7BoQNASbhTUD0Oy5yysI2seJ7vWuGUp///OM4TM=
 github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7/go.mod h1:Y2SaZf2Rzd0pXkLVhLlCiAXFCLSXAIbTKDivVgff/AM=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
@@ -211,25 +190,16 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
-github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
-github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -237,17 +207,26 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mojocn/base64Captcha v1.3.6 h1:gZEKu1nsKpttuIAQgWHO+4Mhhls8cAKyiV2Ew03H+Tw=
 github.com/mojocn/base64Captcha v1.3.6/go.mod h1:i5CtHvm+oMbj1UzEPXaA8IH/xHFZ3DGY3Wh3dBpZ28E=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
+github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -257,50 +236,45 @@ github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8
 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
 github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
 github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
-github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
-github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc=
+github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
 github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI=
-github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk=
-github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/prometheus/prometheus v0.47.1 h1:bd2LiZyxzHn9Oo2Ei4eK2D86vz/L/OiqR1qYo0XmMBo=
 github.com/prometheus/prometheus v0.47.1/go.mod h1:J/bmOSjgH7lFxz2gZhrWEZs2i64vMS+HIuZfmYNhJ/M=
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE=
 github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62/go.mod h1:65XQgovT59RWatovFwnwocoUxiI/eENTnOY5GK3STuY=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
-github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
-github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
-github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -309,12 +283,13 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
-github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/toolkits/pkg v1.3.8 h1:2yamC20c5mHRtbcGiLY99Lm/2mVitFn6onE8KKvMT1o=
@@ -325,64 +300,55 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
+github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
+github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
+github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
 go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
 go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
-go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
-go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
-go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
-go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
-go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
 golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw=
 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
 golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
 golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -391,28 +357,20 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
+golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -425,18 +383,17 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
@@ -444,35 +401,24 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -482,7 +428,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
-gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -492,15 +437,23 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/clickhouse v0.6.1 h1:t7JMB6sLBXxN8hEO6RdzCbJCwq/jAEVZdwXlmQs1Sd4=
+gorm.io/driver/clickhouse v0.6.1/go.mod h1:riMYpJcGZ3sJ/OAZZ1rEP1j/Y0H6cByOAnwz7fo2AyM=
 gorm.io/driver/mysql v1.4.4 h1:MX0K9Qvy0Na4o7qSC/YI7XxqUw5KDw01umqgID+svdQ=
 gorm.io/driver/mysql v1.4.4/go.mod h1:BCg8cKI+R0j/rZRQxeKis/forqRwRSYOR8OM3Wo6hOM=
-gorm.io/driver/postgres v1.4.5 h1:mTeXTTtHAgnS9PgmhN2YeUbazYpLhUI1doLnw42XUZc=
-gorm.io/driver/postgres v1.4.5/go.mod h1:GKNQYSJ14qvWkvPwXljMGehpKrhlDNsqYRr5HnYGncg=
+gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314=
+gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI=
 gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E=
 gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE=
 gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
-gorm.io/gorm v1.24.1-0.20221019064659-5dd2bb482755/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
-gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg=
-gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
+gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
+modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

integrations/CloudWatch/collect/cloudwatch/cloud.toml

@@ -1,3 +1,4 @@
+interval="5m"
 [[instances]]
   ## Amazon Region
   # list of region and endpoint, see https://docs.aws.amazon.com/general/latest/gr/cw_region.html
@@ -40,17 +41,13 @@
   ## particular metric, that metric will not be returned by the CloudWatch API
   ## and will not be collected by Telegraf.
   #
-  ## Requested CloudWatch aggregation Period (required)
-  ## Must be a multiple of 60s.
-  period = "5m"
-
   ## Collection Delay (required)
   ## Must account for metrics availability via CloudWatch API
   delay = "5m"
 
-  ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid
-  ## gaps or overlap in pulled data
-  interval = "5m"
+  ## Requested CloudWatch aggregation Period (required)
+  ## Must be a multiple of 60s.
+  period = "5m"
 
   ## Recommended if "delay" and "period" are both within 3 hours of request
   ## time. Invalid values will be ignored. Recently Active feature will only

integrations/CloudWatch/markdown/README.md

@@ -71,18 +71,14 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
   ## Note that if a period is configured that is smaller than the minimum for a
   ## particular metric, that metric will not be returned by the CloudWatch API
   ## and will not be collected by Categraf.
-  #
-  ## Requested CloudWatch aggregation Period (required)
-  ## Must be a multiple of 60s.
-  period = "5m"
-
+   
   ## Collection Delay (required)
   ## Must account for metrics availability via CloudWatch API
   delay = "5m"
 
-  ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid
-  ## gaps or overlap in pulled data
-  interval = "5m"
+  ## Requested CloudWatch aggregation Period (required)
+  ## Must be a multiple of 60s.
+  period = "5m"
 
   ## Recommended if "delay" and "period" are both within 3 hours of request
   ## time. Invalid values will be ignored. Recently Active feature will only

integrations/MongoDB/dashboards/mongo_by_exporter.json

@@ -1,5 +1,5 @@
 {
-    "name": "MongoDB by instance",
+    "name": "MongoDB Overview by exporter",
     "tags": "Prometheus MongoDB",
     "ident": "",
     "configs": {
@@ -9,11 +9,11 @@
                 "id": "939298f2-b21f-4e2f-9142-c10946cc4032",
                 "layout": {
                     "h": 1,
+                    "i": "939298f2-b21f-4e2f-9142-c10946cc4032",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 0,
-                    "i": "939298f2-b21f-4e2f-9142-c10946cc4032",
-                    "isResizable": false
+                    "y": 0
                 },
                 "name": "Basic Info",
                 "type": "row"
@@ -32,12 +32,12 @@
                 "description": "instance count",
                 "id": "91970d24-3f04-4424-a1ed-73e7d28f5706",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "91970d24-3f04-4424-a1ed-73e7d28f5706",
+                    "isResizable": true,
                     "w": 6,
                     "x": 0,
-                    "y": 1,
-                    "i": "91970d24-3f04-4424-a1ed-73e7d28f5706",
-                    "isResizable": true
+                    "y": 1
                 },
                 "name": "Up",
                 "options": {
@@ -77,54 +77,39 @@
                 "version": "2.0.0"
             },
             {
-                "type": "stat",
-                "id": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
-                "layout": {
-                    "h": 4,
-                    "w": 6,
-                    "x": 6,
-                    "y": 1,
-                    "i": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
-                    "isResizable": true
-                },
-                "version": "3.0.0",
-                "datasourceCate": "prometheus",
-                "datasourceValue": "${prom}",
-                "targets": [
-                    {
-                        "expr": "mongodb_ss_uptime{instance=\"$instance\"}",
-                        "refId": "A",
-                        "maxDataPoints": 240
-                    }
-                ],
-                "transformations": [
-                    {
-                        "id": "organize",
-                        "options": {}
-                    }
-                ],
-                "name": "Uptime",
-                "description": "Uptime",
-                "maxPerRow": 4,
                 "custom": {
-                    "textMode": "value",
-                    "graphMode": "none",
-                    "colorMode": "value",
                     "calc": "lastNotNull",
-                    "valueField": "Value",
                     "colSpan": 1,
+                    "colorMode": "value",
+                    "textMode": "value",
                     "textSize": {
                         "title": null
                     },
-                    "orientation": "auto"
+                    "valueField": "Value"
                 },
+                "datasourceCate": "prometheus",
+                "datasourceValue": "${prom}",
+                "description": "Uptime",
+                "id": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
+                "layout": {
+                    "h": 7,
+                    "i": "c7b52e8e-b417-4c61-a15e-e2f186fccd67",
+                    "isResizable": true,
+                    "w": 6,
+                    "x": 6,
+                    "y": 1
+                },
+                "name": "Uptime",
                 "options": {
+                    "standardOptions": {
+                        "util": "humantimeSeconds"
+                    },
                     "thresholds": {
                         "steps": [
                             {
                                 "color": "#634CD9",
-                                "value": null,
-                                "type": "base"
+                                "type": "base",
+                                "value": null
                             }
                         ]
                     },
@@ -147,51 +132,34 @@
                             },
                             "type": "range"
                         }
-                    ],
-                    "standardOptions": {
-                        "util": "seconds",
-                        "decimals": 2
-                    }
+                    ]
                 },
-                "overrides": [
+                "targets": [
                     {
-                        "matcher": {
-                            "id": "byFrameRefID"
-                        },
-                        "properties": {
-                            "thresholds": {
-                                "steps": [
-                                    {
-                                        "color": "#6C53B1",
-                                        "value": null,
-                                        "type": "base"
-                                    }
-                                ]
-                            },
-                            "standardOptions": {
-                                "decimals": 0
-                            }
-                        }
+                        "expr": "mongodb_ss_uptime{instance=\"$instance\"}",
+                        "refId": "A"
                     }
-                ]
+                ],
+                "type": "stat",
+                "version": "2.0.0"
             },
             {
                 "type": "timeseries",
                 "id": "8446dded-9e11-4ee9-bdad-769b193ddf3e",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "8446dded-9e11-4ee9-bdad-769b193ddf3e",
+                    "isResizable": true,
                     "w": 6,
                     "x": 12,
-                    "y": 1,
-                    "i": "8446dded-9e11-4ee9-bdad-769b193ddf3e",
-                    "isResizable": true
+                    "y": 1
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "mongodb_ss_mem_resident * 1024 * 1024",
+                        "expr": "mongodb_ss_mem_resident{instance='$instance'} * 1024 * 1024",
                         "legend": "{{type}}",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -219,8 +187,7 @@
                         "selectMode": "single"
                     },
                     "standardOptions": {
-                        "util": "bytesIEC",
-                        "decimals": 2
+                        "util": "bytesIEC"
                     },
                     "thresholds": {
                         "steps": [
@@ -275,12 +242,12 @@
                 "description": "Page faults indicate that requests are processed from disk either because an index is missing or there is not enough memory for the data set. Consider increasing memory or sharding out.",
                 "id": "3eda28e7-2480-4ddc-b346-89ced1c33034",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "3eda28e7-2480-4ddc-b346-89ced1c33034",
+                    "isResizable": true,
                     "w": 6,
                     "x": 18,
-                    "y": 1,
-                    "i": "3eda28e7-2480-4ddc-b346-89ced1c33034",
-                    "isResizable": true
+                    "y": 1
                 },
                 "name": "Page Faults",
                 "options": {
@@ -333,12 +300,12 @@
                 "description": "Network traffic (bytes)",
                 "id": "528d0485-f947-470d-95f3-59eae157ebb6",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "528d0485-f947-470d-95f3-59eae157ebb6",
+                    "isResizable": true,
                     "w": 6,
                     "x": 0,
-                    "y": 5,
-                    "i": "528d0485-f947-470d-95f3-59eae157ebb6",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Network I/O",
                 "options": {
@@ -395,12 +362,12 @@
                 "description": "Number of connections Keep in mind the hard limit on the maximum number of connections set by your distribution.",
                 "id": "067e97c3-4e57-447f-a9dc-a49627b6ce18",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "067e97c3-4e57-447f-a9dc-a49627b6ce18",
+                    "isResizable": true,
                     "w": 6,
                     "x": 6,
-                    "y": 5,
-                    "i": "067e97c3-4e57-447f-a9dc-a49627b6ce18",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Connections",
                 "options": {
@@ -450,12 +417,12 @@
                 "description": "Number of assertion errors, Asserts are not important by themselves, but you can correlate spikes with other graphs.",
                 "id": "9e9b7356-cf0e-4e5f-95f5-00258c576bf4",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "9e9b7356-cf0e-4e5f-95f5-00258c576bf4",
+                    "isResizable": true,
                     "w": 6,
                     "x": 12,
-                    "y": 5,
-                    "i": "9e9b7356-cf0e-4e5f-95f5-00258c576bf4",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Assert Events",
                 "options": {
@@ -505,12 +472,12 @@
                 "description": "Number of operations waiting to acquire locks, Any number of queued operations for long periods of time is an indication of possible issues. Find the cause and fix it before requests get stuck in the queue.",
                 "id": "2698f0f8-a76a-499b-99cf-30504f0f4db6",
                 "layout": {
-                    "h": 4,
+                    "h": 7,
+                    "i": "2698f0f8-a76a-499b-99cf-30504f0f4db6",
+                    "isResizable": true,
                     "w": 6,
                     "x": 18,
-                    "y": 5,
-                    "i": "2698f0f8-a76a-499b-99cf-30504f0f4db6",
-                    "isResizable": true
+                    "y": 8
                 },
                 "name": "Lock Queue",
                 "options": {
@@ -547,11 +514,11 @@
                 "id": "2bdb8cc9-92f4-449e-8f70-a4c470a21604",
                 "layout": {
                     "h": 1,
+                    "i": "2bdb8cc9-92f4-449e-8f70-a4c470a21604",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 9,
-                    "i": "2bdb8cc9-92f4-449e-8f70-a4c470a21604",
-                    "isResizable": false
+                    "y": 15
                 },
                 "name": "Operation Info",
                 "type": "row"
@@ -574,12 +541,12 @@
                 "description": "Number of requests received Shows how many times a command is executed per second on average during the selected interval.",
                 "id": "c2819508-95e7-4c63-aeae-ce19f92469cd",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "c2819508-95e7-4c63-aeae-ce19f92469cd",
+                    "isResizable": true,
                     "w": 12,
                     "x": 0,
-                    "y": 10,
-                    "i": "c2819508-95e7-4c63-aeae-ce19f92469cd",
-                    "isResizable": true
+                    "y": 16
                 },
                 "name": "Command Operations",
                 "options": {
@@ -625,12 +592,12 @@
                 "type": "timeseries",
                 "id": "7030d97a-d69f-4916-a415-ec57503ab1ed",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "7030d97a-d69f-4916-a415-ec57503ab1ed",
+                    "isResizable": true,
                     "w": 12,
                     "x": 12,
-                    "y": 10,
-                    "i": "7030d97a-d69f-4916-a415-ec57503ab1ed",
-                    "isResizable": true
+                    "y": 16
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
@@ -704,19 +671,19 @@
                 "type": "timeseries",
                 "id": "1c3b73d5-c25c-449f-995d-26acc9c621e1",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "1c3b73d5-c25c-449f-995d-26acc9c621e1",
+                    "isResizable": true,
                     "w": 8,
                     "x": 0,
-                    "y": 15,
-                    "i": "1c3b73d5-c25c-449f-995d-26acc9c621e1",
-                    "isResizable": true
+                    "y": 23
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "rate(mongodb_ss_opLatencies_latency{}[5m]) / rate(mongodb_ss_opLatencies_latency{}[5m]) / 1000",
+                        "expr": "rate(mongodb_ss_opLatencies_latency{instance='$instance'}[5m]) / rate(mongodb_ss_opLatencies_latency{instance='$instance'}[5m]) / 1000",
                         "legend": "{{op_type}}",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -799,12 +766,12 @@
                 "description": "",
                 "id": "e642183c-8ba2-4f60-abc6-c65de49e7577",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "e642183c-8ba2-4f60-abc6-c65de49e7577",
+                    "isResizable": true,
                     "w": 8,
                     "x": 8,
-                    "y": 15,
-                    "i": "e642183c-8ba2-4f60-abc6-c65de49e7577",
-                    "isResizable": true
+                    "y": 23
                 },
                 "name": "Query Efficiency",
                 "options": {
@@ -861,12 +828,12 @@
                 "description": "number of cursors Helps identify why connections are increasing. Shows active cursors compared to cursors being automatically killed after 10 minutes due to an application not closing the connection.",
                 "id": "8b5a4f44-3291-4822-ab73-f56be6c62674",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "8b5a4f44-3291-4822-ab73-f56be6c62674",
+                    "isResizable": true,
                     "w": 8,
                     "x": 16,
-                    "y": 15,
-                    "i": "8b5a4f44-3291-4822-ab73-f56be6c62674",
-                    "isResizable": true
+                    "y": 23
                 },
                 "name": "Cursors",
                 "options": {
@@ -903,11 +870,11 @@
                 "id": "06946b19-94b4-4f72-bd87-70f87989257d",
                 "layout": {
                     "h": 1,
+                    "i": "06946b19-94b4-4f72-bd87-70f87989257d",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 20,
-                    "i": "06946b19-94b4-4f72-bd87-70f87989257d",
-                    "isResizable": false
+                    "y": 30
                 },
                 "name": "Cache Info",
                 "panels": [],
@@ -917,19 +884,19 @@
                 "type": "timeseries",
                 "id": "bb0ae571-43a1-430b-8f63-256f6f1ebee6",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "bb0ae571-43a1-430b-8f63-256f6f1ebee6",
+                    "isResizable": true,
                     "w": 6,
                     "x": 0,
-                    "y": 21,
-                    "i": "bb0ae571-43a1-430b-8f63-256f6f1ebee6",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "mongodb_ss_wt_cache_bytes_currently_in_the_cache{}",
+                        "expr": "mongodb_ss_wt_cache_bytes_currently_in_the_cache{instance='$instance'}",
                         "legend": "total",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -975,8 +942,7 @@
                         "selectMode": "single"
                     },
                     "standardOptions": {
-                        "util": "bytesIEC",
-                        "decimals": 2
+                        "util": "bytesIEC"
                     },
                     "thresholds": {
                         "steps": [
@@ -1017,19 +983,19 @@
                 "type": "timeseries",
                 "id": "f1ffd169-2a1a-42bc-9647-0e6621be0fef",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "f1ffd169-2a1a-42bc-9647-0e6621be0fef",
+                    "isResizable": true,
                     "w": 6,
                     "x": 6,
-                    "y": 21,
-                    "i": "f1ffd169-2a1a-42bc-9647-0e6621be0fef",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "rate(mongodb_ss_wt_cache_bytes_read_into_cache{}[5m])",
+                        "expr": "rate(mongodb_ss_wt_cache_bytes_read_into_cache{instance='$instance'}[5m])",
                         "legend": "read",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -1104,19 +1070,19 @@
                 "type": "timeseries",
                 "id": "43ee140d-ae6d-474a-9892-fa4743d7f97e",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "43ee140d-ae6d-474a-9892-fa4743d7f97e",
+                    "isResizable": true,
                     "w": 6,
                     "x": 12,
-                    "y": 21,
-                    "i": "43ee140d-ae6d-474a-9892-fa4743d7f97e",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "100 * sum(mongodb_ss_wt_cache_tracked_dirty_pages_in_the_cache{}) / sum(mongodb_ss_wt_cache_pages_currently_held_in_the_cache{})",
+                        "expr": "100 * sum(mongodb_ss_wt_cache_tracked_dirty_pages_in_the_cache{instance='$instance'}) / sum(mongodb_ss_wt_cache_pages_currently_held_in_the_cache{instance='$instance'})",
                         "legend": "dirty rate",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -1185,19 +1151,19 @@
                 "type": "timeseries",
                 "id": "1a22c31a-859a-400c-af2a-ae83c308d0f2",
                 "layout": {
-                    "h": 5,
+                    "h": 7,
+                    "i": "1a22c31a-859a-400c-af2a-ae83c308d0f2",
+                    "isResizable": true,
                     "w": 6,
                     "x": 18,
-                    "y": 21,
-                    "i": "1a22c31a-859a-400c-af2a-ae83c308d0f2",
-                    "isResizable": true
+                    "y": 31
                 },
                 "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
                 "targets": [
                     {
-                        "expr": "rate(mongodb_mongod_wiredtiger_cache_evicted_total{}[5m])",
+                        "expr": "rate(mongodb_mongod_wiredtiger_cache_evicted_total{instance='$instance'}[5m])",
                         "legend": "evicted pages",
                         "refId": "A",
                         "maxDataPoints": 240
@@ -1265,95 +1231,125 @@
                 "id": "b0016f4a-c565-4276-a08d-bacdf94b6b5a",
                 "layout": {
                     "h": 1,
+                    "i": "b0016f4a-c565-4276-a08d-bacdf94b6b5a",
+                    "isResizable": false,
                     "w": 24,
                     "x": 0,
-                    "y": 26,
-                    "i": "b0016f4a-c565-4276-a08d-bacdf94b6b5a",
-                    "isResizable": false
+                    "y": 45
                 },
                 "name": "ReplSet Info",
                 "type": "row"
             },
             {
-                "type": "timeseries",
-                "id": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
-                "layout": {
-                    "h": 5,
-                    "w": 12,
-                    "x": 0,
-                    "y": 27,
-                    "i": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
-                    "isResizable": true
+                "custom": {
+                    "calc": "lastNotNull",
+                    "colSpan": 1,
+                    "colorMode": "value",
+                    "textMode": "value",
+                    "textSize": {},
+                    "valueField": "Value"
                 },
-                "version": "3.0.0",
                 "datasourceCate": "prometheus",
                 "datasourceValue": "${prom}",
-                "targets": [
-                    {
-                        "expr": "mongodb_mongod_replset_member_replication_lag{instance=\"$instance\"}",
-                        "legend": "",
-                        "refId": "A",
-                        "maxDataPoints": 240
-                    }
-                ],
-                "transformations": [
-                    {
-                        "id": "organize",
-                        "options": {}
-                    }
-                ],
-                "name": "Replset Lag Seconds",
-                "description": "replica set member master-slave synchronization delay",
-                "maxPerRow": 4,
+                "description": "",
+                "id": "6187ceee-7c25-43f2-be1b-c44ad612ab52",
+                "layout": {
+                    "h": 7,
+                    "i": "6187ceee-7c25-43f2-be1b-c44ad612ab52",
+                    "isResizable": true,
+                    "w": 12,
+                    "x": 0,
+                    "y": 46
+                },
+                "name": "Replset Election",
                 "options": {
-                    "tooltip": {
-                        "mode": "all",
-                        "sort": "none"
-                    },
-                    "legend": {
-                        "displayMode": "hidden",
-                        "heightInPercentage": 30,
-                        "placement": "bottom",
-                        "behaviour": "showItem",
-                        "selectMode": "single"
-                    },
                     "standardOptions": {
+                        "decimals": 1,
                         "util": "seconds"
                     },
                     "thresholds": {
                         "steps": [
                             {
-                                "color": "#6C53B1",
-                                "value": null,
-                                "type": "base"
+                                "color": "#634CD9",
+                                "type": "base",
+                                "value": null
                             }
                         ]
-                    }
+                    },
+                    "valueMappings": [
+                        {
+                            "match": {
+                                "to": 1800
+                            },
+                            "result": {
+                                "color": "#f24526"
+                            },
+                            "type": "range"
+                        },
+                        {
+                            "match": {
+                                "from": 1800
+                            },
+                            "result": {
+                                "color": "#53b503"
+                            },
+                            "type": "range"
+                        }
+                    ]
                 },
+                "targets": [
+                    {
+                        "expr": "time() - mongodb_mongod_replset_member_election_date",
+                        "refId": "A"
+                    }
+                ],
+                "type": "stat",
+                "version": "2.0.0"
+            },
+            {
                 "custom": {
                     "drawStyle": "lines",
-                    "lineInterpolation": "smooth",
-                    "spanNulls": false,
-                    "lineWidth": 2,
                     "fillOpacity": 0.3,
                     "gradientMode": "opacity",
-                    "stack": "off",
-                    "scaleDistribution": {
-                        "type": "linear"
-                    },
-                    "showPoints": "none",
-                    "pointSize": 5
+                    "lineInterpolation": "smooth",
+                    "lineWidth": 2,
+                    "stack": "off"
                 },
-                "overrides": [
-                    {
-                        "matcher": {
-                            "id": "byFrameRefID"
-                        },
-                        "properties": {
-                            "rightYAxisDisplay": "off"
-                        }
+                "datasourceCate": "prometheus",
+                "datasourceValue": "${prom}",
+                "description": "replica set member master-slave synchronization delay",
+                "id": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
+                "layout": {
+                    "h": 7,
+                    "i": "f73fd0cd-ecbe-41f0-a2dc-4e02f7eaef1c",
+                    "isResizable": true,
+                    "w": 12,
+                    "x": 12,
+                    "y": 46
+                },
+                "name": "Replset Lag Seconds",
+                "options": {
+                    "legend": {
+                        "displayMode": "hidden"
+                    },
+                    "standardOptions": {
+                        "util": "seconds"
+                    },
+                    "thresholds": {},
+                    "tooltip": {
+                        "mode": "all",
+                        "sort": "none"
                     }
-                ]
+                },
+                "targets": [
+                    {
+                        "expr": "mongodb_mongod_replset_member_replication_lag{instance=\"$instance\"}",
+                        "legend": "lag",
+                        "refId": "A"
+                    }
+                ],
+                "type": "timeseries",
+                "version": "2.0.0"
             }
         ],
         "var": [
@@ -1375,4 +1371,4 @@
         "version": "3.0.0"
     },
     "uuid": 1717556328065329000
-}
+}

memsto/notify_config.go

@@ -1,25 +1,28 @@
 package memsto
 
 import (
+	"crypto/tls"
 	"encoding/json"
+	"net/http"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/ccfos/nightingale/v6/pkg/tplx"
-
-	"github.com/BurntSushi/toml"
 	"github.com/ccfos/nightingale/v6/alert/aconf"
 	"github.com/ccfos/nightingale/v6/dumper"
 	"github.com/ccfos/nightingale/v6/models"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
+	"github.com/ccfos/nightingale/v6/pkg/poster"
+	"github.com/ccfos/nightingale/v6/pkg/tplx"
+
+	"github.com/BurntSushi/toml"
 	"github.com/toolkits/pkg/logger"
 )
 
 type NotifyConfigCacheType struct {
 	ctx         *ctx.Context
 	ConfigCache *ConfigCache
-	webhooks    []*models.Webhook
+	webhooks    map[string]*models.Webhook
 	smtp        aconf.SMTPConfig
 	script      models.NotifyScript
 
@@ -47,6 +50,7 @@ func NewNotifyConfigCache(ctx *ctx.Context, configCache *ConfigCache) *NotifyCon
 	w := &NotifyConfigCacheType{
 		ctx:         ctx,
 		ConfigCache: configCache,
+		webhooks:    make(map[string]*models.Webhook),
 	}
 	w.SyncNotifyConfigs()
 	return w
@@ -85,11 +89,60 @@ func (w *NotifyConfigCacheType) syncNotifyConfigs() error {
 	}
 
 	if strings.TrimSpace(cval) != "" {
-		err = json.Unmarshal([]byte(cval), &w.webhooks)
+		var webhooks []*models.Webhook
+		err = json.Unmarshal([]byte(cval), &webhooks)
 		if err != nil {
 			dumper.PutSyncRecord("webhooks", start.Unix(), -1, -1, "failed to unmarshal configs.webhook: "+err.Error())
 			logger.Errorf("failed to unmarshal webhooks:%s error:%v", cval, err)
 		}
+
+		newWebhooks := make(map[string]*models.Webhook, len(webhooks))
+		for i := 0; i < len(webhooks); i++ {
+			if webhooks[i].Batch == 0 {
+				webhooks[i].Batch = 1000
+			}
+
+			if webhooks[i].Timeout == 0 {
+				webhooks[i].Timeout = 10
+			}
+
+			if webhooks[i].RetryCount == 0 {
+				webhooks[i].RetryCount = 10
+			}
+
+			if webhooks[i].RetryInterval == 0 {
+				webhooks[i].RetryInterval = 10
+			}
+
+			if webhooks[i].Client == nil {
+				transport := &http.Transport{
+					TLSClientConfig: &tls.Config{InsecureSkipVerify: webhooks[i].SkipVerify},
+				}
+				if poster.UseProxy(webhooks[i].Url) {
+					transport.Proxy = http.ProxyFromEnvironment
+				}
+				webhooks[i].Client = &http.Client{
+					Timeout:   time.Second * time.Duration(webhooks[i].Timeout),
+					Transport: transport,
+				}
+			}
+
+			newWebhooks[webhooks[i].Url] = webhooks[i]
+		}
+
+		for url, wh := range newWebhooks {
+			if oldWh, has := w.webhooks[url]; has && oldWh.Hash() != wh.Hash() {
+				w.webhooks[url] = wh
+			} else {
+				w.webhooks[url] = wh
+			}
+		}
+
+		for url := range w.webhooks {
+			if _, has := newWebhooks[url]; !has {
+				delete(w.webhooks, url)
+			}
+		}
 	}
 
 	dumper.PutSyncRecord("webhooks", start.Unix(), time.Since(start).Milliseconds(), len(w.webhooks), "success, webhooks:\n"+cval)
@@ -133,7 +186,7 @@ func (w *NotifyConfigCacheType) syncNotifyConfigs() error {
 	return nil
 }
 
-func (w *NotifyConfigCacheType) GetWebhooks() []*models.Webhook {
+func (w *NotifyConfigCacheType) GetWebhooks() map[string]*models.Webhook {
 	w.RWMutex.RLock()
 	defer w.RWMutex.RUnlock()
 	return w.webhooks

models/alert_rule.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/poster"
 	"github.com/ccfos/nightingale/v6/pushgw/pconf"
+	"github.com/robfig/cron/v3"
 
 	"github.com/jinzhu/copier"
 	"github.com/pkg/errors"
@@ -27,6 +28,8 @@ const (
 	PROMETHEUS    = "prometheus"
 	TDENGINE      = "tdengine"
 	ELASTICSEARCH = "elasticsearch"
+
+	CLICKHOUSE = "ck"
 )
 
 const (
@@ -158,9 +161,9 @@ type PromRuleConfig struct {
 type RecoverJudge int
 
 const (
-	Origin             RecoverJudge = 0
-	RecoverWithoutData RecoverJudge = 1
-	RecoverOnCondition RecoverJudge = 2
+	Origin               RecoverJudge = 0
+	NotRecoverWhenNoData RecoverJudge = 1
+	RecoverOnCondition   RecoverJudge = 2
 )
 
 type RecoverConfig struct {
@@ -191,10 +194,20 @@ type HostTrigger struct {
 }
 
 type RuleQuery struct {
-	Version  string        `json:"version"`
-	Inhibit  bool          `json:"inhibit"`
-	Queries  []interface{} `json:"queries"`
-	Triggers []Trigger     `json:"triggers"`
+	Version           string        `json:"version"`
+	Inhibit           bool          `json:"inhibit"`
+	Queries           []interface{} `json:"queries"`
+	ExpTriggerDisable bool          `json:"exp_trigger_disable"`
+	Triggers          []Trigger     `json:"triggers"`
+	NodataTrigger     NodataTrigger `json:"nodata_trigger"`
+	TriggerType       TriggerType   `json:"trigger_type,omitempty"`
+}
+
+type NodataTrigger struct {
+	Enable             bool `json:"enable"`
+	Severity           int  `json:"severity"`
+	ResolveAfterEnable bool `json:"resolve_after_enable"`
+	ResolveAfter       int  `json:"resolve_after"` // 单位秒
 }
 
 type Trigger struct {
@@ -372,12 +385,14 @@ func GetHostsQuery(queries []HostQuery) []map[string]interface{} {
 					blank += " "
 				}
 			} else {
-				blank := " "
+				var args []interface{}
+				var query []string
 				for _, tag := range lst {
-					m["tags not like ?"+blank] = "%" + tag + "%"
-					m["host_tags not like ?"+blank] = "%" + tag + "%"
-					blank += " "
+					query = append(query, "tags not like ?",
+						"(host_tags not like ? or host_tags is null)")
+					args = append(args, "%"+tag+"%", "%"+tag+"%")
 				}
+				m[strings.Join(query, " and ")] = args
 			}
 		case "hosts":
 			lst := []string{}
@@ -398,11 +413,13 @@ func GetHostsQuery(queries []HostQuery) []map[string]interface{} {
 					blank += " "
 				}
 			} else if q.Op == "!~" {
-				blank := " "
+				var args []interface{}
+				var query []string
 				for _, host := range lst {
-					m["ident not like ?"+blank] = strings.ReplaceAll(host, "*", "%")
-					blank += " "
+					query = append(query, "ident not like ?")
+					args = append(args, strings.ReplaceAll(host, "*", "%"))
 				}
+				m[strings.Join(query, " and ")] = args
 			}
 		}
 		query = append(query, m)
@@ -489,6 +506,27 @@ func (ar *AlertRule) Verify() error {
 		}
 	}
 
+	if err := ar.validateCronPattern(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ar *AlertRule) validateCronPattern() error {
+	if ar.CronPattern == "" {
+		return nil
+	}
+
+	// 创建一个临时的 cron scheduler 来验证表达式
+	scheduler := cron.New(cron.WithSeconds())
+
+	// 尝试添加一个空函数来验证 cron 表达式
+	_, err := scheduler.AddFunc(ar.CronPattern, func() {})
+	if err != nil {
+		return fmt.Errorf("invalid cron pattern: %s, error: %v", ar.CronPattern, err)
+	}
+
 	return nil
 }
 
@@ -1120,6 +1158,14 @@ func (ar *AlertRule) GetRuleType() string {
 	return ar.Prod
 }
 
+func (ar *AlertRule) IsClickHouseRule() bool {
+	return ar.Cate == CLICKHOUSE
+}
+
+func (ar *AlertRule) IsElasticSearch() bool {
+	return ar.Cate == ELASTICSEARCH
+}
+
 func (ar *AlertRule) GenerateNewEvent(ctx *ctx.Context) *AlertCurEvent {
 	event := &AlertCurEvent{}
 	ar.UpdateEvent(event)
@@ -1139,8 +1185,6 @@ func (ar *AlertRule) UpdateEvent(event *AlertCurEvent) {
 	event.RuleProd = ar.Prod
 	event.RuleAlgo = ar.Algorithm
 	event.PromForDuration = ar.PromForDuration
-	event.RuleConfig = ar.RuleConfig
-	event.RuleConfigJson = ar.RuleConfigJson
 	event.Callbacks = ar.Callbacks
 	event.CallbacksJSON = ar.CallbacksJSON
 	event.RunbookUrl = ar.RunbookUrl

models/anomaly_point.go

@@ -20,8 +20,16 @@ type AnomalyPoint struct {
 	Values        string                         `json:"values"`
 	ValuesUnit    map[string]unit.FormattedValue `json:"values_unit"`
 	RecoverConfig RecoverConfig                  `json:"recover_config"`
+	TriggerType   TriggerType                    `json:"trigger_type"`
 }
 
+type TriggerType string
+
+const (
+	TriggerTypeNormal TriggerType = "normal"
+	TriggerTypeNodata TriggerType = "nodata"
+)
+
 func NewAnomalyPoint(key string, labels map[string]string, ts int64, value float64, severity int) AnomalyPoint {
 	anomalyPointLabels := make(model.Metric)
 	for k, v := range labels {

models/builtin_component.go

@@ -11,9 +11,10 @@ import (
 // BuiltinComponent represents a builtin component along with its metadata.
 type BuiltinComponent struct {
 	ID        uint64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
-	Ident     string `json:"ident" gorm:"type:varchar(191);not null;uniqueIndex:idx_ident,sort:asc"`
+	Ident     string `json:"ident" gorm:"type:varchar(191);not null;index:idx_ident"`
 	Logo      string `json:"logo" gorm:"type:mediumtext;comment:'logo of component'"`
 	Readme    string `json:"readme" gorm:"type:text;not null;comment:'readme of component'"`
+	Disabled  int    `json:"disabled" gorm:"type:int;not null;default:0;comment:'is disabled or not'"`
 	CreatedAt int64  `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
 	CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
 	UpdatedAt int64  `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
@@ -22,15 +23,20 @@ type BuiltinComponent struct {
 
 type PostgresBuiltinComponent struct {
 	ID        uint64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
-	Ident     string `json:"ident" gorm:"type:varchar(191);not null;uniqueIndex:idx_ident,sort:asc;comment:'identifier of component'"`
+	Ident     string `json:"ident" gorm:"type:varchar(191);not null;index:idx_ident;comment:'identifier of component'"`
 	Logo      string `json:"logo" gorm:"type:text;comment:'logo of component'"`
 	Readme    string `json:"readme" gorm:"type:text;not null;comment:'readme of component'"`
+	Disabled  int    `json:"disabled" gorm:"type:int;not null;default:0;comment:'is disabled or not'"`
 	CreatedAt int64  `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
 	CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
 	UpdatedAt int64  `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
 	UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
 }
 
+func (bc *PostgresBuiltinComponent) TableName() string {
+	return "builtin_components"
+}
+
 func (bc *BuiltinComponent) TableName() string {
 	return "builtin_components"
 }
@@ -97,12 +103,15 @@ func BuiltinComponentDels(ctx *ctx.Context, ids []int64) error {
 	return DB(ctx).Where("id in ?", ids).Delete(new(BuiltinComponent)).Error
 }
 
-func BuiltinComponentGets(ctx *ctx.Context, query string) ([]*BuiltinComponent, error) {
+func BuiltinComponentGets(ctx *ctx.Context, query string, disabled int) ([]*BuiltinComponent, error) {
 	session := DB(ctx)
 	if query != "" {
 		queryPattern := "%" + query + "%"
 		session = session.Where("ident LIKE ?", queryPattern)
 	}
+	if disabled == 0 || disabled == 1 {
+		session = session.Where("disabled = ?", disabled)
+	}
 
 	var lst []*BuiltinComponent
 

models/builtin_payload.go

@@ -168,7 +168,7 @@ func BuiltinPayloadComponents(ctx *ctx.Context, typ, cate string) (string, error
 func InitBuiltinPayloads(ctx *ctx.Context) error {
 	var lst []*BuiltinPayload
 
-	components, err := BuiltinComponentGets(ctx, "")
+	components, err := BuiltinComponentGets(ctx, "", -1)
 	if err != nil {
 		return err
 	}

models/busi_group.go

@@ -115,68 +115,75 @@ func BusiGroupExists(ctx *ctx.Context, where string, args ...interface{}) (bool,
 	return num > 0, err
 }
 
+// RegisterGroupDelCheckEntries 提供给外部注册删除 group 时需要检查的表
+func RegisterGroupDelCheckEntries(e []CheckEntry) {
+	entries = append(entries, e...)
+}
+
+type CheckEntry struct {
+	Entry        interface{}
+	ErrorMessage string
+	FieldName    string
+}
+
+var entries = []CheckEntry{
+	{
+		Entry:        &AlertRule{},
+		ErrorMessage: "Some alert rules still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &AlertMute{},
+		ErrorMessage: "Some alert mutes still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &AlertSubscribe{},
+		ErrorMessage: "Some alert subscribes still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &Board{},
+		ErrorMessage: "Some Board still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &Target{},
+		ErrorMessage: "Some targets still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &RecordingRule{},
+		ErrorMessage: "Some recording rules still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &TaskTpl{},
+		ErrorMessage: "Some recovery scripts still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &TaskRecord{},
+		ErrorMessage: "Some Task Record records still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+	{
+		Entry:        &TargetBusiGroup{},
+		ErrorMessage: "Some target busigroups still in the BusiGroup",
+		FieldName:    "group_id",
+	},
+}
+
 func (bg *BusiGroup) Del(ctx *ctx.Context) error {
-	has, err := Exists(DB(ctx).Model(&AlertMute{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
+	for _, e := range entries {
+		has, err := Exists(DB(ctx).Model(e.Entry).Where(fmt.Sprintf("%s=?", e.FieldName), bg.Id))
+		if err != nil {
+			return err
+		}
 
-	if has {
-		return errors.New("Some alert mutes still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&AlertSubscribe{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some alert subscribes still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&TargetBusiGroup{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some targets still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&Board{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some dashboards still in the BusiGroup")
-	}
-
-	has, err = Exists(DB(ctx).Model(&TaskTpl{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some recovery scripts still in the BusiGroup")
-	}
-
-	// hasCR, err := Exists(DB(ctx).Table("collect_rule").Where("group_id=?", bg.Id))
-	// if err != nil {
-	// 	return err
-	// }
-
-	// if hasCR {
-	// 	return errors.New("Some collect rules still in the BusiGroup")
-	// }
-
-	has, err = Exists(DB(ctx).Model(&AlertRule{}).Where("group_id=?", bg.Id))
-	if err != nil {
-		return err
-	}
-
-	if has {
-		return errors.New("Some alert rules still in the BusiGroup")
+		if has {
+			return errors.New(e.ErrorMessage)
+		}
 	}
 
 	return DB(ctx).Transaction(func(tx *gorm.DB) error {

models/configs.go

@@ -130,7 +130,7 @@ func ConfigsGetAll(ctx *ctx.Context) ([]*Configs, error) { // select built-in ty
 	}
 
 	var lst []*Configs
-	err := DB(ctx).Model(&Configs{}).Select("ckey, cval").
+	err := DB(ctx).Model(&Configs{}).Select("id, ckey, cval").
 		Where("ckey!='' and external=? ", 0).Find(&lst).Error
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed to query configs")

models/dash_annotation.go

@@ -0,0 +1,89 @@
+package models
+
+import (
+	"encoding/json"
+
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
+)
+
+type DashAnnotation struct {
+	Id          int64    `json:"id" gorm:"primaryKey"`
+	DashboardId int64    `json:"dashboard_id"`
+	PanelId     string   `json:"panel_id"`
+	Tags        string   `json:"-"`
+	TagsJSON    []string `json:"tags" gorm:"-"`
+	Description string   `json:"description"`
+	Config      string   `json:"config"`
+	TimeStart   int64    `json:"time_start"`
+	TimeEnd     int64    `json:"time_end"`
+	CreateAt    int64    `json:"create_at"`
+	CreateBy    string   `json:"create_by"`
+	UpdateAt    int64    `json:"update_at"`
+	UpdateBy    string   `json:"update_by"`
+}
+
+func (da *DashAnnotation) TableName() string {
+	return "dash_annotation"
+}
+
+func (da *DashAnnotation) DB2FE() error {
+	return json.Unmarshal([]byte(da.Tags), &da.TagsJSON)
+}
+
+func (da *DashAnnotation) FE2DB() error {
+	b, err := json.Marshal(da.TagsJSON)
+	if err != nil {
+		return err
+	}
+	da.Tags = string(b)
+	return nil
+}
+
+func (da *DashAnnotation) Add(ctx *ctx.Context) error {
+	if err := da.FE2DB(); err != nil {
+		return err
+	}
+	return Insert(ctx, da)
+}
+
+func (da *DashAnnotation) Update(ctx *ctx.Context) error {
+	if err := da.FE2DB(); err != nil {
+		return err
+	}
+	return DB(ctx).Model(da).Select("dashboard_id", "panel_id", "tags", "description", "config", "time_start", "time_end", "update_at", "update_by").Updates(da).Error
+}
+
+func DashAnnotationDel(ctx *ctx.Context, id int64) error {
+	return DB(ctx).Where("id = ?", id).Delete(&DashAnnotation{}).Error
+}
+
+func DashAnnotationGet(ctx *ctx.Context, where string, args ...interface{}) (*DashAnnotation, error) {
+	var lst []*DashAnnotation
+	err := DB(ctx).Where(where, args...).Find(&lst).Error
+	if err != nil {
+		return nil, err
+	}
+
+	if len(lst) == 0 {
+		return nil, nil
+	}
+
+	err = lst[0].DB2FE()
+	return lst[0], err
+}
+
+func DashAnnotationGets(ctx *ctx.Context, dashboardId int64, from, to int64, limit int) ([]DashAnnotation, error) {
+	session := DB(ctx).Where("dashboard_id = ? AND time_start <= ? AND time_end >= ?", dashboardId, to, from)
+
+	var lst []DashAnnotation
+	err := session.Order("id").Limit(limit).Find(&lst).Error
+	if err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(lst); i++ {
+		lst[i].DB2FE()
+	}
+
+	return lst, nil
+}

models/es_index_pattern.go

@@ -20,6 +20,7 @@ type EsIndexPattern struct {
 	CreateBy                   string `json:"create_by"`
 	UpdateAt                   int64  `json:"update_at"`
 	UpdateBy                   string `json:"update_by"`
+	CrossClusterEnabled        int    `json:"cross_cluster_enabled"`
 }
 
 func (t *EsIndexPattern) TableName() string {

models/migrate/migrate.go

@@ -67,7 +67,7 @@ func MigrateTables(db *gorm.DB) error {
 		&TaskRecord{}, &ChartShare{}, &Target{}, &Configs{}, &Datasource{}, &NotifyTpl{},
 		&Board{}, &BoardBusigroup{}, &Users{}, &SsoConfig{}, &models.BuiltinMetric{},
 		&models.MetricFilter{}, &models.NotificaitonRecord{},
-		&models.TargetBusiGroup{}}
+		&models.TargetBusiGroup{}, &EsIndexPatternMigrate{}, &DashAnnotation{}}
 
 	if isPostgres(db) {
 		dts = append(dts, &models.PostgresBuiltinComponent{})
@@ -92,7 +92,7 @@ func MigrateTables(db *gorm.DB) error {
 
 			for _, dt := range asyncDts {
 				if err := db.AutoMigrate(dt); err != nil {
-					logger.Errorf("failed to migrate table: %v", err)
+					logger.Errorf("failed to migrate table %+v err:%v", dt, err)
 				}
 			}
 		}()
@@ -188,14 +188,20 @@ func InsertPermPoints(db *gorm.DB) {
 	})
 
 	for _, op := range ops {
-		exists, err := models.Exists(db.Model(&models.RoleOperation{}).Where("operation = ? and role_name = ?", op.Operation, op.RoleName))
+		var count int64
+
+		err := db.Raw("SELECT COUNT(*) FROM role_operation WHERE operation = ? AND role_name = ?",
+			op.Operation, op.RoleName).Scan(&count).Error
+
 		if err != nil {
 			logger.Errorf("check role operation exists failed, %v", err)
 			continue
 		}
-		if exists {
+
+		if count > 0 {
 			continue
 		}
+
 		err = db.Create(&op).Error
 		if err != nil {
 			logger.Errorf("insert role operation failed, %v", err)
@@ -313,3 +319,30 @@ type TaskHostDoing struct {
 func (TaskHostDoing) TableName() string {
 	return "task_host_doing"
 }
+
+type EsIndexPatternMigrate struct {
+	CrossClusterEnabled int `gorm:"column:cross_cluster_enabled;type:int;default:0"`
+}
+
+func (EsIndexPatternMigrate) TableName() string {
+	return "es_index_pattern"
+}
+
+type DashAnnotation struct {
+	Id          int64  `gorm:"column:id;primaryKey;autoIncrement"`
+	DashboardId int64  `gorm:"column:dashboard_id;not null"`
+	PanelId     string `gorm:"column:panel_id;type:varchar(191);not null"`
+	Tags        string `gorm:"column:tags;type:text"`
+	Description string `gorm:"column:description;type:text"`
+	Config      string `gorm:"column:config;type:text"`
+	TimeStart   int64  `gorm:"column:time_start;not null;default:0"`
+	TimeEnd     int64  `gorm:"column:time_end;not null;default:0"`
+	CreateAt    int64  `gorm:"column:create_at;not null;default:0"`
+	CreateBy    string `gorm:"column:create_by;type:varchar(64);not null;default:''"`
+	UpdateAt    int64  `gorm:"column:update_at;not null;default:0"`
+	UpdateBy    string `gorm:"column:update_by;type:varchar(64);not null;default:''"`
+}
+
+func (DashAnnotation) TableName() string {
+	return "dash_annotation"
+}

models/migrate/migrate_test.go

@@ -0,0 +1,69 @@
+package migrate
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/ccfos/nightingale/v6/models"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/schema"
+)
+
+func TestInsertPermPoints(t *testing.T) {
+	db, err := gorm.Open(mysql.Open("root:1234@tcp(127.0.0.1:3306)/n9e_v6?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true"), &gorm.Config{NamingStrategy: schema.NamingStrategy{
+		SingularTable: true,
+	}})
+	if err != nil {
+		fmt.Printf("failed to connect database: %v", err)
+	}
+
+	var ops []models.RoleOperation
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/alert-mutes/put",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/log/index-patterns",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/help/variable-configs",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Admin",
+		Operation: "/permissions",
+	})
+
+	ops = append(ops, models.RoleOperation{
+		RoleName:  "Standard",
+		Operation: "/ibex-settings",
+	})
+
+	db = db.Debug()
+	for _, op := range ops {
+		var count int64
+
+		err := db.Raw("SELECT COUNT(*) FROM role_operation WHERE operation = ? AND role_name = ?",
+			op.Operation, op.RoleName).Scan(&count).Error
+		fmt.Printf("count: %d\n", count)
+
+		if err != nil {
+			fmt.Printf("check role operation exists failed, %v", err)
+			continue
+		}
+
+		if count > 0 {
+			continue
+		}
+
+		err = db.Create(&op).Error
+		if err != nil {
+			fmt.Printf("insert role operation failed, %v", err)
+		}
+	}
+}

models/notify_config.go

@@ -1,5 +1,12 @@
 package models
 
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/toolkits/pkg/str"
+)
+
 const WEBHOOKKEY = "webhook"
 const NOTIFYSCRIPT = "notify_script"
 const NOTIFYCHANNEL = "notify_channel"
@@ -24,6 +31,11 @@ type Webhook struct {
 	RetryCount    int               `json:"retry_count"`
 	RetryInterval int               `json:"retry_interval"`
 	Batch         int               `json:"batch"`
+	Client        *http.Client      `json:"-"`
+}
+
+func (w *Webhook) Hash() string {
+	return str.MD5(fmt.Sprintf("%d_%t_%s_%s_%s_%d_%v_%t_%s_%d_%d_%d", w.Type, w.Enable, w.Url, w.BasicAuthUser, w.BasicAuthPass, w.Timeout, w.HeaderMap, w.SkipVerify, w.Note, w.RetryCount, w.RetryInterval, w.Batch))
 }
 
 type NotifyScript struct {

models/target.go

@@ -185,8 +185,16 @@ func BuildTargetWhereWithQuery(query string) BuildTargetWhereOption {
 		if query != "" {
 			arr := strings.Fields(query)
 			for i := 0; i < len(arr); i++ {
-				q := "%" + arr[i] + "%"
-				session = session.Where("ident like ? or host_ip like ? or note like ? or tags like ? or host_tags like ? or os like ?", q, q, q, q, q, q)
+				if strings.HasPrefix(arr[i], "-") {
+					q := "%" + arr[i][1:] + "%"
+					session = session.Where("ident not like ? and host_ip not like ? and "+
+						"note not like ? and tags not like ? and (host_tags not like ? or "+
+						"host_tags is null) and os not like ?", q, q, q, q, q, q)
+				} else {
+					q := "%" + arr[i] + "%"
+					session = session.Where("ident like ? or host_ip like ? or note like ? or "+
+						"tags like ? or host_tags like ? or os like ?", q, q, q, q, q, q)
+				}
 			}
 		}
 		return session
@@ -197,6 +205,8 @@ func BuildTargetWhereWithDowntime(downtime int64) BuildTargetWhereOption {
 	return func(session *gorm.DB) *gorm.DB {
 		if downtime > 0 {
 			session = session.Where("target.update_at < ?", time.Now().Unix()-downtime)
+		} else if downtime < 0 {
+			session = session.Where("target.update_at > ?", time.Now().Unix()+downtime)
 		}
 		return session
 	}
@@ -270,7 +280,11 @@ func TargetFilterQueryBuild(ctx *ctx.Context, query []map[string]interface{}, li
 	for _, q := range query {
 		tx := DB(ctx).Model(&Target{})
 		for k, v := range q {
-			tx = tx.Or(k, v)
+			if strings.Count(k, "?") > 1 {
+				tx = tx.Or(k, v.([]interface{})...)
+			} else {
+				tx = tx.Or(k, v)
+			}
 		}
 		sub = sub.Where(tx)
 	}

models/ts.go

@@ -53,6 +53,12 @@ func (d *DataResp) MetricName() string {
 	return string(metric)
 }
 
+// labels 转换为 string
+func (d *DataResp) LabelsString() string {
+	labels := d.Metric
+	return labels.String()
+}
+
 type RelationKey struct {
 	LeftKey  string `json:"left_key"`
 	RightKey string `json:"right_key"`

pkg/cfg/cfg.go

@@ -29,16 +29,22 @@ func LoadConfigByDir(configDir string, configPtr interface{}) error {
 	if err != nil {
 		return fmt.Errorf("failed to list files under: %s : %v", configDir, err)
 	}
+
+	found := false
+
 	s := NewFileScanner()
 	for _, fpath := range files {
 		switch {
 		case strings.HasSuffix(fpath, ".toml"):
+			found = true
 			s.Read(path.Join(configDir, fpath))
 			tBuf = append(tBuf, s.Data()...)
 			tBuf = append(tBuf, []byte("\n")...)
 		case strings.HasSuffix(fpath, ".json"):
+			found = true
 			loaders = append(loaders, &multiconfig.JSONLoader{Path: path.Join(configDir, fpath)})
 		case strings.HasSuffix(fpath, ".yaml") || strings.HasSuffix(fpath, ".yml"):
+			found = true
 			loaders = append(loaders, &multiconfig.YAMLLoader{Path: path.Join(configDir, fpath)})
 		}
 		if s.Err() != nil {
@@ -46,6 +52,10 @@ func LoadConfigByDir(configDir string, configPtr interface{}) error {
 		}
 	}
 
+	if !found {
+		return fmt.Errorf("fail to found config file, config dir path: %v", configDir)
+	}
+
 	if len(tBuf) != 0 {
 		loaders = append(loaders, &multiconfig.TOMLLoader{Reader: bytes.NewReader(tBuf)})
 	}

pkg/macros/macros.go

@@ -0,0 +1,11 @@
+package macros
+
+var Macro func(sql string, start, end int64) (string, error)
+
+func RegisterMacro(f func(sql string, start, end int64) (string, error)) {
+	Macro = f
+}
+
+func MacroInVain(sql string, start, end int64) (string, error) {
+	return sql, nil
+}

pkg/ormx/database_init.go

@@ -340,42 +340,43 @@ func (InitChartShare) TableOptions() string {
 }
 
 type InitAlertRule struct {
-	ID               uint64 `gorm:"primaryKey;autoIncrement"`
-	GroupID          uint64 `gorm:"not null;default:0;comment:busi group id;index"`
-	Cate             string `gorm:"size:128;not null"`
-	DatasourceIDs    string `gorm:"size:255;not null;default:'';comment:datasource ids"`
-	Cluster          string `gorm:"size:128;not null"`
-	Name             string `gorm:"size:255;not null"`
-	Note             string `gorm:"size:1024;not null;default:''"`
-	Prod             string `gorm:"size:255;not null;default:''"`
-	Algorithm        string `gorm:"size:255;not null;default:''"`
-	AlgoParams       string `gorm:"size:255"`
-	Delay            int32  `gorm:"not null;default:0"`
-	Severity         int16  `gorm:"type:tinyint(1);not null;comment:1:Emergency 2:Warning 3:Notice"`
-	Disabled         bool   `gorm:"type:tinyint(1);not null;comment:0:enabled 1:disabled"`
-	PromForDuration  int32  `gorm:"not null;comment:prometheus for, unit:s"`
-	RuleConfig       string `gorm:"type:text;not null;comment:rule_config"`
-	PromQL           string `gorm:"type:text;not null;comment:promql"`
-	PromEvalInterval int32  `gorm:"not null;comment:evaluate interval"`
-	EnableStime      string `gorm:"size:255;not null;default:'00:00'"`
-	EnableEtime      string `gorm:"size:255;not null;default:'23:59'"`
-	EnableDaysOfWeek string `gorm:"size:255;not null;default:'';comment:split by space: 0 1 2 3 4 5 6"`
-	EnableInBg       bool   `gorm:"type:tinyint(1);not null;default:0;comment:1: only this bg 0: global"`
-	NotifyRecovered  bool   `gorm:"type:tinyint(1);not null;comment:whether notify when recovery"`
-	NotifyChannels   string `gorm:"size:255;not null;default:'';comment:split by space: sms voice email dingtalk wecom"`
-	NotifyGroups     string `gorm:"size:255;not null;default:'';comment:split by space: 233 43"`
-	NotifyRepeatStep int32  `gorm:"not null;default:0;comment:unit: min"`
-	NotifyMaxNumber  int32  `gorm:"not null;default:0"`
-	RecoverDuration  int32  `gorm:"not null;default:0;comment:unit: s"`
-	Callbacks        string `gorm:"size:4096;not null;default:'';comment:split by space: http://a.com/api/x http://a.com/api/y"`
-	RunbookURL       string `gorm:"size:4096"`
-	AppendTags       string `gorm:"size:255;not null;default:'';comment:split by space: service=n9e mod=api"`
-	Annotations      string `gorm:"type:text;not null;comment:annotations"`
-	ExtraConfig      string `gorm:"type:text;not null;comment:extra_config"`
-	CreateAt         int64  `gorm:"not null;default:0"`
-	CreateBy         string `gorm:"size:64;not null;default:''"`
-	UpdateAt         int64  `gorm:"not null;default:0;index"`
-	UpdateBy         string `gorm:"size:64;not null;default:''"`
+	ID                uint64 `gorm:"primaryKey;autoIncrement"`
+	GroupID           uint64 `gorm:"not null;default:0;comment:busi group id;index"`
+	Cate              string `gorm:"size:128;not null"`
+	DatasourceIDs     string `gorm:"size:255;not null;default:'';comment:datasource ids"`
+	Cluster           string `gorm:"size:128;not null"`
+	Name              string `gorm:"size:255;not null"`
+	Note              string `gorm:"size:1024;not null;default:''"`
+	Prod              string `gorm:"size:255;not null;default:''"`
+	Algorithm         string `gorm:"size:255;not null;default:''"`
+	AlgoParams        string `gorm:"size:255"`
+	Delay             int32  `gorm:"not null;default:0"`
+	Severity          int16  `gorm:"type:tinyint(1);not null;comment:1:Emergency 2:Warning 3:Notice"`
+	Disabled          bool   `gorm:"type:tinyint(1);not null;comment:0:enabled 1:disabled"`
+	PromForDuration   int32  `gorm:"not null;comment:prometheus for, unit:s"`
+	RuleConfig        string `gorm:"type:text;not null;comment:rule_config"`
+	PromQL            string `gorm:"type:text;not null;comment:promql"`
+	PromEvalInterval  int32  `gorm:"not null;comment:evaluate interval"`
+	EnableStime       string `gorm:"size:255;not null;default:'00:00'"`
+	EnableEtime       string `gorm:"size:255;not null;default:'23:59'"`
+	EnableDaysOfWeek  string `gorm:"size:255;not null;default:'';comment:split by space: 0 1 2 3 4 5 6"`
+	EnableInBg        bool   `gorm:"type:tinyint(1);not null;default:0;comment:1: only this bg 0: global"`
+	NotifyRecovered   bool   `gorm:"type:tinyint(1);not null;comment:whether notify when recovery"`
+	NotifyChannels    string `gorm:"size:255;not null;default:'';comment:split by space: sms voice email dingtalk wecom"`
+	NotifyGroups      string `gorm:"size:255;not null;default:'';comment:split by space: 233 43"`
+	NotifyRepeatStep  int32  `gorm:"not null;default:0;comment:unit: min"`
+	NotifyMaxNumber   int32  `gorm:"not null;default:0"`
+	RecoverDuration   int32  `gorm:"not null;default:0;comment:unit: s"`
+	Callbacks         string `gorm:"size:4096;not null;default:'';comment:split by space: http://a.com/api/x http://a.com/api/y"`
+	RunbookURL        string `gorm:"size:4096"`
+	AppendTags        string `gorm:"size:255;not null;default:'';comment:split by space: service=n9e mod=api"`
+	Annotations       string `gorm:"type:text;not null;comment:annotations"`
+	ExtraConfig       string `gorm:"type:text;not null;comment:extra_config"`
+	CreateAt          int64  `gorm:"not null;default:0"`
+	CreateBy          string `gorm:"size:64;not null;default:''"`
+	UpdateAt          int64  `gorm:"not null;default:0;index"`
+	UpdateBy          string `gorm:"size:64;not null;default:''"`
+	DatasourceQueries string `gorm:"type:text"`
 }
 
 func (InitAlertRule) TableName() string {
@@ -387,42 +388,43 @@ func (InitAlertRule) TableOptions() string {
 }
 
 type InitPostgresAlertRule struct {
-	ID               uint64 `gorm:"primaryKey;autoIncrement"`
-	GroupID          uint64 `gorm:"not null;default:0;comment:busi group id;index"`
-	Cate             string `gorm:"size:128;not null"`
-	DatasourceIDs    string `gorm:"size:255;not null;default:'';comment:datasource ids"`
-	Cluster          string `gorm:"size:128;not null"`
-	Name             string `gorm:"size:255;not null"`
-	Note             string `gorm:"size:1024;not null;default:''"`
-	Prod             string `gorm:"size:255;not null;default:''"`
-	Algorithm        string `gorm:"size:255;not null;default:''"`
-	AlgoParams       string `gorm:"size:255"`
-	Delay            int32  `gorm:"not null;default:0"`
-	Severity         int16  `gorm:"type:smallint;not null;comment:1:Emergency 2:Warning 3:Notice"`
-	Disabled         int16  `gorm:"type:smallint;not null;comment:0:enabled 1:disabled"`
-	PromForDuration  int32  `gorm:"not null;comment:prometheus for, unit:s"`
-	RuleConfig       string `gorm:"type:text;not null;comment:rule_config"`
-	PromQL           string `gorm:"type:text;not null;comment:promql"`
-	PromEvalInterval int32  `gorm:"not null;comment:evaluate interval"`
-	EnableStime      string `gorm:"size:255;not null;default:'00:00'"`
-	EnableEtime      string `gorm:"size:255;not null;default:'23:59'"`
-	EnableDaysOfWeek string `gorm:"size:255;not null;default:'';comment:split by space: 0 1 2 3 4 5 6"`
-	EnableInBg       int16  `gorm:"type:smallint;not null;default:0;comment:1: only this bg 0: global"`
-	NotifyRecovered  int16  `gorm:"type:smallint;not null;comment:whether notify when recovery"`
-	NotifyChannels   string `gorm:"size:255;not null;default:'';comment:split by space: sms voice email dingtalk wecom"`
-	NotifyGroups     string `gorm:"size:255;not null;default:'';comment:split by space: 233 43"`
-	NotifyRepeatStep int32  `gorm:"not null;default:0;comment:unit: min"`
-	NotifyMaxNumber  int32  `gorm:"not null;default:0"`
-	RecoverDuration  int32  `gorm:"not null;default:0;comment:unit: s"`
-	Callbacks        string `gorm:"size:4096;not null;default:'';comment:split by space: http://a.com/api/x http://a.com/api/y"`
-	RunbookURL       string `gorm:"size:4096"`
-	AppendTags       string `gorm:"size:255;not null;default:'';comment:split by space: service=n9e mod=api"`
-	Annotations      string `gorm:"type:text;not null;comment:annotations"`
-	ExtraConfig      string `gorm:"type:text;not null;comment:extra_config"`
-	CreateAt         int64  `gorm:"not null;default:0"`
-	CreateBy         string `gorm:"size:64;not null;default:''"`
-	UpdateAt         int64  `gorm:"not null;default:0;index"`
-	UpdateBy         string `gorm:"size:64;not null;default:''"`
+	ID                uint64 `gorm:"primaryKey;autoIncrement"`
+	GroupID           uint64 `gorm:"not null;default:0;comment:busi group id;index"`
+	Cate              string `gorm:"size:128;not null"`
+	DatasourceIDs     string `gorm:"size:255;not null;default:'';comment:datasource ids"`
+	Cluster           string `gorm:"size:128;not null"`
+	Name              string `gorm:"size:255;not null"`
+	Note              string `gorm:"size:1024;not null;default:''"`
+	Prod              string `gorm:"size:255;not null;default:''"`
+	Algorithm         string `gorm:"size:255;not null;default:''"`
+	AlgoParams        string `gorm:"size:255"`
+	Delay             int32  `gorm:"not null;default:0"`
+	Severity          int16  `gorm:"type:smallint;not null;comment:1:Emergency 2:Warning 3:Notice"`
+	Disabled          int16  `gorm:"type:smallint;not null;comment:0:enabled 1:disabled"`
+	PromForDuration   int32  `gorm:"not null;comment:prometheus for, unit:s"`
+	RuleConfig        string `gorm:"type:text;not null;comment:rule_config"`
+	PromQL            string `gorm:"type:text;not null;comment:promql"`
+	PromEvalInterval  int32  `gorm:"not null;comment:evaluate interval"`
+	EnableStime       string `gorm:"size:255;not null;default:'00:00'"`
+	EnableEtime       string `gorm:"size:255;not null;default:'23:59'"`
+	EnableDaysOfWeek  string `gorm:"size:255;not null;default:'';comment:split by space: 0 1 2 3 4 5 6"`
+	EnableInBg        int16  `gorm:"type:smallint;not null;default:0;comment:1: only this bg 0: global"`
+	NotifyRecovered   int16  `gorm:"type:smallint;not null;comment:whether notify when recovery"`
+	NotifyChannels    string `gorm:"size:255;not null;default:'';comment:split by space: sms voice email dingtalk wecom"`
+	NotifyGroups      string `gorm:"size:255;not null;default:'';comment:split by space: 233 43"`
+	NotifyRepeatStep  int32  `gorm:"not null;default:0;comment:unit: min"`
+	NotifyMaxNumber   int32  `gorm:"not null;default:0"`
+	RecoverDuration   int32  `gorm:"not null;default:0;comment:unit: s"`
+	Callbacks         string `gorm:"size:4096;not null;default:'';comment:split by space: http://a.com/api/x http://a.com/api/y"`
+	RunbookURL        string `gorm:"size:4096"`
+	AppendTags        string `gorm:"size:255;not null;default:'';comment:split by space: service=n9e mod=api"`
+	Annotations       string `gorm:"type:text;not null;comment:annotations"`
+	ExtraConfig       string `gorm:"type:text;not null;comment:extra_config"`
+	CreateAt          int64  `gorm:"not null;default:0"`
+	CreateBy          string `gorm:"size:64;not null;default:''"`
+	UpdateAt          int64  `gorm:"not null;default:0;index"`
+	UpdateBy          string `gorm:"size:64;not null;default:''"`
+	DatasourceQueries string `gorm:"type:text"`
 }
 
 func (InitPostgresAlertRule) TableName() string {
@@ -612,22 +614,23 @@ func (InitPostgresMetricView) TableName() string {
 }
 
 type InitRecordingRule struct {
-	ID               uint64 `gorm:"primaryKey;autoIncrement"`
-	GroupID          uint64 `gorm:"not null;default:0;comment:group_id;index"`
-	DatasourceIDs    string `gorm:"size:255;not null;default:'';comment:datasource ids"`
-	Cluster          string `gorm:"size:128;not null"`
-	Name             string `gorm:"size:255;not null;comment:new metric name"`
-	Note             string `gorm:"size:255;not null;comment:rule note"`
-	Disabled         bool   `gorm:"type:tinyint(1);not null;default:0;comment:0:enabled 1:disabled"`
-	PromQL           string `gorm:"size:8192;not null;comment:promql"`
-	PromEvalInterval int32  `gorm:"not null;comment:evaluate interval"`
-	CronPattern      string `gorm:"size:255;default:'';comment:cron pattern"`
-	AppendTags       string `gorm:"size:255;default:'';comment:split by space: service=n9e mod=api"`
-	QueryConfigs     string `gorm:"type:text;not null;comment:query configs"`
-	CreateAt         int64  `gorm:"default:0"`
-	CreateBy         string `gorm:"size:64;default:''"`
-	UpdateAt         int64  `gorm:"default:0;index"`
-	UpdateBy         string `gorm:"size:64;default:''"`
+	ID                uint64 `gorm:"primaryKey;autoIncrement"`
+	GroupID           uint64 `gorm:"not null;default:0;comment:group_id;index"`
+	DatasourceIDs     string `gorm:"size:255;not null;default:'';comment:datasource ids"`
+	Cluster           string `gorm:"size:128;not null"`
+	Name              string `gorm:"size:255;not null;comment:new metric name"`
+	Note              string `gorm:"size:255;not null;comment:rule note"`
+	Disabled          bool   `gorm:"type:tinyint(1);not null;default:0;comment:0:enabled 1:disabled"`
+	PromQL            string `gorm:"size:8192;not null;comment:promql"`
+	PromEvalInterval  int32  `gorm:"not null;comment:evaluate interval"`
+	CronPattern       string `gorm:"size:255;default:'';comment:cron pattern"`
+	AppendTags        string `gorm:"size:255;default:'';comment:split by space: service=n9e mod=api"`
+	QueryConfigs      string `gorm:"type:text;not null;comment:query configs"`
+	CreateAt          int64  `gorm:"default:0"`
+	CreateBy          string `gorm:"size:64;default:''"`
+	UpdateAt          int64  `gorm:"default:0;index"`
+	UpdateBy          string `gorm:"size:64;default:''"`
+	DatasourceQueries string `gorm:"type:text"`
 }
 
 func (InitRecordingRule) TableName() string {
@@ -639,22 +642,23 @@ func (InitRecordingRule) TableOptions() string {
 }
 
 type InitPostgresRecordingRule struct {
-	ID               uint64 `gorm:"primaryKey;autoIncrement"`
-	GroupID          uint64 `gorm:"not null;default:0;comment:group_id;index"`
-	DatasourceIDs    string `gorm:"size:255;not null;default:'';comment:datasource ids"`
-	Cluster          string `gorm:"size:128;not null"`
-	Name             string `gorm:"size:255;not null;comment:new metric name"`
-	Note             string `gorm:"size:255;not null;comment:rule note"`
-	Disabled         int16  `gorm:"type:smallint;not null;default:0;comment:0:enabled 1:disabled"`
-	PromQL           string `gorm:"size:8192;not null;comment:promql"`
-	PromEvalInterval int32  `gorm:"not null;comment:evaluate interval"`
-	CronPattern      string `gorm:"size:255;default:'';comment:cron pattern"`
-	AppendTags       string `gorm:"size:255;default:'';comment:split by space: service=n9e mod=api"`
-	QueryConfigs     string `gorm:"type:text;not null;comment:query configs"`
-	CreateAt         int64  `gorm:"default:0"`
-	CreateBy         string `gorm:"size:64;default:''"`
-	UpdateAt         int64  `gorm:"default:0;index"`
-	UpdateBy         string `gorm:"size:64;default:''"`
+	ID                uint64 `gorm:"primaryKey;autoIncrement"`
+	GroupID           uint64 `gorm:"not null;default:0;comment:group_id;index"`
+	DatasourceIDs     string `gorm:"size:255;not null;default:'';comment:datasource ids"`
+	Cluster           string `gorm:"size:128;not null"`
+	Name              string `gorm:"size:255;not null;comment:new metric name"`
+	Note              string `gorm:"size:255;not null;comment:rule note"`
+	Disabled          int16  `gorm:"type:smallint;not null;default:0;comment:0:enabled 1:disabled"`
+	PromQL            string `gorm:"size:8192;not null;comment:promql"`
+	PromEvalInterval  int32  `gorm:"not null;comment:evaluate interval"`
+	CronPattern       string `gorm:"size:255;default:'';comment:cron pattern"`
+	AppendTags        string `gorm:"size:255;default:'';comment:split by space: service=n9e mod=api"`
+	QueryConfigs      string `gorm:"type:text;not null;comment:query configs"`
+	CreateAt          int64  `gorm:"default:0"`
+	CreateBy          string `gorm:"size:64;default:''"`
+	UpdateAt          int64  `gorm:"default:0;index"`
+	UpdateBy          string `gorm:"size:64;default:''"`
+	DatasourceQueries string `gorm:"type:text"`
 }
 
 func (InitPostgresRecordingRule) TableName() string {
@@ -1207,6 +1211,7 @@ type InitBuiltinMetric struct {
 	CreatedBy  string `gorm:"size:191;not null;default:'';comment:creator"`
 	UpdatedAt  int64  `gorm:"not null;default:0;comment:update time"`
 	UpdatedBy  string `gorm:"size:191;not null;default:'';comment:updater"`
+	UUID       int64  `gorm:"not null;default:0;comment:'uuid'"`
 }
 
 func (InitBuiltinMetric) TableName() string {
@@ -1230,6 +1235,7 @@ type InitSqliteBuiltinMetric struct {
 	CreatedBy  string `gorm:"size:191;not null;default:'';comment:creator"`
 	UpdatedAt  int64  `gorm:"not null;default:0;comment:update time"`
 	UpdatedBy  string `gorm:"size:191;not null;default:'';comment:updater"`
+	UUID       int64  `gorm:"not null;default:0;comment:'uuid'"`
 }
 
 func (InitSqliteBuiltinMetric) TableName() string {

pkg/ormx/database_init_test.go

@@ -1,6 +1,7 @@
 package ormx
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -10,6 +11,43 @@ import (
 	"gorm.io/gorm"
 )
 
+func TestCheckPostgresDatabaseExist(t *testing.T) {
+	tests := []struct {
+		name   string
+		config DBConfig
+	}{
+		{
+			name: "MySQL",
+			config: DBConfig{
+				DBType: "mysql",
+				DSN:    "root:1234@tcp(127.0.0.1:3306)/test?charset=utf8mb4&parseTime=True&loc=Local&allowNativePasswords=true",
+			},
+		},
+		{
+			name: "Postgres",
+			config: DBConfig{
+				DBType: "postgres",
+				DSN:    "host=127.0.0.1 port=5432 user=root dbname=n9e_v6 password=1234 sslmode=disable",
+			},
+		},
+		{
+			name: "SQLite",
+			config: DBConfig{
+				DBType: "sqlite",
+				DSN:    "./test.db",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			exist, err := checkPostgresDatabaseExist(tt.config)
+			fmt.Printf("exitst: %v", exist)
+			assert.NoError(t, err)
+		})
+	}
+}
+
 func TestDataBaseInit(t *testing.T) {
 	tests := []struct {
 		name   string

pkg/ormx/ormx.go

@@ -7,10 +7,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/glebarez/sqlite"
 	tklog "github.com/toolkits/pkg/logger"
 	"gorm.io/driver/mysql"
 	"gorm.io/driver/postgres"
-	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
@@ -91,7 +91,7 @@ func createSqliteDatabase(dsn string, gconfig *gorm.Config) error {
 	if err != nil {
 		return fmt.Errorf("failed to open temporary connection: %v", err)
 	}
-	
+
 	fmt.Println("sqlite file created")
 
 	return nil
@@ -110,7 +110,7 @@ func createPostgresDatabase(dsn string, gconfig *gorm.Config) error {
 		}
 	}
 
-	createDBQuery := fmt.Sprintf("CREATE DATABASE %s ENCODING='UTF8' LC_COLLATE='en_US.UTF-8' LC_CTYPE='en_US.UTF-8';", dbName)
+	createDBQuery := fmt.Sprintf("CREATE DATABASE %s ENCODING='UTF8' LC_COLLATE='en_US.utf8' LC_CTYPE='en_US.utf8';", dbName)
 
 	tempDialector := postgres.Open(connectionWithoutDB)
 
@@ -187,18 +187,16 @@ func checkSqliteDatabaseExist(c DBConfig) (bool, error) {
 
 func checkPostgresDatabaseExist(c DBConfig) (bool, error) {
 	dsnParts := strings.Split(c.DSN, " ")
-	dbName := ""
-	connectionWithoutDB := ""
-	for _, part := range dsnParts {
-		if strings.HasPrefix(part, "dbname=") {
-			dbName = part[strings.Index(part, "=")+1:]
-		} else {
-			connectionWithoutDB += part
-			connectionWithoutDB += " "
-		}
-	}
-
-	dialector := postgres.Open(connectionWithoutDB)
+    dbName := ""
+    dbpair := ""
+    for _, part := range dsnParts {
+        if strings.HasPrefix(part, "dbname=") {
+            dbName = part[strings.Index(part, "=")+1:]
+            dbpair = part
+        }
+    }
+    connectionStr := strings.Replace(c.DSN, dbpair, "dbname=postgres", 1)
+    dialector := postgres.Open(connectionStr)
 
 	gconfig := &gorm.Config{
 		NamingStrategy: schema.NamingStrategy{
@@ -213,89 +211,89 @@ func checkPostgresDatabaseExist(c DBConfig) (bool, error) {
 		return false, fmt.Errorf("failed to open database: %v", err)
 	}
 
-    var databases []string
-    query := genQuery(c)
-    if err := db.Raw(query).Scan(&databases).Error; err != nil {
-        return false, fmt.Errorf("failed to query: %v", err)
-    }
+	var databases []string
+	query := genQuery(c)
+	if err := db.Raw(query).Scan(&databases).Error; err != nil {
+		return false, fmt.Errorf("failed to query: %v", err)
+	}
 
-    for _, database := range databases {
-        if database == dbName {
+	for _, database := range databases {
+		if database == dbName {
 			fmt.Println("Database exist")
-            return true, nil
-        }
-    }
+			return true, nil
+		}
+	}
 
-    return false, nil
+	return false, nil
 }
 
 func checkMysqlDatabaseExist(c DBConfig) (bool, error) {
 	dsnParts := strings.SplitN(c.DSN, "/", 2)
-    if len(dsnParts) != 2 {
-        return false, fmt.Errorf("failed to parse DSN: %s", c.DSN)
-    }
+	if len(dsnParts) != 2 {
+		return false, fmt.Errorf("failed to parse DSN: %s", c.DSN)
+	}
 
-    connectionInfo := dsnParts[0]
-    dbInfo := dsnParts[1]
-    dbName := dbInfo
+	connectionInfo := dsnParts[0]
+	dbInfo := dsnParts[1]
+	dbName := dbInfo
 
-    queryIndex := strings.Index(dbInfo, "?")
-    if queryIndex != -1 {
-        dbName = dbInfo[:queryIndex]
-    } else {
-        return false, fmt.Errorf("failed to parse database name from DSN: %s", c.DSN)
-    }
+	queryIndex := strings.Index(dbInfo, "?")
+	if queryIndex != -1 {
+		dbName = dbInfo[:queryIndex]
+	} else {
+		return false, fmt.Errorf("failed to parse database name from DSN: %s", c.DSN)
+	}
 
-    connectionWithoutDB := connectionInfo + "/?" + dbInfo[queryIndex+1:]
+	connectionWithoutDB := connectionInfo + "/?" + dbInfo[queryIndex+1:]
 
-    var dialector gorm.Dialector
-    switch strings.ToLower(c.DBType) {
-    case "mysql":
-        dialector = mysql.Open(connectionWithoutDB)
-    case "postgres":
-        dialector = postgres.Open(connectionWithoutDB)
-    default:
-        return false, fmt.Errorf("unsupported database type: %s", c.DBType)
-    }
+	var dialector gorm.Dialector
+	switch strings.ToLower(c.DBType) {
+	case "mysql":
+		dialector = mysql.Open(connectionWithoutDB)
+	case "postgres":
+		dialector = postgres.Open(connectionWithoutDB)
+	default:
+		return false, fmt.Errorf("unsupported database type: %s", c.DBType)
+	}
 
-    gconfig := &gorm.Config{
-        NamingStrategy: schema.NamingStrategy{
-            TablePrefix:   c.TablePrefix,
-            SingularTable: true,
-        },
-        Logger: gormLogger,
-    }
+	gconfig := &gorm.Config{
+		NamingStrategy: schema.NamingStrategy{
+			TablePrefix:   c.TablePrefix,
+			SingularTable: true,
+		},
+		Logger: gormLogger,
+	}
 
-    db, err := gorm.Open(dialector, gconfig)
-    if err != nil {
-        return false, fmt.Errorf("failed to open database: %v", err)
-    }
+	db, err := gorm.Open(dialector, gconfig)
+	if err != nil {
+		return false, fmt.Errorf("failed to open database: %v", err)
+	}
 
-    var databases []string
-    query := genQuery(c)
-    if err := db.Raw(query).Scan(&databases).Error; err != nil {
-        return false, fmt.Errorf("failed to query: %v", err)
-    }
+	var databases []string
+	query := genQuery(c)
+	if err := db.Raw(query).Scan(&databases).Error; err != nil {
+		return false, fmt.Errorf("failed to query: %v", err)
+	}
 
-    for _, database := range databases {
-        if database == dbName {
-            return true, nil
-        }
-    }
+	for _, database := range databases {
+		if database == dbName {
+			return true, nil
+		}
+	}
 
-    return false, nil
+	return false, nil
 }
 
 func genQuery(c DBConfig) string {
 	switch strings.ToLower(c.DBType) {
-    case "mysql":
-        return "SHOW DATABASES"
-    case "postgres":
-        return "SELECT datname FROM pg_database"
-    case "sqlite":
-        return ""
-    default:
-        return ""
+	case "mysql":
+		return "SHOW DATABASES"
+	case "postgres":
+		return "SELECT datname FROM pg_database"
+	case "sqlite":
+		return ""
+	default:
+		return ""
 	}
 }
 

pkg/parser/calc.go

@@ -97,3 +97,13 @@ func between(target float64, arr []interface{}) bool {
 
 	return target >= min && target <= max
 }
+
+func CalcWithRid(s string, data map[string]interface{}, rid int64) bool {
+	v, err := MathCalc(s, data)
+	if err != nil {
+		logger.Errorf("rid:%d exp:%s data:%v error: %v", rid, s, data, err)
+		return false
+	}
+
+	return v > 0
+}

pkg/poster/post.go

@@ -63,7 +63,7 @@ func GetByUrl[T any](url string, cfg conf.CenterApi) (T, error) {
 		Timeout: time.Duration(cfg.Timeout) * time.Millisecond,
 	}
 
-	if useProxy(url) {
+	if UseProxy(url) {
 		client.Transport = ProxyTransporter
 	}
 
@@ -147,7 +147,7 @@ func PostByUrl[T any](url string, cfg conf.CenterApi, v interface{}) (t T, err e
 		Timeout: time.Duration(cfg.Timeout) * time.Millisecond,
 	}
 
-	if useProxy(url) {
+	if UseProxy(url) {
 		client.Transport = ProxyTransporter
 	}
 
@@ -195,7 +195,7 @@ var ProxyTransporter = &http.Transport{
 	Proxy: http.ProxyFromEnvironment,
 }
 
-func useProxy(url string) bool {
+func UseProxy(url string) bool {
 	// N9E_PROXY_URL=oapi.dingtalk.com,feishu.com
 	patterns := os.Getenv("N9E_PROXY_URL")
 	if patterns != "" {
@@ -228,7 +228,7 @@ func PostJSON(url string, timeout time.Duration, v interface{}, retries ...int)
 		Timeout: timeout,
 	}
 
-	if useProxy(url) {
+	if UseProxy(url) {
 		client.Transport = ProxyTransporter
 	}
 

pkg/promql/parser.go

@@ -0,0 +1,181 @@
+package promql
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/VictoriaMetrics/metricsql"
+	"github.com/prometheus/prometheus/promql/parser"
+)
+
+func SplitBinaryOp(code string) ([]string, error) {
+	var lst []string
+	expr, err := metricsql.Parse(code)
+
+	if err != nil {
+		return lst, err
+	}
+
+	m := make(map[string]struct{})
+	ParseExpr(expr, false, m)
+	for k := range m {
+		lst = append(lst, k)
+	}
+
+	return lst, nil
+}
+
+func GetMetric(ql string) (map[string]string, error) {
+	metrics := make(map[string]string)
+	expr, err := parser.ParseExpr(ql)
+	if err != nil {
+		return metrics, err
+	}
+
+	selectors := parser.ExtractSelectors(expr)
+	for i := 0; i < len(selectors); i++ {
+		var metric string
+		var labels []string
+		for j := 0; j < len(selectors[i]); j++ {
+			if selectors[i][j].Name == "__name__" {
+				metric = selectors[i][j].Value
+			} else {
+				labels = append(labels, selectors[i][j].Name+selectors[i][j].Type.String()+"\""+selectors[i][j].Value+"\"")
+			}
+		}
+
+		if len(labels) != 0 {
+			metrics[metric] = metric + "{" + strings.Join(labels, ",") + "}"
+		} else {
+			metrics[metric] = metric
+		}
+	}
+	return metrics, nil
+}
+
+// GetLabels 解析PromQL查询并返回其中的所有标签和它们的值。
+func GetLabels(ql string) (map[string]string, error) {
+	labels := make(map[string]string)
+
+	// 解析PromQL表达式
+	expr, err := parser.ParseExpr(ql)
+	if err != nil {
+		return labels, err
+	}
+
+	// 提取所有的选择器
+	selectors := parser.ExtractSelectors(expr)
+	for _, selector := range selectors {
+		for _, labelMatcher := range selector {
+			if labelMatcher.Name != "__name__" {
+				labels[labelMatcher.Name] = labelMatcher.Value
+			}
+		}
+	}
+
+	return labels, nil
+}
+
+func GetLabelsAndMetricName(ql string) (map[string]string, string, error) {
+	labels := make(map[string]string)
+	metricName := ""
+
+	// 解析PromQL表达式
+	expr, err := parser.ParseExpr(ql)
+	if err != nil {
+		return labels, metricName, err
+	}
+
+	// 提取所有的选择器
+	selectors := parser.ExtractSelectors(expr)
+	for _, selector := range selectors {
+		for _, labelMatcher := range selector {
+			if labelMatcher.Name != "__name__" {
+				labels[labelMatcher.Name] = labelMatcher.Value
+			} else {
+				metricName = labelMatcher.Value
+			}
+		}
+	}
+
+	return labels, metricName, nil
+}
+
+type Label struct {
+	Name  string
+	Value string
+	Op    string
+}
+
+func GetLabelsAndMetricNameWithReplace(ql string, rep string) (map[string]Label, string, error) {
+	labels := make(map[string]Label)
+	metricName := ""
+
+	ql = strings.ReplaceAll(ql, rep, "____")
+	ql = removeBrackets(ql)
+	// 解析PromQL表达式
+	expr, err := parser.ParseExpr(ql)
+	if err != nil {
+		return labels, metricName, err
+	}
+
+	// 提取所有的选择器
+	selectors := parser.ExtractSelectors(expr)
+	for _, selector := range selectors {
+		for _, labelMatcher := range selector {
+			labelMatcher.Value = strings.ReplaceAll(labelMatcher.Value, "____", rep)
+			if labelMatcher.Name != "__name__" {
+				label := Label{
+					Name:  labelMatcher.Name,
+					Value: labelMatcher.Value,
+					Op:    labelMatcher.Type.String(),
+				}
+				labels[labelMatcher.Name] = label
+			} else {
+				if strings.Contains(labelMatcher.Value, "$") {
+					continue
+				}
+				metricName = labelMatcher.Value
+			}
+		}
+	}
+
+	return labels, metricName, nil
+}
+
+func GetFirstMetric(ql string) (string, error) {
+	var metric string
+	expr, err := parser.ParseExpr(ql)
+	if err != nil {
+		return metric, err
+	}
+
+	selectors := parser.ExtractSelectors(expr)
+	for i := 0; i < len(selectors); i++ {
+		for j := 0; j < len(selectors[i]); j++ {
+			if selectors[i][j].Name == "__name__" {
+				metric = selectors[i][j].Value
+				return metric, nil
+			}
+		}
+	}
+	return metric, nil
+}
+
+func removeBrackets(promql string) string {
+	if strings.Contains(promql, "_over_time") || strings.Contains(promql, "rate") || strings.Contains(promql, "increase") ||
+		strings.Contains(promql, "predict_linear") || strings.Contains(promql, "resets") ||
+		strings.Contains(promql, "changes") || strings.Contains(promql, "holt_winters") ||
+		strings.Contains(promql, "delta") || strings.Contains(promql, "deriv") {
+		return promql
+	}
+
+	if !strings.Contains(promql, "[") {
+		return promql
+	}
+
+	// 使用正则表达式匹配 [xx] 形式的内容，xx 可以是任何字符序列
+	re := regexp.MustCompile(`\[[^\]]*\]`)
+	// 删除匹配到的内容
+	return re.ReplaceAllString(promql, "")
+}

pkg/promql/perser_test.go

@@ -0,0 +1,218 @@
+package promql
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGetMetric(t *testing.T) {
+	tests := []struct {
+		name    string
+		ql      string
+		want    map[string]string
+		wantErr error
+	}{
+		{
+			name:    "Valid query with labels",
+			ql:      "metric_name{label1=\"value1\",label2=\"value2\"}",
+			want:    map[string]string{"metric_name": "metric_name{label1=\"value1\",label2=\"value2\"}"},
+			wantErr: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetMetric(tt.ql)
+			if err != tt.wantErr && err != nil {
+				t.Errorf("GetMetric() error = %v, wantErr %v ql:%s", err, tt.wantErr, tt.ql)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetMetric() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestGetLabels(t *testing.T) {
+	tests := []struct {
+		name    string
+		ql      string
+		want    map[string]string
+		wantErr bool
+	}{
+		{
+			name: "Valid query with multiple labels",
+			ql:   "metric_name{label1=\"value1\", label2=\"value2\"} > 3",
+			want: map[string]string{"label1": "value1", "label2": "value2"},
+		},
+		{
+			name: "Valid query with multiple labels",
+			ql:   "metric_name{label1=\"$value1\", label2=\"$value2\"} > 3",
+			want: map[string]string{"label1": "$value1", "label2": "$value2"},
+		},
+		{
+			name: "Query without labels",
+			ql:   "metric_name",
+			want: map[string]string{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetLabels(tt.ql)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetLabels() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetLabels() = %v, want %v ql:%s", got, tt.want, tt.ql)
+			}
+		})
+	}
+}
+
+func TestGetLabelsAndMetricNameWithReplace(t *testing.T) {
+	// 定义测试案例
+	tests := []struct {
+		name               string
+		ql                 string
+		rep                string
+		expectedLabels     map[string]Label
+		expectedMetricName string
+		expectError        bool
+	}{
+		{
+			name: "正常情况",
+			ql:   `(snmp_arista_system_cpuuse{ent_descr="$ent_descr"} / 100 > $cpu_high_threshold[1m])`,
+			rep:  "$",
+			expectedLabels: map[string]Label{
+				"ent_descr": {Name: "ent_descr", Value: "$ent_descr", Op: "="},
+			},
+			expectedMetricName: "snmp_arista_system_cpuuse",
+			expectError:        false,
+		},
+		{
+			name: "正常情况",
+			ql:   `rate(snmp_interface_incoming{agent_host='$agent_host',ifname='$ifname'}[2m]) * 8 / 10^9 > snmp_interface_speed{agent_host='$agent_host',ifname='$ifname'}/ 10^3 *  $traffic_in and snmp_interface_speed{agent_host='$agent_host',ifname='$ifname'} > 0`,
+			rep:  "$",
+			expectedLabels: map[string]Label{
+				"agent_host": {Name: "agent_host", Value: "$agent_host", Op: "="},
+				"ifname":     {Name: "ifname", Value: "$ifname", Op: "="},
+			},
+			expectedMetricName: "snmp_interface_speed",
+			expectError:        false,
+		},
+		{
+			name: "正常情况",
+			ql:   `rate(snmp_interface_incoming{agent_host='$agent_host',ifname='$ifname'}[2m]) * 8 / 10^9 > snmp_interface_speed{agent_host='$agent_host',ifname='$ifname'}/ 10^3 *  $traffic_in`,
+			rep:  "$",
+			expectedLabels: map[string]Label{
+				"agent_host": {Name: "agent_host", Value: "$agent_host", Op: "="},
+				"ifname":     {Name: "ifname", Value: "$ifname", Op: "="},
+			},
+			expectedMetricName: "snmp_interface_speed",
+			expectError:        false,
+		},
+		{
+			name: "正常情况",
+			ql:   `rate(snmp_interface_incoming{agent_host='$agent_host',ifname='$ifname'}[2m]) * 8 / 10^9 > 10`,
+			rep:  "$",
+			expectedLabels: map[string]Label{
+				"agent_host": {Name: "agent_host", Value: "$agent_host", Op: "="},
+				"ifname":     {Name: "ifname", Value: "$ifname", Op: "="},
+			},
+			expectedMetricName: "snmp_interface_incoming",
+			expectError:        false,
+		},
+		{
+			name: "带有替换字符",
+			ql:   `rate(snmp_interface_outgoing{Role=~'ZRT.*',agent_host='$agent_host',ifname='$ifname'}[2m]) * 8 / 10^9 > snmp_interface_speed{Role=~'ZRT.*',agent_host='$agent_host',ifname='$ifname'}/ 10^3 * $outgoing_warning and snmp_interface_speed{Role=~'ZRT.*',agent_host='$agent_host',ifname='$ifname'} > 0`,
+			rep:  "$",
+			expectedLabels: map[string]Label{
+				"agent_host": {Name: "agent_host", Value: "$agent_host", Op: "="},
+				"ifname":     {Name: "ifname", Value: "$ifname", Op: "="},
+				"Role":       {Name: "Role", Value: "ZRT.*", Op: "=~"},
+			},
+			expectedMetricName: "snmp_interface_speed",
+			expectError:        false,
+		},
+		// 更多测试案例...
+		{
+			name:               "告警规则支持变量",
+			ql:                 `mem{test1="$test1", test2="$test2", test3="test3"} > $val`,
+			rep:                "$",
+			expectedLabels:     map[string]Label{},
+			expectedMetricName: "snmp_interface_speed",
+			expectError:        false,
+		},
+	}
+
+	// 运行测试案例
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			labels, metricName, err := GetLabelsAndMetricNameWithReplace(tc.ql, tc.rep)
+
+			if (err != nil) != tc.expectError {
+				t.Errorf("ql:%s 测试 '%v' 发生错误: %v, 期望的错误状态: %v", tc.ql, tc.name, err, tc.expectError)
+			}
+
+			if !reflect.DeepEqual(labels, tc.expectedLabels) {
+				t.Errorf("ql:%s 测试 '%v' 返回的标签不匹配: got %v, want %v", tc.ql, tc.name, labels, tc.expectedLabels)
+			}
+
+			if metricName != tc.expectedMetricName {
+				t.Errorf("ql:%s 测试 '%v' 返回的度量名称不匹配: got %s, want %s", tc.ql, tc.name, metricName, tc.expectedMetricName)
+			}
+		})
+	}
+}
+
+func TestSplitBinaryOp(t *testing.T) {
+	tests := []struct {
+		name    string
+		code    string
+		want    []string
+		wantErr bool
+	}{
+		{
+			name: "valid binary operation with spaces",
+			code: "cpu_usage  +  memory_usage",
+			want: []string{"cpu_usage + memory_usage"},
+		},
+		{
+			name: "12",
+			code: "cpu_usage > 0 and memory_usage>0",
+			want: []string{"cpu_usage", "memory_usage"},
+		},
+		{
+			name: "12",
+			code: "cpu_usage +1> 0",
+			want: []string{"cpu_usage + 1"},
+		},
+		{
+			name: "valid complex binary operation",
+			code: "(cpu_usage + memory_usage) / 2",
+			want: []string{"(cpu_usage + memory_usage) / 2"},
+		},
+		{
+			name:    "invalid binary operation",
+			code:    "cpu_usage + ",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := SplitBinaryOp(tt.code)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("SplitBinaryOp() code:%s error = %v, wantErr %v", tt.code, err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("SplitBinaryOp() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/promql/promql.go

@@ -0,0 +1,104 @@
+package promql
+
+import (
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// copy from https://github.com/laixintao/promqlpy/blob/main/go/promql/promql.go
+// ModifierExpr represents MetricsQL modifier such as `<op> (...)`
+type ModifierExpr struct {
+	// Op is modifier operation.
+	Op string `json:"op"`
+
+	// Args contains modifier args from parens.
+	Args []string `json:"args"`
+}
+
+type Expression struct {
+	// if true, all fields are set
+	// if false, then it's a normal expression, only `code` is set
+	IsBinaryOp bool `json:"is_binary_op"`
+
+	Left  *Expression `json:"left"`
+	Right *Expression `json:"right"`
+	Op    string      `json:"op"`
+	// GroupModifier contains modifier such as "on" or "ignoring".
+	GroupModifier ModifierExpr `json:"group_modifier"`
+	// JoinModifier contains modifier such as "group_left" or "group_right".
+	JoinModifier ModifierExpr `json:"join_modifier"`
+
+	Code string `json:"code"`
+}
+
+var compareOps = map[string]bool{
+	"==": true,
+	"!=": true,
+	">":  true,
+	"<":  true,
+	">=": true,
+	"<=": true,
+}
+
+var logicalOps = map[string]bool{
+	"and":    true,
+	"or":     true,
+	"unless": true,
+}
+
+// if `mustBeExpression` is true, means that the last level is compareOps
+// or ready.
+// example:
+// (a > 10) > b
+// result: a > 10 is expression, compare to b
+func ParseExpr(expr metricsql.Expr, mustBeExpression bool, m map[string]struct{}) *Expression {
+
+	// I am sure it is a normal expression!
+	if mustBeExpression {
+		return &Expression{
+			Code:       string(expr.AppendString(nil)),
+			IsBinaryOp: false,
+		}
+	}
+
+	if bop, ok := expr.(*metricsql.BinaryOpExpr); ok {
+
+		if logicalOps[bop.Op] {
+
+			return &Expression{
+				Left:          ParseExpr(bop.Left, false, m),
+				Right:         ParseExpr(bop.Right, false, m),
+				GroupModifier: ModifierExpr(bop.GroupModifier),
+				JoinModifier:  ModifierExpr(bop.JoinModifier),
+				Op:            bop.Op,
+				Code:          string(bop.AppendString(nil)),
+				IsBinaryOp:    true,
+			}
+		}
+
+		if compareOps[bop.Op] {
+			m[string(bop.Left.AppendString(nil))] = struct{}{}
+
+			return &Expression{
+				Left:          ParseExpr(bop.Left, true, m),
+				Right:         ParseExpr(bop.Right, true, m),
+				GroupModifier: ModifierExpr(bop.GroupModifier),
+				JoinModifier:  ModifierExpr(bop.JoinModifier),
+				Op:            bop.Op,
+				Code:          string(bop.AppendString(nil)),
+				IsBinaryOp:    true,
+			}
+
+		}
+	}
+
+	if len(m) == 0 {
+		m[string(expr.AppendString(nil))] = struct{}{}
+	}
+
+	// treat +,-,* etc still as normal expression
+	// default: just return the literal code as it is
+	return &Expression{
+		Code:       string(expr.AppendString(nil)),
+		IsBinaryOp: false,
+	}
+}

pushgw/pconf/conf.go

@@ -24,8 +24,10 @@ type Pushgw struct {
 }
 
 type WriterGlobalOpt struct {
-	QueueMaxSize int
-	QueuePopSize int
+	QueueMaxSize            int
+	QueuePopSize            int
+	AllQueueMaxSize         int
+	AllQueueMaxSizeInterval int
 }
 
 type WriterOptions struct {
@@ -77,6 +79,14 @@ func (p *Pushgw) PreCheck() {
 		p.WriterOpt.QueuePopSize = 1000
 	}
 
+	if p.WriterOpt.AllQueueMaxSize <= 0 {
+		p.WriterOpt.AllQueueMaxSize = 10000000
+	}
+
+	if p.WriterOpt.AllQueueMaxSizeInterval <= 0 {
+		p.WriterOpt.AllQueueMaxSizeInterval = 200
+	}
+
 	if p.WriteConcurrency <= 0 {
 		p.WriteConcurrency = 5000
 	}

pushgw/pushgw.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/ccfos/nightingale/v6/center/metas"
 	"github.com/ccfos/nightingale/v6/conf"
+	"github.com/ccfos/nightingale/v6/dscache"
 	"github.com/ccfos/nightingale/v6/memsto"
 	"github.com/ccfos/nightingale/v6/pkg/ctx"
 	"github.com/ccfos/nightingale/v6/pkg/httpx"
@@ -55,7 +56,7 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
 	r := httpx.GinEngine(config.Global.RunMode, config.HTTP, configCvalCache.PrintBodyPaths, configCvalCache.PrintAccessLog)
 	rt := router.New(config.HTTP, config.Pushgw, config.Alert, targetCache, busiGroupCache, idents, metas, writers, ctx)
 	rt.Config(r)
-
+	dscache.Init(ctx, false)
 	httpClean := httpx.Init(config.HTTP, r)
 
 	return func() {

pushgw/router/router_remotewrite.go

@@ -114,7 +114,7 @@ func (rt *Router) remoteWrite(c *gin.Context) {
 
 	var (
 		ignoreIdent = ginx.QueryBool(c, "ignore_ident", false)
-		ignoreHost  = ginx.QueryBool(c, "ignore_host", false)
+		ignoreHost  = ginx.QueryBool(c, "ignore_host", true) // 默认值改成 true，要不然答疑成本太高。发版的时候通知 telegraf 用户，让他们设置 ignore_host=false
 		ids         = make(map[string]struct{})
 	)
 

pushgw/writer/writer.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ccfos/nightingale/v6/pkg/fasttime"
@@ -138,9 +139,10 @@ func (w WriterType) Post(req []byte, headers ...map[string]string) error {
 }
 
 type WritersType struct {
-	pushgw   pconf.Pushgw
-	backends map[string]WriterType
-	queues   map[string]*IdentQueue
+	pushgw      pconf.Pushgw
+	backends    map[string]WriterType
+	queues      map[string]*IdentQueue
+	allQueueLen atomic.Value
 	sync.RWMutex
 }
 
@@ -160,14 +162,30 @@ func (ws *WritersType) ReportQueueStats(ident string, identQueue *IdentQueue) (i
 	}
 }
 
+func (ws *WritersType) SetAllQueueLen() {
+	for {
+		curMetricLen := 0
+		ws.RLock()
+		for _, q := range ws.queues {
+			curMetricLen += q.list.Len()
+		}
+		ws.RUnlock()
+		ws.allQueueLen.Store(curMetricLen)
+		time.Sleep(time.Duration(ws.pushgw.WriterOpt.AllQueueMaxSizeInterval) * time.Millisecond)
+	}
+}
+
 func NewWriters(pushgwConfig pconf.Pushgw) *WritersType {
 	writers := &WritersType{
-		backends: make(map[string]WriterType),
-		queues:   make(map[string]*IdentQueue),
-		pushgw:   pushgwConfig,
+		backends:    make(map[string]WriterType),
+		queues:      make(map[string]*IdentQueue),
+		pushgw:      pushgwConfig,
+		allQueueLen: atomic.Value{},
 	}
 
 	writers.Init()
+
+	go writers.SetAllQueueLen()
 	go writers.CleanExpQueue()
 	return writers
 }
@@ -217,6 +235,13 @@ func (ws *WritersType) PushSample(ident string, v interface{}) {
 	}
 
 	identQueue.ts = time.Now().Unix()
+	curLen := ws.allQueueLen.Load().(int)
+	if curLen > ws.pushgw.WriterOpt.AllQueueMaxSize {
+		logger.Warningf("Write %+v full, metric count over limit: %d", v, curLen)
+		CounterPushQueueErrorTotal.WithLabelValues(ident).Inc()
+		return
+	}
+
 	succ := identQueue.list.PushFront(v)
 	if !succ {
 		logger.Warningf("Write channel(%s) full, current channel size: %d", ident, identQueue.list.Len())
@@ -245,6 +270,7 @@ func (ws *WritersType) StartConsumer(identQueue *IdentQueue) {
 
 func (ws *WritersType) Init() error {
 	opts := ws.pushgw.Writers
+	ws.allQueueLen.Store(0)
 
 	for i := 0; i < len(opts); i++ {
 		tlsConf, err := opts[i].ClientConfig.TLSConfig()

storage/redis.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/alicebob/miniredis/v2"
 	"github.com/ccfos/nightingale/v6/pkg/tlsx"
 	"github.com/redis/go-redis/v9"
 	"github.com/toolkits/pkg/logger"
@@ -28,6 +29,7 @@ type Redis redis.Cmdable
 
 func NewRedis(cfg RedisConfig) (Redis, error) {
 	var redisClient Redis
+
 	switch cfg.RedisType {
 	case "standalone", "":
 		redisOptions := &redis.Options{
@@ -88,6 +90,16 @@ func NewRedis(cfg RedisConfig) (Redis, error) {
 
 		redisClient = redis.NewFailoverClient(redisOptions)
 
+	case "miniredis":
+		s, err := miniredis.Run()
+		if err != nil {
+			fmt.Println("failed to init miniredis:", err)
+			os.Exit(1)
+		}
+		redisClient = redis.NewClient(&redis.Options{
+			Addr: s.Addr(),
+		})
+
 	default:
 		fmt.Println("failed to init redis , redis type is illegal:", cfg.RedisType)
 		os.Exit(1)

storage/redis_test.go

@@ -0,0 +1,44 @@
+package storage
+
+import (
+	"context"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMiniRedisMGet(t *testing.T) {
+	s, err := miniredis.Run()
+	if err != nil {
+		t.Fatalf("failed to start miniredis: %v", err)
+	}
+	defer s.Close()
+
+	rdb := redis.NewClient(&redis.Options{
+		Addr: s.Addr(),
+	})
+
+	err = rdb.Ping(context.Background()).Err()
+	if err != nil {
+		t.Fatalf("failed to ping miniredis: %v", err)
+	}
+
+	mp := make(map[string]interface{})
+	mp["key1"] = "value1"
+	mp["key2"] = "value2"
+	mp["key3"] = "value3"
+
+	err = MSet(context.Background(), rdb, mp)
+	if err != nil {
+		t.Fatalf("failed to set miniredis value: %v", err)
+	}
+
+	ctx := context.Background()
+	keys := []string{"key1", "key2", "key3", "key4"}
+	vals := MGet(ctx, rdb, keys)
+
+	expected := [][]byte{[]byte("value1"), []byte("value2"), []byte("value3")}
+	assert.Equal(t, expected, vals)
+}

tdengine/option.go

@@ -1,93 +0,0 @@
-package tdengine
-
-import (
-	"sync"
-
-	"github.com/ccfos/nightingale/v6/pkg/tlsx"
-)
-
-type TdengineOption struct {
-	DatasourceName string
-	Url            string
-	BasicAuthUser  string
-	BasicAuthPass  string
-	Token          string
-
-	Timeout     int64
-	DialTimeout int64
-
-	MaxIdleConnsPerHost int
-
-	Headers []string
-
-	tlsx.ClientConfig
-}
-
-func (po *TdengineOption) Equal(target TdengineOption) bool {
-	if po.Url != target.Url {
-		return false
-	}
-
-	if po.BasicAuthUser != target.BasicAuthUser {
-		return false
-	}
-
-	if po.BasicAuthPass != target.BasicAuthPass {
-		return false
-	}
-
-	if po.Token != target.Token {
-		return false
-	}
-
-	if po.Timeout != target.Timeout {
-		return false
-	}
-
-	if po.DialTimeout != target.DialTimeout {
-		return false
-	}
-
-	if po.MaxIdleConnsPerHost != target.MaxIdleConnsPerHost {
-		return false
-	}
-
-	if len(po.Headers) != len(target.Headers) {
-		return false
-	}
-
-	for i := 0; i < len(po.Headers); i++ {
-		if po.Headers[i] != target.Headers[i] {
-			return false
-		}
-	}
-
-	return true
-}
-
-type TdengineOptionsStruct struct {
-	Data map[int64]TdengineOption
-	sync.RWMutex
-}
-
-func (pos *TdengineOptionsStruct) Set(datasourceId int64, po TdengineOption) {
-	pos.Lock()
-	pos.Data[datasourceId] = po
-	pos.Unlock()
-}
-
-func (pos *TdengineOptionsStruct) Del(datasourceId int64) {
-	pos.Lock()
-	delete(pos.Data, datasourceId)
-	pos.Unlock()
-}
-
-func (pos *TdengineOptionsStruct) Get(datasourceId int64) (TdengineOption, bool) {
-	pos.RLock()
-	defer pos.RUnlock()
-	ret, has := pos.Data[datasourceId]
-	return ret, has
-}
-
-// Data key is cluster name
-var TdengineOptions = &TdengineOptionsStruct{Data: make(map[int64]TdengineOption)}

tdengine/reload.go

@@ -1,140 +0,0 @@
-package tdengine
-
-import (
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/ccfos/nightingale/v6/alert/aconf"
-	"github.com/ccfos/nightingale/v6/models"
-	"github.com/ccfos/nightingale/v6/pkg/ctx"
-	"github.com/ccfos/nightingale/v6/pkg/poster"
-
-	"github.com/toolkits/pkg/logger"
-)
-
-func NewTdengineClient(ctx *ctx.Context, heartbeat aconf.HeartbeatConfig) *TdengineClientMap {
-	pc := &TdengineClientMap{
-		ReaderClients: make(map[int64]*tdengineClient),
-		heartbeat:     heartbeat,
-		ctx:           ctx,
-	}
-	pc.InitReader()
-	return pc
-}
-
-func (pc *TdengineClientMap) InitReader() error {
-	go func() {
-		for {
-			pc.loadFromDatabase()
-			time.Sleep(time.Second)
-		}
-	}()
-	return nil
-}
-
-func (pc *TdengineClientMap) loadFromDatabase() {
-	var datasources []*models.Datasource
-	var err error
-	if !pc.ctx.IsCenter {
-		datasources, err = poster.GetByUrls[[]*models.Datasource](pc.ctx, "/v1/n9e/datasources?typ="+models.TDENGINE)
-		if err != nil {
-			logger.Errorf("failed to get datasources, error: %v", err)
-			return
-		}
-		for i := 0; i < len(datasources); i++ {
-			datasources[i].FE2DB()
-		}
-	} else {
-		datasources, err = models.GetDatasourcesGetsBy(pc.ctx, models.TDENGINE, "", "", "")
-		if err != nil {
-			logger.Errorf("failed to get datasources, error: %v", err)
-			return
-		}
-	}
-
-	newCluster := make(map[int64]struct{})
-	for _, ds := range datasources {
-		dsId := ds.Id
-		var header []string
-		for k, v := range ds.HTTPJson.Headers {
-			header = append(header, k)
-			header = append(header, v)
-		}
-
-		po := TdengineOption{
-			DatasourceName:      ds.Name,
-			Url:                 ds.HTTPJson.Url,
-			BasicAuthUser:       ds.AuthJson.BasicAuthUser,
-			BasicAuthPass:       ds.AuthJson.BasicAuthPassword,
-			Timeout:             ds.HTTPJson.Timeout,
-			DialTimeout:         ds.HTTPJson.DialTimeout,
-			MaxIdleConnsPerHost: ds.HTTPJson.MaxIdleConnsPerHost,
-			Headers:             header,
-		}
-
-		if strings.HasPrefix(ds.HTTPJson.Url, "https") {
-			po.UseTLS = true
-			po.InsecureSkipVerify = ds.HTTPJson.TLS.SkipTlsVerify
-		}
-
-		if len(ds.SettingsJson) > 0 {
-			for k, v := range ds.SettingsJson {
-				if strings.Contains(k, "token") {
-					po.Headers = append(po.Headers, "Authorization")
-					po.Headers = append(po.Headers, "Taosd "+v.(string))
-				}
-			}
-		}
-
-		newCluster[dsId] = struct{}{}
-		if pc.IsNil(dsId) {
-			// first time
-			if err = pc.setClientFromTdengineOption(dsId, po); err != nil {
-				logger.Errorf("failed to setClientFromTdengineOption po:%+v err:%v", po, err)
-				continue
-			}
-
-			logger.Info("setClientFromTdengineOption success: ", dsId)
-			TdengineOptions.Set(dsId, po)
-			continue
-		}
-
-		localPo, has := TdengineOptions.Get(dsId)
-		if !has || !localPo.Equal(po) {
-			if err = pc.setClientFromTdengineOption(dsId, po); err != nil {
-				logger.Errorf("failed to setClientFromTdengineOption: %v", err)
-				continue
-			}
-
-			TdengineOptions.Set(dsId, po)
-		}
-	}
-
-	// delete useless cluster
-	oldIds := pc.GetDatasourceIds()
-	for _, oldId := range oldIds {
-		if _, has := newCluster[oldId]; !has {
-			pc.Del(oldId)
-			TdengineOptions.Del(oldId)
-			logger.Info("delete cluster: ", oldId)
-		}
-	}
-}
-
-func (pc *TdengineClientMap) setClientFromTdengineOption(datasourceId int64, po TdengineOption) error {
-	if datasourceId < 0 {
-		return fmt.Errorf("argument clusterName is blank")
-	}
-
-	if po.Url == "" {
-		return fmt.Errorf("prometheus url is blank")
-	}
-
-	reader := newTdengine(po)
-
-	logger.Debugf("setClientFromTdengineOption: %d, %+v", datasourceId, po)
-	pc.Set(datasourceId, reader)
-
-	return nil
-}