mirror of
https://github.com/ccfos/nightingale.git
synced 2026-03-03 14:38:55 +00:00
Compare commits
95 Commits
search-vie
...
dev21
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b02ddeec7b | ||
|
|
d5528541c3 | ||
|
|
33ec277ac1 | ||
|
|
a63d6a1e49 | ||
|
|
84a179c4f4 | ||
|
|
f5811bc5f7 | ||
|
|
f27bbb4a51 | ||
|
|
5de63d7307 | ||
|
|
6a44da4dda | ||
|
|
b0fbca21b8 | ||
|
|
0a65616fbb | ||
|
|
a0e8c5f764 | ||
|
|
0c71eeac2a | ||
|
|
d64dbb6909 | ||
|
|
656b91e976 | ||
|
|
fe6dce403f | ||
|
|
48820a6bd5 | ||
|
|
faa348a086 | ||
|
|
635b781ae1 | ||
|
|
f60771ad9c | ||
|
|
6bd2f9a89f | ||
|
|
a76049822c | ||
|
|
97746f7469 | ||
|
|
903d75e4b8 | ||
|
|
52421f2477 | ||
|
|
42637e546d | ||
|
|
a9ab02e1ad | ||
|
|
7bf000932d | ||
|
|
3202cd1410 | ||
|
|
e28dd079f9 | ||
|
|
72cb35a4ed | ||
|
|
80d0193ac0 | ||
|
|
e5acc9199b | ||
|
|
ec7fbf313b | ||
|
|
1180066df3 | ||
|
|
b3ee1e56ad | ||
|
|
0b71d1ef82 | ||
|
|
2934dab4c7 | ||
|
|
d908240912 | ||
|
|
ff1aa83b8c | ||
|
|
d54bcdd722 | ||
|
|
54a8e2590e | ||
|
|
81b5ce20ae | ||
|
|
806b3effe9 | ||
|
|
b296d5bcc3 | ||
|
|
cd0b529b69 | ||
|
|
9e99e4a63a | ||
|
|
996c9812bd | ||
|
|
0f8bb8b2af | ||
|
|
8c54a97292 | ||
|
|
47cab69088 | ||
|
|
c432636d8d | ||
|
|
6b25a4ce90 | ||
|
|
1a50d22573 | ||
|
|
959b0389c6 | ||
|
|
3d8f1b3ef5 | ||
|
|
ce838036ad | ||
|
|
578ac096e5 | ||
|
|
48ee6117e9 | ||
|
|
5afd6a60e9 | ||
|
|
37372ae9ea | ||
|
|
48e7c34ebf | ||
|
|
acd0ec4bef | ||
|
|
c1ad946bc5 | ||
|
|
4c2affc7da | ||
|
|
273d282beb | ||
|
|
3e86656381 | ||
|
|
f942772d2b | ||
|
|
fbc0c22d7a | ||
|
|
abd452a6df | ||
|
|
47f05627d9 | ||
|
|
edd8e2a3db | ||
|
|
c4ca2920ef | ||
|
|
afc8d7d21c | ||
|
|
46083d741d | ||
|
|
3eeb705b39 | ||
|
|
8d87e69ee7 | ||
|
|
c0e13e2870 | ||
|
|
4f186a71ba | ||
|
|
104c275f2d | ||
|
|
2ba7a970e8 | ||
|
|
3da85d8e28 | ||
|
|
b50410b88a | ||
|
|
c98241b3fd | ||
|
|
b30caf625b | ||
|
|
32e8b961c2 | ||
|
|
2ff0a8fdbb | ||
|
|
7ff74d0948 | ||
|
|
da58d825c0 | ||
|
|
0014b77c4d | ||
|
|
fc7fdde2d5 | ||
|
|
61b63fc75c | ||
|
|
80f564ec63 | ||
|
|
203c2a885b | ||
|
|
9bee3e1379 |
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/ccfos/nightingale/v6/alert/astats"
|
||||
"github.com/ccfos/nightingale/v6/alert/common"
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline"
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline/engine"
|
||||
"github.com/ccfos/nightingale/v6/alert/sender"
|
||||
"github.com/ccfos/nightingale/v6/memsto"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
@@ -231,6 +232,8 @@ func shouldSkipNotify(ctx *ctx.Context, event *models.AlertCurEvent, notifyRuleI
|
||||
}
|
||||
|
||||
func HandleEventPipeline(pipelineConfigs []models.PipelineConfig, eventOrigin, event *models.AlertCurEvent, eventProcessorCache *memsto.EventProcessorCacheType, ctx *ctx.Context, id int64, from string) *models.AlertCurEvent {
|
||||
workflowEngine := engine.NewWorkflowEngine(ctx)
|
||||
|
||||
for _, pipelineConfig := range pipelineConfigs {
|
||||
if !pipelineConfig.Enable {
|
||||
continue
|
||||
@@ -247,23 +250,28 @@ func HandleEventPipeline(pipelineConfigs []models.PipelineConfig, eventOrigin, e
|
||||
continue
|
||||
}
|
||||
|
||||
processors := eventProcessorCache.GetProcessorsById(pipelineConfig.PipelineId)
|
||||
for _, processor := range processors {
|
||||
var res string
|
||||
var err error
|
||||
logger.Infof("processor_by_%s_id:%d pipeline_id:%d, before processor:%+v, event: %+v", from, id, pipelineConfig.PipelineId, processor, event)
|
||||
event, res, err = processor.Process(ctx, event)
|
||||
if event == nil {
|
||||
logger.Infof("processor_by_%s_id:%d pipeline_id:%d, event dropped, after processor:%+v, event: %+v", from, id, pipelineConfig.PipelineId, processor, eventOrigin)
|
||||
|
||||
if from == "notify_rule" {
|
||||
// alert_rule 获取不到 eventId 记录没有意义
|
||||
sender.NotifyRecord(ctx, []*models.AlertCurEvent{eventOrigin}, id, "", "", res, fmt.Errorf("processor_by_%s_id:%d pipeline_id:%d, drop by processor", from, id, pipelineConfig.PipelineId))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
logger.Infof("processor_by_%s_id:%d pipeline_id:%d, after processor:%+v, event: %+v, res:%v, err:%v", from, id, pipelineConfig.PipelineId, processor, event, res, err)
|
||||
// 统一使用工作流引擎执行(兼容线性模式和工作流模式)
|
||||
triggerCtx := &models.WorkflowTriggerContext{
|
||||
Mode: models.TriggerModeEvent,
|
||||
TriggerBy: from + "_" + strconv.FormatInt(id, 10),
|
||||
}
|
||||
|
||||
resultEvent, result, err := workflowEngine.Execute(eventPipeline, event, triggerCtx)
|
||||
if err != nil {
|
||||
logger.Errorf("processor_by_%s_id:%d pipeline_id:%d, pipeline execute error: %v", from, id, pipelineConfig.PipelineId, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if resultEvent == nil {
|
||||
logger.Infof("processor_by_%s_id:%d pipeline_id:%d, event dropped, event: %+v", from, id, pipelineConfig.PipelineId, eventOrigin)
|
||||
if from == "notify_rule" {
|
||||
sender.NotifyRecord(ctx, []*models.AlertCurEvent{eventOrigin}, id, "", "", result.Message, fmt.Errorf("processor_by_%s_id:%d pipeline_id:%d, drop by pipeline", from, id, pipelineConfig.PipelineId))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
event = resultEvent
|
||||
logger.Infof("processor_by_%s_id:%d pipeline_id:%d, pipeline executed, status:%s, message:%s", from, id, pipelineConfig.PipelineId, result.Status, result.Message)
|
||||
}
|
||||
|
||||
event.FE2DB()
|
||||
@@ -538,7 +546,7 @@ func SendNotifyRuleMessage(ctx *ctx.Context, userCache *memsto.UserCacheType, us
|
||||
for i := range flashDutyChannelIDs {
|
||||
start := time.Now()
|
||||
respBody, err := notifyChannel.SendFlashDuty(events, flashDutyChannelIDs[i], notifyChannelCache.GetHttpClient(notifyChannel.ID))
|
||||
respBody = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), respBody)
|
||||
respBody = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), respBody)
|
||||
logger.Infof("duty_sender notify_id: %d, channel_name: %v, event:%+v, IntegrationUrl: %v dutychannel_id: %v, respBody: %v, err: %v", notifyRuleId, notifyChannel.Name, events[0], notifyChannel.RequestConfig.FlashDutyRequestConfig.IntegrationUrl, flashDutyChannelIDs[i], respBody, err)
|
||||
sender.NotifyRecord(ctx, events, notifyRuleId, notifyChannel.Name, strconv.FormatInt(flashDutyChannelIDs[i], 10), respBody, err)
|
||||
}
|
||||
@@ -547,7 +555,7 @@ func SendNotifyRuleMessage(ctx *ctx.Context, userCache *memsto.UserCacheType, us
|
||||
for _, routingKey := range pagerdutyRoutingKeys {
|
||||
start := time.Now()
|
||||
respBody, err := notifyChannel.SendPagerDuty(events, routingKey, siteInfo.SiteUrl, notifyChannelCache.GetHttpClient(notifyChannel.ID))
|
||||
respBody = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), respBody)
|
||||
respBody = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), respBody)
|
||||
logger.Infof("pagerduty_sender notify_id: %d, channel_name: %v, event:%+v, respBody: %v, err: %v", notifyRuleId, notifyChannel.Name, events[0], respBody, err)
|
||||
sender.NotifyRecord(ctx, events, notifyRuleId, notifyChannel.Name, "", respBody, err)
|
||||
}
|
||||
@@ -578,7 +586,7 @@ func SendNotifyRuleMessage(ctx *ctx.Context, userCache *memsto.UserCacheType, us
|
||||
case "script":
|
||||
start := time.Now()
|
||||
target, res, err := notifyChannel.SendScript(events, tplContent, customParams, sendtos)
|
||||
res = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), res)
|
||||
res = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), res)
|
||||
logger.Infof("script_sender notify_id: %d, channel_name: %v, event:%+v, tplContent:%s, customParams:%v, target:%s, res:%s, err:%v", notifyRuleId, notifyChannel.Name, events[0], tplContent, customParams, target, res, err)
|
||||
sender.NotifyRecord(ctx, events, notifyRuleId, notifyChannel.Name, target, res, err)
|
||||
default:
|
||||
@@ -825,12 +833,12 @@ func (e *Dispatch) HandleIbex(rule *models.AlertRule, event *models.AlertCurEven
|
||||
|
||||
if len(t.Host) == 0 {
|
||||
sender.CallIbex(e.ctx, t.TplId, event.TargetIdent,
|
||||
e.taskTplsCache, e.targetCache, e.userCache, event)
|
||||
e.taskTplsCache, e.targetCache, e.userCache, event, "")
|
||||
continue
|
||||
}
|
||||
for _, host := range t.Host {
|
||||
sender.CallIbex(e.ctx, t.TplId, host,
|
||||
e.taskTplsCache, e.targetCache, e.userCache, event)
|
||||
e.taskTplsCache, e.targetCache, e.userCache, event, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/alert/astats"
|
||||
@@ -24,6 +25,7 @@ import (
|
||||
"github.com/ccfos/nightingale/v6/pkg/poster"
|
||||
promsdk "github.com/ccfos/nightingale/v6/pkg/prom"
|
||||
promql2 "github.com/ccfos/nightingale/v6/pkg/promql"
|
||||
"github.com/ccfos/nightingale/v6/pkg/tplx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/unit"
|
||||
"github.com/ccfos/nightingale/v6/prom"
|
||||
"github.com/prometheus/common/model"
|
||||
@@ -60,6 +62,7 @@ const (
|
||||
CHECK_QUERY = "check_query_config"
|
||||
GET_CLIENT = "get_client"
|
||||
QUERY_DATA = "query_data"
|
||||
EXEC_TEMPLATE = "exec_template"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -151,7 +154,7 @@ func (arw *AlertRuleWorker) Eval() {
|
||||
if len(message) == 0 {
|
||||
logger.Infof("rule_eval:%s finished, duration:%v", arw.Key(), time.Since(begin))
|
||||
} else {
|
||||
logger.Infof("rule_eval:%s finished, duration:%v, message:%s", arw.Key(), time.Since(begin), message)
|
||||
logger.Warningf("rule_eval:%s finished, duration:%v, message:%s", arw.Key(), time.Since(begin), message)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -186,8 +189,7 @@ func (arw *AlertRuleWorker) Eval() {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
logger.Errorf("rule_eval:%s get anomaly point err:%s", arw.Key(), err.Error())
|
||||
message = "failed to get anomaly points"
|
||||
message = fmt.Sprintf("failed to get anomaly points: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -842,7 +844,7 @@ func (arw *AlertRuleWorker) GetHostAnomalyPoint(ruleConfig string) ([]models.Ano
|
||||
}
|
||||
m["ident"] = target.Ident
|
||||
|
||||
lst = append(lst, models.NewAnomalyPoint(trigger.Type, m, now, float64(now-target.UpdateAt), trigger.Severity))
|
||||
lst = append(lst, models.NewAnomalyPoint(trigger.Type, m, now, float64(now-target.BeatTime), trigger.Severity))
|
||||
}
|
||||
case "offset":
|
||||
idents, exists := arw.Processor.TargetsOfAlertRuleCache.Get(arw.Processor.EngineName, arw.Rule.Id)
|
||||
@@ -871,7 +873,7 @@ func (arw *AlertRuleWorker) GetHostAnomalyPoint(ruleConfig string) ([]models.Ano
|
||||
continue
|
||||
}
|
||||
if target, exists := targetMap[ident]; exists {
|
||||
if now-target.UpdateAt > 120 {
|
||||
if now-target.BeatTime > 120 {
|
||||
// means this target is not a active host, do not check offset
|
||||
continue
|
||||
}
|
||||
@@ -1484,6 +1486,16 @@ func (arw *AlertRuleWorker) GetAnomalyPoint(rule *models.AlertRule, dsId int64)
|
||||
return points, recoverPoints, fmt.Errorf("rule_eval:%d datasource:%d not exists", rule.Id, dsId)
|
||||
}
|
||||
|
||||
if err = ExecuteQueryTemplate(rule.Cate, query, nil); err != nil {
|
||||
logger.Warningf("rule_eval rid:%d execute query template error: %v", rule.Id, err)
|
||||
arw.Processor.Stats.CounterRuleEvalErrorTotal.WithLabelValues(fmt.Sprintf("%v", arw.Processor.DatasourceId()), EXEC_TEMPLATE, arw.Processor.BusiGroupCache.GetNameByBusiGroupId(arw.Rule.GroupId), fmt.Sprintf("%v", arw.Rule.Id)).Inc()
|
||||
arw.Processor.Stats.GaugeQuerySeriesCount.WithLabelValues(
|
||||
fmt.Sprintf("%v", arw.Rule.Id),
|
||||
fmt.Sprintf("%v", arw.Processor.DatasourceId()),
|
||||
fmt.Sprintf("%v", i),
|
||||
).Set(-3)
|
||||
}
|
||||
|
||||
ctx := context.WithValue(context.Background(), "delay", int64(rule.Delay))
|
||||
series, err := plug.QueryData(ctx, query)
|
||||
arw.Processor.Stats.CounterQueryDataTotal.WithLabelValues(fmt.Sprintf("%d", arw.DatasourceId), fmt.Sprintf("%d", rule.Id)).Inc()
|
||||
@@ -1602,11 +1614,15 @@ func (arw *AlertRuleWorker) GetAnomalyPoint(rule *models.AlertRule, dsId int64)
|
||||
continue
|
||||
}
|
||||
|
||||
switch v.(type) {
|
||||
case float64:
|
||||
values += fmt.Sprintf("%s:%.3f ", k, v)
|
||||
case string:
|
||||
values += fmt.Sprintf("%s:%s ", k, v)
|
||||
if u, exists := valuesUnitMap[k]; exists { // 配置了单位,优先用配置了单位的值
|
||||
values += fmt.Sprintf("%s:%s ", k, u.Text)
|
||||
} else {
|
||||
switch v.(type) {
|
||||
case float64:
|
||||
values += fmt.Sprintf("%s:%.3f ", k, v)
|
||||
case string:
|
||||
values += fmt.Sprintf("%s:%s ", k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1699,3 +1715,61 @@ func (arw *AlertRuleWorker) GetAnomalyPoint(rule *models.AlertRule, dsId int64)
|
||||
|
||||
return points, recoverPoints, nil
|
||||
}
|
||||
|
||||
// ExecuteQueryTemplate 根据数据源类型对 Query 进行模板渲染处理
|
||||
// cate: 数据源类别,如 "mysql", "pgsql" 等
|
||||
// query: 查询对象,如果是数据库类型的数据源,会处理其中的 sql 字段
|
||||
// data: 模板数据对象,如果为 nil 则使用空结构体(不支持变量渲染),如果不为 nil 则使用传入的数据(支持变量渲染)
|
||||
func ExecuteQueryTemplate(cate string, query interface{}, data interface{}) error {
|
||||
// 检查 query 是否是 map,且包含 sql 字段
|
||||
queryMap, ok := query.(map[string]interface{})
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
sqlVal, exists := queryMap["sql"]
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
sqlStr, ok := sqlVal.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 调用 ExecuteSqlTemplate 处理 sql 字段
|
||||
processedSQL, err := ExecuteSqlTemplate(sqlStr, data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute sql template error: %w", err)
|
||||
}
|
||||
|
||||
// 更新 query 中的 sql 字段
|
||||
queryMap["sql"] = processedSQL
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExecuteSqlTemplate 执行 query 中的 golang 模板语法函数
|
||||
// query: 要处理的 query 字符串
|
||||
// data: 模板数据对象,如果为 nil 则使用空结构体(不支持变量渲染),如果不为 nil 则使用传入的数据(支持变量渲染)
|
||||
func ExecuteSqlTemplate(query string, data interface{}) (string, error) {
|
||||
if !strings.Contains(query, "{{") || !strings.Contains(query, "}}") {
|
||||
return query, nil
|
||||
}
|
||||
|
||||
tmpl, err := template.New("query").Funcs(tplx.TemplateFuncMap).Parse(query)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("query tmpl parse error: %w", err)
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
templateData := data
|
||||
if templateData == nil {
|
||||
templateData = struct{}{}
|
||||
}
|
||||
|
||||
if err := tmpl.Execute(&buf, templateData); err != nil {
|
||||
return "", fmt.Errorf("query tmpl execute error: %w", err)
|
||||
}
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
380
alert/pipeline/engine/engine.go
Normal file
380
alert/pipeline/engine/engine.go
Normal file
@@ -0,0 +1,380 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/google/uuid"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
)
|
||||
|
||||
type WorkflowEngine struct {
|
||||
ctx *ctx.Context
|
||||
}
|
||||
|
||||
func NewWorkflowEngine(c *ctx.Context) *WorkflowEngine {
|
||||
return &WorkflowEngine{ctx: c}
|
||||
}
|
||||
|
||||
func (e *WorkflowEngine) Execute(pipeline *models.EventPipeline, event *models.AlertCurEvent, triggerCtx *models.WorkflowTriggerContext) (*models.AlertCurEvent, *models.WorkflowResult, error) {
|
||||
startTime := time.Now()
|
||||
|
||||
wfCtx := e.initWorkflowContext(pipeline, event, triggerCtx)
|
||||
|
||||
nodes := pipeline.GetWorkflowNodes()
|
||||
connections := pipeline.GetWorkflowConnections()
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return event, &models.WorkflowResult{
|
||||
Event: event,
|
||||
Status: models.ExecutionStatusSuccess,
|
||||
Message: "no nodes to execute",
|
||||
}, nil
|
||||
}
|
||||
|
||||
nodeMap := make(map[string]*models.WorkflowNode)
|
||||
for i := range nodes {
|
||||
if nodes[i].RetryInterval == 0 {
|
||||
nodes[i].RetryInterval = 1
|
||||
}
|
||||
|
||||
if nodes[i].MaxRetries == 0 {
|
||||
nodes[i].MaxRetries = 1
|
||||
}
|
||||
|
||||
nodeMap[nodes[i].ID] = &nodes[i]
|
||||
}
|
||||
|
||||
result := e.executeDAG(nodeMap, connections, wfCtx)
|
||||
result.Event = wfCtx.Event
|
||||
|
||||
duration := time.Since(startTime).Milliseconds()
|
||||
|
||||
if triggerCtx != nil && triggerCtx.Mode != "" {
|
||||
e.saveExecutionRecord(pipeline, wfCtx, result, triggerCtx, startTime.Unix(), duration)
|
||||
}
|
||||
|
||||
return wfCtx.Event, result, nil
|
||||
}
|
||||
|
||||
func (e *WorkflowEngine) initWorkflowContext(pipeline *models.EventPipeline, event *models.AlertCurEvent, triggerCtx *models.WorkflowTriggerContext) *models.WorkflowContext {
|
||||
// 合并输入参数
|
||||
inputs := pipeline.GetInputsMap()
|
||||
if triggerCtx != nil && triggerCtx.InputsOverrides != nil {
|
||||
for k, v := range triggerCtx.InputsOverrides {
|
||||
inputs[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
metadata := map[string]string{
|
||||
"start_time": fmt.Sprintf("%d", time.Now().Unix()),
|
||||
"pipeline_id": fmt.Sprintf("%d", pipeline.ID),
|
||||
}
|
||||
|
||||
// 是否启用流式输出
|
||||
stream := false
|
||||
if triggerCtx != nil {
|
||||
metadata["request_id"] = triggerCtx.RequestID
|
||||
metadata["trigger_mode"] = triggerCtx.Mode
|
||||
metadata["trigger_by"] = triggerCtx.TriggerBy
|
||||
stream = triggerCtx.Stream
|
||||
}
|
||||
|
||||
return &models.WorkflowContext{
|
||||
Event: event,
|
||||
Inputs: inputs,
|
||||
Vars: make(map[string]interface{}), // 初始化空的 Vars,供节点间传递数据
|
||||
Metadata: metadata,
|
||||
Stream: stream,
|
||||
}
|
||||
}
|
||||
|
||||
// executeDAG 使用 Kahn 算法执行 DAG
|
||||
func (e *WorkflowEngine) executeDAG(nodeMap map[string]*models.WorkflowNode, connections models.Connections, wfCtx *models.WorkflowContext) *models.WorkflowResult {
|
||||
result := &models.WorkflowResult{
|
||||
Status: models.ExecutionStatusSuccess,
|
||||
NodeResults: make([]*models.NodeExecutionResult, 0),
|
||||
Stream: wfCtx.Stream, // 从上下文继承流式输出设置
|
||||
}
|
||||
|
||||
// 计算每个节点的入度
|
||||
inDegree := make(map[string]int)
|
||||
for nodeID := range nodeMap {
|
||||
inDegree[nodeID] = 0
|
||||
}
|
||||
|
||||
// 遍历连接,计算入度
|
||||
for _, nodeConns := range connections {
|
||||
for _, targets := range nodeConns.Main {
|
||||
for _, target := range targets {
|
||||
inDegree[target.Node]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 找到所有入度为 0 的节点(起始节点)
|
||||
queue := make([]string, 0)
|
||||
for nodeID, degree := range inDegree {
|
||||
if degree == 0 {
|
||||
queue = append(queue, nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有起始节点,说明存在循环依赖
|
||||
if len(queue) == 0 && len(nodeMap) > 0 {
|
||||
result.Status = models.ExecutionStatusFailed
|
||||
result.Message = "workflow has circular dependency"
|
||||
return result
|
||||
}
|
||||
|
||||
// 记录已执行的节点
|
||||
executed := make(map[string]bool)
|
||||
// 记录节点的分支选择结果
|
||||
branchResults := make(map[string]*int)
|
||||
|
||||
for len(queue) > 0 {
|
||||
// 取出队首节点
|
||||
nodeID := queue[0]
|
||||
queue = queue[1:]
|
||||
|
||||
// 检查是否已执行
|
||||
if executed[nodeID] {
|
||||
continue
|
||||
}
|
||||
|
||||
node, exists := nodeMap[nodeID]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
// 执行节点
|
||||
nodeResult, nodeOutput := e.executeNode(node, wfCtx)
|
||||
result.NodeResults = append(result.NodeResults, nodeResult)
|
||||
|
||||
if nodeOutput != nil && nodeOutput.Stream && nodeOutput.StreamChan != nil {
|
||||
// 流式输出节点通常是最后一个节点
|
||||
// 直接传递 StreamChan 给 WorkflowResult,不阻塞等待
|
||||
result.Stream = true
|
||||
result.StreamChan = nodeOutput.StreamChan
|
||||
result.Event = wfCtx.Event
|
||||
result.Status = "streaming"
|
||||
result.Message = fmt.Sprintf("streaming output from node: %s", node.Name)
|
||||
|
||||
// 更新节点状态为 streaming
|
||||
nodeResult.Status = "streaming"
|
||||
nodeResult.Message = "streaming in progress"
|
||||
|
||||
// 立即返回,让 API 层处理流式响应
|
||||
return result
|
||||
}
|
||||
executed[nodeID] = true
|
||||
|
||||
// 保存分支结果
|
||||
if nodeResult.BranchIndex != nil {
|
||||
branchResults[nodeID] = nodeResult.BranchIndex
|
||||
}
|
||||
|
||||
// 检查执行状态
|
||||
if nodeResult.Status == "failed" {
|
||||
if !node.ContinueOnFail {
|
||||
result.Status = models.ExecutionStatusFailed
|
||||
result.ErrorNode = nodeID
|
||||
result.Message = fmt.Sprintf("node %s failed: %s", node.Name, nodeResult.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否终止
|
||||
if nodeResult.Status == "terminated" {
|
||||
result.Message = fmt.Sprintf("workflow terminated at node %s", node.Name)
|
||||
return result
|
||||
}
|
||||
|
||||
// 更新后继节点的入度
|
||||
if nodeConns, ok := connections[nodeID]; ok {
|
||||
for outputIndex, targets := range nodeConns.Main {
|
||||
// 检查是否应该走这个分支
|
||||
if !e.shouldFollowBranch(nodeID, outputIndex, branchResults) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, target := range targets {
|
||||
inDegree[target.Node]--
|
||||
if inDegree[target.Node] == 0 {
|
||||
queue = append(queue, target.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// executeNode 执行单个节点
|
||||
// 返回:节点执行结果、节点输出(用于流式输出检测)
|
||||
func (e *WorkflowEngine) executeNode(node *models.WorkflowNode, wfCtx *models.WorkflowContext) (*models.NodeExecutionResult, *models.NodeOutput) {
|
||||
startTime := time.Now()
|
||||
nodeResult := &models.NodeExecutionResult{
|
||||
NodeID: node.ID,
|
||||
NodeName: node.Name,
|
||||
NodeType: node.Type,
|
||||
StartedAt: startTime.Unix(),
|
||||
}
|
||||
|
||||
var nodeOutput *models.NodeOutput
|
||||
|
||||
// 跳过禁用的节点
|
||||
if node.Disabled {
|
||||
nodeResult.Status = "skipped"
|
||||
nodeResult.Message = "node is disabled"
|
||||
nodeResult.FinishedAt = time.Now().Unix()
|
||||
nodeResult.DurationMs = time.Since(startTime).Milliseconds()
|
||||
return nodeResult, nil
|
||||
}
|
||||
|
||||
// 获取处理器
|
||||
processor, err := models.GetProcessorByType(node.Type, node.Config)
|
||||
if err != nil {
|
||||
nodeResult.Status = "failed"
|
||||
nodeResult.Error = fmt.Sprintf("failed to get processor: %v", err)
|
||||
nodeResult.FinishedAt = time.Now().Unix()
|
||||
nodeResult.DurationMs = time.Since(startTime).Milliseconds()
|
||||
return nodeResult, nil
|
||||
}
|
||||
|
||||
// 执行处理器(带重试)
|
||||
var retries int
|
||||
maxRetries := node.MaxRetries
|
||||
if !node.RetryOnFail {
|
||||
maxRetries = 0
|
||||
}
|
||||
|
||||
for retries <= maxRetries {
|
||||
// 检查是否为分支处理器
|
||||
if branchProcessor, ok := processor.(models.BranchProcessor); ok {
|
||||
output, err := branchProcessor.ProcessWithBranch(e.ctx, wfCtx)
|
||||
if err != nil {
|
||||
if retries < maxRetries {
|
||||
retries++
|
||||
time.Sleep(time.Duration(node.RetryInterval) * time.Second)
|
||||
continue
|
||||
}
|
||||
nodeResult.Status = "failed"
|
||||
nodeResult.Error = err.Error()
|
||||
} else {
|
||||
nodeResult.Status = "success"
|
||||
if output != nil {
|
||||
nodeOutput = output
|
||||
if output.WfCtx != nil {
|
||||
wfCtx = output.WfCtx
|
||||
}
|
||||
nodeResult.Message = output.Message
|
||||
nodeResult.BranchIndex = output.BranchIndex
|
||||
if output.Terminate {
|
||||
nodeResult.Status = "terminated"
|
||||
}
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// 普通处理器
|
||||
newWfCtx, msg, err := processor.Process(e.ctx, wfCtx)
|
||||
if err != nil {
|
||||
if retries < maxRetries {
|
||||
retries++
|
||||
time.Sleep(time.Duration(node.RetryInterval) * time.Second)
|
||||
continue
|
||||
}
|
||||
nodeResult.Status = "failed"
|
||||
nodeResult.Error = err.Error()
|
||||
} else {
|
||||
nodeResult.Status = "success"
|
||||
nodeResult.Message = msg
|
||||
if newWfCtx != nil {
|
||||
wfCtx = newWfCtx
|
||||
|
||||
// 检测流式输出标记
|
||||
if newWfCtx.Stream && newWfCtx.StreamChan != nil {
|
||||
nodeOutput = &models.NodeOutput{
|
||||
WfCtx: newWfCtx,
|
||||
Message: msg,
|
||||
Stream: true,
|
||||
StreamChan: newWfCtx.StreamChan,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果事件被 drop(返回 nil 或 Event 为 nil),标记为终止
|
||||
if newWfCtx == nil || newWfCtx.Event == nil {
|
||||
nodeResult.Status = "terminated"
|
||||
nodeResult.Message = msg
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
nodeResult.FinishedAt = time.Now().Unix()
|
||||
nodeResult.DurationMs = time.Since(startTime).Milliseconds()
|
||||
|
||||
logger.Infof("workflow: executed node %s (type=%s) status=%s msg=%s duration=%dms",
|
||||
node.Name, node.Type, nodeResult.Status, nodeResult.Message, nodeResult.DurationMs)
|
||||
|
||||
return nodeResult, nodeOutput
|
||||
}
|
||||
|
||||
// shouldFollowBranch 判断是否应该走某个分支
|
||||
func (e *WorkflowEngine) shouldFollowBranch(nodeID string, outputIndex int, branchResults map[string]*int) bool {
|
||||
branchIndex, hasBranch := branchResults[nodeID]
|
||||
if !hasBranch {
|
||||
// 没有分支结果,说明不是分支节点,只走第一个输出
|
||||
return outputIndex == 0
|
||||
}
|
||||
|
||||
if branchIndex == nil {
|
||||
// branchIndex 为 nil,走默认分支(通常是最后一个)
|
||||
return true
|
||||
}
|
||||
|
||||
// 只走选中的分支
|
||||
return outputIndex == *branchIndex
|
||||
}
|
||||
|
||||
func (e *WorkflowEngine) saveExecutionRecord(pipeline *models.EventPipeline, wfCtx *models.WorkflowContext, result *models.WorkflowResult, triggerCtx *models.WorkflowTriggerContext, startTime int64, duration int64) {
|
||||
executionID := triggerCtx.RequestID
|
||||
if executionID == "" {
|
||||
executionID = uuid.New().String()
|
||||
}
|
||||
|
||||
execution := &models.EventPipelineExecution{
|
||||
ID: executionID,
|
||||
PipelineID: pipeline.ID,
|
||||
PipelineName: pipeline.Name,
|
||||
Mode: triggerCtx.Mode,
|
||||
Status: result.Status,
|
||||
ErrorMessage: result.Message,
|
||||
ErrorNode: result.ErrorNode,
|
||||
CreatedAt: startTime,
|
||||
FinishedAt: time.Now().Unix(),
|
||||
DurationMs: duration,
|
||||
TriggerBy: triggerCtx.TriggerBy,
|
||||
}
|
||||
|
||||
if wfCtx.Event != nil {
|
||||
execution.EventID = wfCtx.Event.Id
|
||||
}
|
||||
|
||||
if err := execution.SetNodeResults(result.NodeResults); err != nil {
|
||||
logger.Errorf("workflow: failed to set node results: pipeline_id=%d, error=%v", pipeline.ID, err)
|
||||
}
|
||||
|
||||
if err := execution.SetInputsSnapshot(wfCtx.Inputs); err != nil {
|
||||
logger.Errorf("workflow: failed to set inputs snapshot: pipeline_id=%d, error=%v", pipeline.ID, err)
|
||||
}
|
||||
|
||||
if err := models.CreateEventPipelineExecution(e.ctx, execution); err != nil {
|
||||
logger.Errorf("workflow: failed to save execution record: pipeline_id=%d, error=%v", pipeline.ID, err)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
_ "github.com/ccfos/nightingale/v6/alert/pipeline/processor/callback"
|
||||
_ "github.com/ccfos/nightingale/v6/alert/pipeline/processor/eventdrop"
|
||||
_ "github.com/ccfos/nightingale/v6/alert/pipeline/processor/eventupdate"
|
||||
_ "github.com/ccfos/nightingale/v6/alert/pipeline/processor/logic"
|
||||
_ "github.com/ccfos/nightingale/v6/alert/pipeline/processor/relabel"
|
||||
)
|
||||
|
||||
|
||||
@@ -55,23 +55,24 @@ func (c *AISummaryConfig) Init(settings interface{}) (models.Processor, error) {
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (c *AISummaryConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent) (*models.AlertCurEvent, string, error) {
|
||||
func (c *AISummaryConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
event := wfCtx.Event
|
||||
if c.Client == nil {
|
||||
if err := c.initHTTPClient(); err != nil {
|
||||
return event, "", fmt.Errorf("failed to initialize HTTP client: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to initialize HTTP client: %v processor: %v", err, c)
|
||||
}
|
||||
}
|
||||
|
||||
// 准备告警事件信息
|
||||
eventInfo, err := c.prepareEventInfo(event)
|
||||
eventInfo, err := c.prepareEventInfo(wfCtx)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to prepare event info: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to prepare event info: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
// 调用AI模型生成总结
|
||||
summary, err := c.generateAISummary(eventInfo)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to generate AI summary: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to generate AI summary: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
// 将总结添加到annotations字段
|
||||
@@ -83,11 +84,11 @@ func (c *AISummaryConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent)
|
||||
// 更新Annotations字段
|
||||
b, err := json.Marshal(event.AnnotationsJSON)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to marshal annotations: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to marshal annotations: %v processor: %v", err, c)
|
||||
}
|
||||
event.Annotations = string(b)
|
||||
|
||||
return event, "", nil
|
||||
return wfCtx, "", nil
|
||||
}
|
||||
|
||||
func (c *AISummaryConfig) initHTTPClient() error {
|
||||
@@ -110,9 +111,10 @@ func (c *AISummaryConfig) initHTTPClient() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *AISummaryConfig) prepareEventInfo(event *models.AlertCurEvent) (string, error) {
|
||||
func (c *AISummaryConfig) prepareEventInfo(wfCtx *models.WorkflowContext) (string, error) {
|
||||
var defs = []string{
|
||||
"{{$event := .}}",
|
||||
"{{$event := .Event}}",
|
||||
"{{$inputs := .Inputs}}",
|
||||
}
|
||||
|
||||
text := strings.Join(append(defs, c.PromptTemplate), "")
|
||||
@@ -122,7 +124,7 @@ func (c *AISummaryConfig) prepareEventInfo(event *models.AlertCurEvent) (string,
|
||||
}
|
||||
|
||||
var body bytes.Buffer
|
||||
err = t.Execute(&body, event)
|
||||
err = t.Execute(&body, wfCtx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to execute prompt template: %v", err)
|
||||
}
|
||||
|
||||
@@ -42,8 +42,14 @@ func TestAISummaryConfig_Process(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
// 创建 WorkflowContext
|
||||
wfCtx := &models.WorkflowContext{
|
||||
Event: event,
|
||||
Inputs: map[string]string{},
|
||||
}
|
||||
|
||||
// 测试模板处理
|
||||
eventInfo, err := config.prepareEventInfo(event)
|
||||
eventInfo, err := config.prepareEventInfo(wfCtx)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, eventInfo, "Test Rule")
|
||||
assert.Contains(t, eventInfo, "1")
|
||||
@@ -54,18 +60,18 @@ func TestAISummaryConfig_Process(t *testing.T) {
|
||||
assert.NotNil(t, processor)
|
||||
|
||||
// 测试处理函数
|
||||
result, _, err := processor.Process(&ctx.Context{}, event)
|
||||
result, _, err := processor.Process(&ctx.Context{}, wfCtx)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.NotEmpty(t, result.AnnotationsJSON["ai_summary"])
|
||||
assert.NotEmpty(t, result.Event.AnnotationsJSON["ai_summary"])
|
||||
|
||||
// 展示处理结果
|
||||
t.Log("\n=== 处理结果 ===")
|
||||
t.Logf("告警规则: %s", result.RuleName)
|
||||
t.Logf("严重程度: %d", result.Severity)
|
||||
t.Logf("标签: %v", result.TagsMap)
|
||||
t.Logf("原始注释: %v", result.AnnotationsJSON["description"])
|
||||
t.Logf("AI总结: %s", result.AnnotationsJSON["ai_summary"])
|
||||
t.Logf("告警规则: %s", result.Event.RuleName)
|
||||
t.Logf("严重程度: %d", result.Event.Severity)
|
||||
t.Logf("标签: %v", result.Event.TagsMap)
|
||||
t.Logf("原始注释: %v", result.Event.AnnotationsJSON["description"])
|
||||
t.Logf("AI总结: %s", result.Event.AnnotationsJSON["ai_summary"])
|
||||
}
|
||||
|
||||
func TestConvertCustomParam(t *testing.T) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline/processor/common"
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline/processor/utils"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
@@ -43,7 +44,8 @@ func (c *CallbackConfig) Init(settings interface{}) (models.Processor, error) {
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (c *CallbackConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent) (*models.AlertCurEvent, string, error) {
|
||||
func (c *CallbackConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
event := wfCtx.Event
|
||||
if c.Client == nil {
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: c.SkipSSLVerify},
|
||||
@@ -52,7 +54,7 @@ func (c *CallbackConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent)
|
||||
if c.Proxy != "" {
|
||||
proxyURL, err := url.Parse(c.Proxy)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to parse proxy url: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to parse proxy url: %v processor: %v", err, c)
|
||||
} else {
|
||||
transport.Proxy = http.ProxyURL(proxyURL)
|
||||
}
|
||||
@@ -70,14 +72,19 @@ func (c *CallbackConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent)
|
||||
headers[k] = v
|
||||
}
|
||||
|
||||
body, err := json.Marshal(event)
|
||||
url, err := utils.TplRender(wfCtx, c.URL)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to marshal event: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to render url template: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", c.URL, strings.NewReader(string(body)))
|
||||
body, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to create request: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to marshal event: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return wfCtx, "", fmt.Errorf("failed to create request: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
for k, v := range headers {
|
||||
@@ -90,14 +97,14 @@ func (c *CallbackConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent)
|
||||
|
||||
resp, err := c.Client.Do(req)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to send request: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to send request: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to read response body: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to read response body: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
logger.Debugf("callback processor response body: %s", string(b))
|
||||
return event, "callback success", nil
|
||||
return wfCtx, "callback success", nil
|
||||
}
|
||||
|
||||
@@ -26,35 +26,38 @@ func (c *EventDropConfig) Init(settings interface{}) (models.Processor, error) {
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (c *EventDropConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent) (*models.AlertCurEvent, string, error) {
|
||||
func (c *EventDropConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
// 使用背景是可以根据此处理器,实现对事件进行更加灵活的过滤的逻辑
|
||||
// 在标签过滤和属性过滤都不满足需求时可以使用
|
||||
// 如果模板执行结果为 true,则删除该事件
|
||||
event := wfCtx.Event
|
||||
|
||||
var defs = []string{
|
||||
"{{ $event := . }}",
|
||||
"{{ $labels := .TagsMap }}",
|
||||
"{{ $value := .TriggerValue }}",
|
||||
"{{ $event := .Event }}",
|
||||
"{{ $labels := .Event.TagsMap }}",
|
||||
"{{ $value := .Event.TriggerValue }}",
|
||||
"{{ $inputs := .Inputs }}",
|
||||
}
|
||||
|
||||
text := strings.Join(append(defs, c.Content), "")
|
||||
|
||||
tpl, err := texttemplate.New("eventdrop").Funcs(tplx.TemplateFuncMap).Parse(text)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("processor failed to parse template: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("processor failed to parse template: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
var body bytes.Buffer
|
||||
if err = tpl.Execute(&body, event); err != nil {
|
||||
return event, "", fmt.Errorf("processor failed to execute template: %v processor: %v", err, c)
|
||||
if err = tpl.Execute(&body, wfCtx); err != nil {
|
||||
return wfCtx, "", fmt.Errorf("processor failed to execute template: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
result := strings.TrimSpace(body.String())
|
||||
logger.Infof("processor eventdrop result: %v", result)
|
||||
if result == "true" {
|
||||
wfCtx.Event = nil
|
||||
logger.Infof("processor eventdrop drop event: %v", event)
|
||||
return nil, "drop event success", nil
|
||||
return wfCtx, "drop event success", nil
|
||||
}
|
||||
|
||||
return event, "drop event failed", nil
|
||||
return wfCtx, "drop event failed", nil
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ func (c *EventUpdateConfig) Init(settings interface{}) (models.Processor, error)
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (c *EventUpdateConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent) (*models.AlertCurEvent, string, error) {
|
||||
func (c *EventUpdateConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
event := wfCtx.Event
|
||||
if c.Client == nil {
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: c.SkipSSLVerify},
|
||||
@@ -40,7 +41,7 @@ func (c *EventUpdateConfig) Process(ctx *ctx.Context, event *models.AlertCurEven
|
||||
if c.Proxy != "" {
|
||||
proxyURL, err := url.Parse(c.Proxy)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to parse proxy url: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to parse proxy url: %v processor: %v", err, c)
|
||||
} else {
|
||||
transport.Proxy = http.ProxyURL(proxyURL)
|
||||
}
|
||||
@@ -60,12 +61,12 @@ func (c *EventUpdateConfig) Process(ctx *ctx.Context, event *models.AlertCurEven
|
||||
|
||||
body, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to marshal event: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to marshal event: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", c.URL, strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to create request: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to create request: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
for k, v := range headers {
|
||||
@@ -78,7 +79,7 @@ func (c *EventUpdateConfig) Process(ctx *ctx.Context, event *models.AlertCurEven
|
||||
|
||||
resp, err := c.Client.Do(req)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to send request: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to send request: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
@@ -89,8 +90,8 @@ func (c *EventUpdateConfig) Process(ctx *ctx.Context, event *models.AlertCurEven
|
||||
|
||||
err = json.Unmarshal(b, &event)
|
||||
if err != nil {
|
||||
return event, "", fmt.Errorf("failed to unmarshal response body: %v processor: %v", err, c)
|
||||
return wfCtx, "", fmt.Errorf("failed to unmarshal response body: %v processor: %v", err, c)
|
||||
}
|
||||
|
||||
return event, "", nil
|
||||
return wfCtx, "", nil
|
||||
}
|
||||
|
||||
197
alert/pipeline/processor/logic/if.go
Normal file
197
alert/pipeline/processor/logic/if.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
alertCommon "github.com/ccfos/nightingale/v6/alert/common"
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline/processor/common"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/tplx"
|
||||
)
|
||||
|
||||
// 判断模式常量
|
||||
const (
|
||||
ConditionModeExpression = "expression" // 表达式模式(默认)
|
||||
ConditionModeTags = "tags" // 标签/属性模式
|
||||
)
|
||||
|
||||
// IfConfig If 条件处理器配置
|
||||
type IfConfig struct {
|
||||
// 判断模式:expression(表达式)或 tags(标签/属性)
|
||||
Mode string `json:"mode,omitempty"`
|
||||
|
||||
// 表达式模式配置
|
||||
// 条件表达式(支持 Go 模板语法)
|
||||
// 例如:{{ if eq .Severity 1 }}true{{ end }}
|
||||
Condition string `json:"condition,omitempty"`
|
||||
|
||||
// 标签/属性模式配置
|
||||
LabelKeys []models.TagFilter `json:"label_keys,omitempty"` // 适用标签
|
||||
Attributes []models.TagFilter `json:"attributes,omitempty"` // 适用属性
|
||||
|
||||
// 内部使用,解析后的过滤器
|
||||
parsedLabelKeys []models.TagFilter `json:"-"`
|
||||
parsedAttributes []models.TagFilter `json:"-"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
models.RegisterProcessor("logic.if", &IfConfig{})
|
||||
}
|
||||
|
||||
func (c *IfConfig) Init(settings interface{}) (models.Processor, error) {
|
||||
result, err := common.InitProcessor[*IfConfig](settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 解析标签过滤器
|
||||
if len(result.LabelKeys) > 0 {
|
||||
// Deep copy to avoid concurrent map writes on cached objects
|
||||
labelKeysCopy := make([]models.TagFilter, len(result.LabelKeys))
|
||||
copy(labelKeysCopy, result.LabelKeys)
|
||||
for i := range labelKeysCopy {
|
||||
if labelKeysCopy[i].Func == "" {
|
||||
labelKeysCopy[i].Func = labelKeysCopy[i].Op
|
||||
}
|
||||
}
|
||||
result.parsedLabelKeys, err = models.ParseTagFilter(labelKeysCopy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse label_keys: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 解析属性过滤器
|
||||
if len(result.Attributes) > 0 {
|
||||
// Deep copy to avoid concurrent map writes on cached objects
|
||||
attributesCopy := make([]models.TagFilter, len(result.Attributes))
|
||||
copy(attributesCopy, result.Attributes)
|
||||
for i := range attributesCopy {
|
||||
if attributesCopy[i].Func == "" {
|
||||
attributesCopy[i].Func = attributesCopy[i].Op
|
||||
}
|
||||
}
|
||||
result.parsedAttributes, err = models.ParseTagFilter(attributesCopy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse attributes: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Process 实现 Processor 接口(兼容旧模式)
|
||||
func (c *IfConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
result, err := c.evaluateCondition(wfCtx)
|
||||
if err != nil {
|
||||
return wfCtx, "", fmt.Errorf("if processor: failed to evaluate condition: %v", err)
|
||||
}
|
||||
|
||||
if result {
|
||||
return wfCtx, "condition matched (true branch)", nil
|
||||
}
|
||||
return wfCtx, "condition not matched (false branch)", nil
|
||||
}
|
||||
|
||||
// ProcessWithBranch 实现 BranchProcessor 接口
|
||||
func (c *IfConfig) ProcessWithBranch(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.NodeOutput, error) {
|
||||
result, err := c.evaluateCondition(wfCtx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("if processor: failed to evaluate condition: %v", err)
|
||||
}
|
||||
|
||||
output := &models.NodeOutput{
|
||||
WfCtx: wfCtx,
|
||||
}
|
||||
|
||||
if result {
|
||||
// 条件为 true,走输出 0(true 分支)
|
||||
branchIndex := 0
|
||||
output.BranchIndex = &branchIndex
|
||||
output.Message = "condition matched (true branch)"
|
||||
} else {
|
||||
// 条件为 false,走输出 1(false 分支)
|
||||
branchIndex := 1
|
||||
output.BranchIndex = &branchIndex
|
||||
output.Message = "condition not matched (false branch)"
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// evaluateCondition 评估条件
|
||||
func (c *IfConfig) evaluateCondition(wfCtx *models.WorkflowContext) (bool, error) {
|
||||
mode := c.Mode
|
||||
if mode == "" {
|
||||
mode = ConditionModeExpression // 默认表达式模式
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case ConditionModeTags:
|
||||
return c.evaluateTagsCondition(wfCtx.Event)
|
||||
default:
|
||||
return c.evaluateExpressionCondition(wfCtx)
|
||||
}
|
||||
}
|
||||
|
||||
// evaluateExpressionCondition 评估表达式条件
|
||||
func (c *IfConfig) evaluateExpressionCondition(wfCtx *models.WorkflowContext) (bool, error) {
|
||||
if c.Condition == "" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// 构建模板数据
|
||||
var defs = []string{
|
||||
"{{ $event := .Event }}",
|
||||
"{{ $labels := .Event.TagsMap }}",
|
||||
"{{ $value := .Event.TriggerValue }}",
|
||||
"{{ $inputs := .Inputs }}",
|
||||
}
|
||||
|
||||
text := strings.Join(append(defs, c.Condition), "")
|
||||
|
||||
tpl, err := template.New("if_condition").Funcs(tplx.TemplateFuncMap).Parse(text)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err = tpl.Execute(&buf, wfCtx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
result := strings.TrimSpace(strings.ToLower(buf.String()))
|
||||
return result == "true" || result == "1", nil
|
||||
}
|
||||
|
||||
// evaluateTagsCondition 评估标签/属性条件
|
||||
func (c *IfConfig) evaluateTagsCondition(event *models.AlertCurEvent) (bool, error) {
|
||||
// 如果没有配置任何过滤条件,默认返回 true
|
||||
if len(c.parsedLabelKeys) == 0 && len(c.parsedAttributes) == 0 {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// 匹配标签 (TagsMap)
|
||||
if len(c.parsedLabelKeys) > 0 {
|
||||
tagsMap := event.TagsMap
|
||||
if tagsMap == nil {
|
||||
tagsMap = make(map[string]string)
|
||||
}
|
||||
if !alertCommon.MatchTags(tagsMap, c.parsedLabelKeys) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 匹配属性 (JsonTagsAndValue - 所有 JSON 字段)
|
||||
if len(c.parsedAttributes) > 0 {
|
||||
attributesMap := event.JsonTagsAndValue()
|
||||
if !alertCommon.MatchTags(attributesMap, c.parsedAttributes) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
224
alert/pipeline/processor/logic/switch.go
Normal file
224
alert/pipeline/processor/logic/switch.go
Normal file
@@ -0,0 +1,224 @@
|
||||
package logic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
alertCommon "github.com/ccfos/nightingale/v6/alert/common"
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline/processor/common"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/tplx"
|
||||
)
|
||||
|
||||
// SwitchCase Switch 分支定义
|
||||
type SwitchCase struct {
|
||||
// 判断模式:expression(表达式)或 tags(标签/属性)
|
||||
Mode string `json:"mode,omitempty"`
|
||||
|
||||
// 表达式模式配置
|
||||
// 条件表达式(支持 Go 模板语法)
|
||||
Condition string `json:"condition,omitempty"`
|
||||
|
||||
// 标签/属性模式配置
|
||||
LabelKeys []models.TagFilter `json:"label_keys,omitempty"` // 适用标签
|
||||
Attributes []models.TagFilter `json:"attributes,omitempty"` // 适用属性
|
||||
|
||||
// 分支名称(可选,用于日志)
|
||||
Name string `json:"name,omitempty"`
|
||||
|
||||
// 内部使用,解析后的过滤器
|
||||
parsedLabelKeys []models.TagFilter `json:"-"`
|
||||
parsedAttributes []models.TagFilter `json:"-"`
|
||||
}
|
||||
|
||||
// SwitchConfig Switch 多分支处理器配置
|
||||
type SwitchConfig struct {
|
||||
// 分支条件列表
|
||||
// 按顺序匹配,第一个为 true 的分支将被选中
|
||||
Cases []SwitchCase `json:"cases"`
|
||||
// 是否允许多个分支同时匹配(默认 false,只走第一个匹配的)
|
||||
AllowMultiple bool `json:"allow_multiple,omitempty"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
models.RegisterProcessor("logic.switch", &SwitchConfig{})
|
||||
}
|
||||
|
||||
func (c *SwitchConfig) Init(settings interface{}) (models.Processor, error) {
|
||||
result, err := common.InitProcessor[*SwitchConfig](settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 解析每个 case 的标签和属性过滤器
|
||||
for i := range result.Cases {
|
||||
if len(result.Cases[i].LabelKeys) > 0 {
|
||||
// Deep copy to avoid concurrent map writes on cached objects
|
||||
labelKeysCopy := make([]models.TagFilter, len(result.Cases[i].LabelKeys))
|
||||
copy(labelKeysCopy, result.Cases[i].LabelKeys)
|
||||
for j := range labelKeysCopy {
|
||||
if labelKeysCopy[j].Func == "" {
|
||||
labelKeysCopy[j].Func = labelKeysCopy[j].Op
|
||||
}
|
||||
}
|
||||
result.Cases[i].parsedLabelKeys, err = models.ParseTagFilter(labelKeysCopy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse label_keys for case[%d]: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(result.Cases[i].Attributes) > 0 {
|
||||
// Deep copy to avoid concurrent map writes on cached objects
|
||||
attributesCopy := make([]models.TagFilter, len(result.Cases[i].Attributes))
|
||||
copy(attributesCopy, result.Cases[i].Attributes)
|
||||
for j := range attributesCopy {
|
||||
if attributesCopy[j].Func == "" {
|
||||
attributesCopy[j].Func = attributesCopy[j].Op
|
||||
}
|
||||
}
|
||||
result.Cases[i].parsedAttributes, err = models.ParseTagFilter(attributesCopy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse attributes for case[%d]: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Process 实现 Processor 接口(兼容旧模式)
|
||||
func (c *SwitchConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
index, caseName, err := c.evaluateCases(wfCtx)
|
||||
if err != nil {
|
||||
return wfCtx, "", fmt.Errorf("switch processor: failed to evaluate cases: %v", err)
|
||||
}
|
||||
|
||||
if index >= 0 {
|
||||
if caseName != "" {
|
||||
return wfCtx, fmt.Sprintf("matched case[%d]: %s", index, caseName), nil
|
||||
}
|
||||
return wfCtx, fmt.Sprintf("matched case[%d]", index), nil
|
||||
}
|
||||
|
||||
// 走默认分支(最后一个输出)
|
||||
return wfCtx, "no case matched, using default branch", nil
|
||||
}
|
||||
|
||||
// ProcessWithBranch 实现 BranchProcessor 接口
|
||||
func (c *SwitchConfig) ProcessWithBranch(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.NodeOutput, error) {
|
||||
index, caseName, err := c.evaluateCases(wfCtx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("switch processor: failed to evaluate cases: %v", err)
|
||||
}
|
||||
|
||||
output := &models.NodeOutput{
|
||||
WfCtx: wfCtx,
|
||||
}
|
||||
|
||||
if index >= 0 {
|
||||
output.BranchIndex = &index
|
||||
if caseName != "" {
|
||||
output.Message = fmt.Sprintf("matched case[%d]: %s", index, caseName)
|
||||
} else {
|
||||
output.Message = fmt.Sprintf("matched case[%d]", index)
|
||||
}
|
||||
} else {
|
||||
// 默认分支的索引是 cases 数量(即最后一个输出端口)
|
||||
defaultIndex := len(c.Cases)
|
||||
output.BranchIndex = &defaultIndex
|
||||
output.Message = "no case matched, using default branch"
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// evaluateCases 评估所有分支条件
|
||||
// 返回匹配的分支索引和分支名称,如果没有匹配返回 -1
|
||||
func (c *SwitchConfig) evaluateCases(wfCtx *models.WorkflowContext) (int, string, error) {
|
||||
for i := range c.Cases {
|
||||
matched, err := c.evaluateCaseCondition(&c.Cases[i], wfCtx)
|
||||
if err != nil {
|
||||
return -1, "", fmt.Errorf("case[%d] evaluation error: %v", i, err)
|
||||
}
|
||||
if matched {
|
||||
return i, c.Cases[i].Name, nil
|
||||
}
|
||||
}
|
||||
return -1, "", nil
|
||||
}
|
||||
|
||||
// evaluateCaseCondition 评估单个分支条件
|
||||
func (c *SwitchConfig) evaluateCaseCondition(caseItem *SwitchCase, wfCtx *models.WorkflowContext) (bool, error) {
|
||||
mode := caseItem.Mode
|
||||
if mode == "" {
|
||||
mode = ConditionModeExpression // 默认表达式模式
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case ConditionModeTags:
|
||||
return c.evaluateTagsCondition(caseItem, wfCtx.Event)
|
||||
default:
|
||||
return c.evaluateExpressionCondition(caseItem.Condition, wfCtx)
|
||||
}
|
||||
}
|
||||
|
||||
// evaluateExpressionCondition 评估表达式条件
|
||||
func (c *SwitchConfig) evaluateExpressionCondition(condition string, wfCtx *models.WorkflowContext) (bool, error) {
|
||||
if condition == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var defs = []string{
|
||||
"{{ $event := .Event }}",
|
||||
"{{ $labels := .Event.TagsMap }}",
|
||||
"{{ $value := .Event.TriggerValue }}",
|
||||
"{{ $inputs := .Inputs }}",
|
||||
}
|
||||
|
||||
text := strings.Join(append(defs, condition), "")
|
||||
|
||||
tpl, err := template.New("switch_condition").Funcs(tplx.TemplateFuncMap).Parse(text)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err = tpl.Execute(&buf, wfCtx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
result := strings.TrimSpace(strings.ToLower(buf.String()))
|
||||
return result == "true" || result == "1", nil
|
||||
}
|
||||
|
||||
// evaluateTagsCondition 评估标签/属性条件
|
||||
func (c *SwitchConfig) evaluateTagsCondition(caseItem *SwitchCase, event *models.AlertCurEvent) (bool, error) {
|
||||
// 如果没有配置任何过滤条件,默认返回 false(不匹配)
|
||||
if len(caseItem.parsedLabelKeys) == 0 && len(caseItem.parsedAttributes) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// 匹配标签 (TagsMap)
|
||||
if len(caseItem.parsedLabelKeys) > 0 {
|
||||
tagsMap := event.TagsMap
|
||||
if tagsMap == nil {
|
||||
tagsMap = make(map[string]string)
|
||||
}
|
||||
if !alertCommon.MatchTags(tagsMap, caseItem.parsedLabelKeys) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 匹配属性 (JsonTagsAndValue - 所有 JSON 字段)
|
||||
if len(caseItem.parsedAttributes) > 0 {
|
||||
attributesMap := event.JsonTagsAndValue()
|
||||
if !alertCommon.MatchTags(attributesMap, caseItem.parsedAttributes) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -42,7 +42,7 @@ func (r *RelabelConfig) Init(settings interface{}) (models.Processor, error) {
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (r *RelabelConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent) (*models.AlertCurEvent, string, error) {
|
||||
func (r *RelabelConfig) Process(ctx *ctx.Context, wfCtx *models.WorkflowContext) (*models.WorkflowContext, string, error) {
|
||||
sourceLabels := make([]model.LabelName, len(r.SourceLabels))
|
||||
for i := range r.SourceLabels {
|
||||
sourceLabels[i] = model.LabelName(strings.ReplaceAll(r.SourceLabels[i], ".", REPLACE_DOT))
|
||||
@@ -63,8 +63,8 @@ func (r *RelabelConfig) Process(ctx *ctx.Context, event *models.AlertCurEvent) (
|
||||
},
|
||||
}
|
||||
|
||||
EventRelabel(event, relabelConfigs)
|
||||
return event, "", nil
|
||||
EventRelabel(wfCtx.Event, relabelConfigs)
|
||||
return wfCtx, "", nil
|
||||
}
|
||||
|
||||
func EventRelabel(event *models.AlertCurEvent, relabelConfigs []*pconf.RelabelConfig) {
|
||||
|
||||
32
alert/pipeline/processor/utils/utils.go
Normal file
32
alert/pipeline/processor/utils/utils.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/tplx"
|
||||
)
|
||||
|
||||
func TplRender(wfCtx *models.WorkflowContext, content string) (string, error) {
|
||||
var defs = []string{
|
||||
"{{ $event := .Event }}",
|
||||
"{{ $labels := .Event.TagsMap }}",
|
||||
"{{ $value := .Event.TriggerValue }}",
|
||||
"{{ $inputs := .Inputs }}",
|
||||
}
|
||||
text := strings.Join(append(defs, content), "")
|
||||
tpl, err := template.New("tpl").Funcs(tplx.TemplateFuncMap).Parse(text)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse template: %v", err)
|
||||
}
|
||||
|
||||
var body bytes.Buffer
|
||||
if err = tpl.Execute(&body, wfCtx); err != nil {
|
||||
return "", fmt.Errorf("failed to execute template: %v", err)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(body.String()), nil
|
||||
}
|
||||
@@ -135,9 +135,7 @@ func (c *DefaultCallBacker) CallBack(ctx CallBackContext) {
|
||||
|
||||
func doSendAndRecord(ctx *ctx.Context, url, token string, body interface{}, channel string,
|
||||
stats *astats.Stats, events []*models.AlertCurEvent) {
|
||||
start := time.Now()
|
||||
res, err := doSend(url, body, channel, stats)
|
||||
res = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), res)
|
||||
NotifyRecord(ctx, events, 0, channel, token, res, err)
|
||||
}
|
||||
|
||||
@@ -171,11 +169,11 @@ func doSend(url string, body interface{}, channel string, stats *astats.Stats) (
|
||||
|
||||
start := time.Now()
|
||||
res, code, err := poster.PostJSON(url, time.Second*5, body, 3)
|
||||
res = []byte(fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), res))
|
||||
res = []byte(fmt.Sprintf("duration: %d ms status_code:%d, response:%s", time.Since(start).Milliseconds(), code, string(res)))
|
||||
if err != nil {
|
||||
logger.Errorf("%s_sender: result=fail url=%s code=%d error=%v req:%v response=%s", channel, url, code, err, body, string(res))
|
||||
stats.AlertNotifyErrorTotal.WithLabelValues(channel).Inc()
|
||||
return "", err
|
||||
return string(res), err
|
||||
}
|
||||
|
||||
logger.Infof("%s_sender: result=succ url=%s code=%d req:%v response=%s", channel, url, code, body, string(res))
|
||||
|
||||
@@ -86,13 +86,13 @@ func (c *IbexCallBacker) handleIbex(ctx *ctx.Context, url string, event *models.
|
||||
return
|
||||
}
|
||||
|
||||
CallIbex(ctx, id, host, c.taskTplCache, c.targetCache, c.userCache, event)
|
||||
CallIbex(ctx, id, host, c.taskTplCache, c.targetCache, c.userCache, event, "")
|
||||
}
|
||||
|
||||
func CallIbex(ctx *ctx.Context, id int64, host string,
|
||||
taskTplCache *memsto.TaskTplCache, targetCache *memsto.TargetCacheType,
|
||||
userCache *memsto.UserCacheType, event *models.AlertCurEvent) (int64, error) {
|
||||
logger.Infof("event_callback_ibex: id: %d, host: %s, event: %+v", id, host, event)
|
||||
userCache *memsto.UserCacheType, event *models.AlertCurEvent, args string) (int64, error) {
|
||||
logger.Infof("event_callback_ibex: id: %d, host: %s, args: %s, event: %+v", id, host, args, event)
|
||||
|
||||
tpl := taskTplCache.Get(id)
|
||||
if tpl == nil {
|
||||
@@ -102,7 +102,7 @@ func CallIbex(ctx *ctx.Context, id int64, host string,
|
||||
}
|
||||
// check perm
|
||||
// tpl.GroupId - host - account 三元组校验权限
|
||||
can, err := canDoIbex(tpl.UpdateBy, tpl, host, targetCache, userCache)
|
||||
can, err := CanDoIbex(tpl.UpdateBy, tpl, host, targetCache, userCache)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("event_callback_ibex: check perm fail: %v, event: %+v", err, event)
|
||||
logger.Errorf("%s", err)
|
||||
@@ -142,6 +142,10 @@ func CallIbex(ctx *ctx.Context, id int64, host string,
|
||||
}
|
||||
|
||||
// call ibex
|
||||
taskArgs := tpl.Args
|
||||
if args != "" {
|
||||
taskArgs = args
|
||||
}
|
||||
in := models.TaskForm{
|
||||
Title: tpl.Title + " FH: " + host,
|
||||
Account: tpl.Account,
|
||||
@@ -150,7 +154,7 @@ func CallIbex(ctx *ctx.Context, id int64, host string,
|
||||
Timeout: tpl.Timeout,
|
||||
Pause: tpl.Pause,
|
||||
Script: tpl.Script,
|
||||
Args: tpl.Args,
|
||||
Args: taskArgs,
|
||||
Stdin: string(tags),
|
||||
Action: "start",
|
||||
Creator: tpl.UpdateBy,
|
||||
@@ -190,7 +194,7 @@ func CallIbex(ctx *ctx.Context, id int64, host string,
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func canDoIbex(username string, tpl *models.TaskTpl, host string, targetCache *memsto.TargetCacheType, userCache *memsto.UserCacheType) (bool, error) {
|
||||
func CanDoIbex(username string, tpl *models.TaskTpl, host string, targetCache *memsto.TargetCacheType, userCache *memsto.UserCacheType) (bool, error) {
|
||||
user := userCache.GetByUsername(username)
|
||||
if user != nil && user.IsAdmin() {
|
||||
return true, nil
|
||||
|
||||
@@ -89,7 +89,7 @@ func alertingCallScript(ctx *ctx.Context, stdinBytes []byte, notifyScript models
|
||||
err, isTimeout := sys.WrapTimeout(cmd, time.Duration(config.Timeout)*time.Second)
|
||||
|
||||
res := buf.String()
|
||||
res = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), res)
|
||||
res = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), res)
|
||||
|
||||
// 截断超出长度的输出
|
||||
if len(res) > 512 {
|
||||
|
||||
@@ -119,11 +119,11 @@ func sendWebhook(webhook *models.Webhook, event interface{}, stats *astats.Stats
|
||||
|
||||
if resp.StatusCode == 429 {
|
||||
logger.Errorf("event_%s_fail, url: %s, response code: %d, body: %s event:%s", channel, conf.Url, resp.StatusCode, string(body), string(bs))
|
||||
return true, string(body), fmt.Errorf("status code is 429")
|
||||
return true, fmt.Sprintf("status_code:%d, response:%s", resp.StatusCode, string(body)), fmt.Errorf("status code is 429")
|
||||
}
|
||||
|
||||
logger.Debugf("event_%s_succ, url: %s, response code: %d, body: %s event:%s", channel, conf.Url, resp.StatusCode, string(body), string(bs))
|
||||
return false, string(body), nil
|
||||
return false, fmt.Sprintf("status_code:%d, response:%s", resp.StatusCode, string(body)), nil
|
||||
}
|
||||
|
||||
func SingleSendWebhooks(ctx *ctx.Context, webhooks map[string]*models.Webhook, event *models.AlertCurEvent, stats *astats.Stats) {
|
||||
@@ -132,7 +132,7 @@ func SingleSendWebhooks(ctx *ctx.Context, webhooks map[string]*models.Webhook, e
|
||||
for retryCount < 3 {
|
||||
start := time.Now()
|
||||
needRetry, res, err := sendWebhook(conf, event, stats)
|
||||
res = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), res)
|
||||
res = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), res)
|
||||
NotifyRecord(ctx, []*models.AlertCurEvent{event}, 0, "webhook", conf.Url, res, err)
|
||||
if !needRetry {
|
||||
break
|
||||
@@ -204,7 +204,7 @@ func StartConsumer(ctx *ctx.Context, queue *WebhookQueue, popSize int, webhook *
|
||||
for retryCount < webhook.RetryCount {
|
||||
start := time.Now()
|
||||
needRetry, res, err := sendWebhook(webhook, events, stats)
|
||||
res = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), res)
|
||||
res = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), res)
|
||||
go NotifyRecord(ctx, events, 0, "webhook", webhook.Url, res, err)
|
||||
if !needRetry {
|
||||
break
|
||||
|
||||
@@ -7,19 +7,20 @@ import (
|
||||
)
|
||||
|
||||
type Center struct {
|
||||
Plugins []Plugin
|
||||
MetricsYamlFile string
|
||||
OpsYamlFile string
|
||||
BuiltinIntegrationsDir string
|
||||
I18NHeaderKey string
|
||||
MetricDesc MetricDescType
|
||||
AnonymousAccess AnonymousAccess
|
||||
UseFileAssets bool
|
||||
FlashDuty FlashDuty
|
||||
EventHistoryGroupView bool
|
||||
CleanNotifyRecordDay int
|
||||
MigrateBusiGroupLabel bool
|
||||
RSA httpx.RSAConfig
|
||||
Plugins []Plugin
|
||||
MetricsYamlFile string
|
||||
OpsYamlFile string
|
||||
BuiltinIntegrationsDir string
|
||||
I18NHeaderKey string
|
||||
MetricDesc MetricDescType
|
||||
AnonymousAccess AnonymousAccess
|
||||
UseFileAssets bool
|
||||
FlashDuty FlashDuty
|
||||
EventHistoryGroupView bool
|
||||
CleanNotifyRecordDay int
|
||||
CleanPipelineExecutionDay int
|
||||
MigrateBusiGroupLabel bool
|
||||
RSA httpx.RSAConfig
|
||||
}
|
||||
|
||||
type Plugin struct {
|
||||
|
||||
@@ -134,6 +134,7 @@ func Initialize(configDir string, cryptoKey string) (func(), error) {
|
||||
go version.GetGithubVersion()
|
||||
|
||||
go cron.CleanNotifyRecord(ctx, config.Center.CleanNotifyRecordDay)
|
||||
go cron.CleanPipelineExecution(ctx, config.Center.CleanPipelineExecutionDay)
|
||||
|
||||
alertrtRouter := alertrt.New(config.HTTP, config.Alert, alertMuteCache, targetCache, busiGroupCache, alertStats, ctx, externalProcessors)
|
||||
centerRouter := centerrt.New(config.HTTP, config.Center, config.Alert, config.Ibex,
|
||||
|
||||
@@ -271,10 +271,8 @@ func Init(ctx *ctx.Context, builtinIntegrationsDir string) {
|
||||
}
|
||||
|
||||
for _, metric := range metrics {
|
||||
if metric.UUID == 0 {
|
||||
time.Sleep(time.Microsecond)
|
||||
metric.UUID = time.Now().UnixMicro()
|
||||
}
|
||||
time.Sleep(time.Microsecond)
|
||||
metric.UUID = time.Now().UnixMicro()
|
||||
metric.ID = metric.UUID
|
||||
metric.CreatedBy = SYSTEM
|
||||
metric.UpdatedBy = SYSTEM
|
||||
|
||||
@@ -118,7 +118,7 @@ func (s *Set) updateTargets(m map[string]models.HostMeta) error {
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
err := storage.MSet(context.Background(), s.redis, newMap)
|
||||
err := storage.MSet(context.Background(), s.redis, newMap, 7*24*time.Hour)
|
||||
if err != nil {
|
||||
cstats.RedisOperationLatency.WithLabelValues("mset_target_meta", "fail").Observe(time.Since(start).Seconds())
|
||||
return err
|
||||
@@ -127,7 +127,7 @@ func (s *Set) updateTargets(m map[string]models.HostMeta) error {
|
||||
}
|
||||
|
||||
if len(extendMap) > 0 {
|
||||
err = storage.MSet(context.Background(), s.redis, extendMap)
|
||||
err = storage.MSet(context.Background(), s.redis, extendMap, 7*24*time.Hour)
|
||||
if err != nil {
|
||||
cstats.RedisOperationLatency.WithLabelValues("mset_target_extend", "fail").Observe(time.Since(start).Seconds())
|
||||
return err
|
||||
|
||||
@@ -211,8 +211,8 @@ func (rt *Router) Config(r *gin.Engine) {
|
||||
pages.GET("/datasource/brief", rt.auth(), rt.user(), rt.datasourceBriefs)
|
||||
pages.POST("/datasource/query", rt.auth(), rt.user(), rt.datasourceQuery)
|
||||
|
||||
pages.POST("/ds-query", rt.auth(), rt.QueryData)
|
||||
pages.POST("/logs-query", rt.auth(), rt.QueryLogV2)
|
||||
pages.POST("/ds-query", rt.auth(), rt.user(), rt.QueryData)
|
||||
pages.POST("/logs-query", rt.auth(), rt.user(), rt.QueryLogV2)
|
||||
|
||||
pages.POST("/tdengine-databases", rt.auth(), rt.tdengineDatabases)
|
||||
pages.POST("/tdengine-tables", rt.auth(), rt.tdengineTables)
|
||||
@@ -251,10 +251,12 @@ func (rt *Router) Config(r *gin.Engine) {
|
||||
pages.GET("/auth/redirect/cas", rt.loginRedirectCas)
|
||||
pages.GET("/auth/redirect/oauth", rt.loginRedirectOAuth)
|
||||
pages.GET("/auth/redirect/dingtalk", rt.loginRedirectDingTalk)
|
||||
pages.GET("/auth/redirect/feishu", rt.loginRedirectFeiShu)
|
||||
pages.GET("/auth/callback", rt.loginCallback)
|
||||
pages.GET("/auth/callback/cas", rt.loginCallbackCas)
|
||||
pages.GET("/auth/callback/oauth", rt.loginCallbackOAuth)
|
||||
pages.GET("/auth/callback/dingtalk", rt.loginCallbackDingTalk)
|
||||
pages.GET("/auth/callback/feishu", rt.loginCallbackFeiShu)
|
||||
pages.GET("/auth/perms", rt.allPerms)
|
||||
|
||||
pages.GET("/metrics/desc", rt.metricsDescGetFile)
|
||||
@@ -389,8 +391,8 @@ func (rt *Router) Config(r *gin.Engine) {
|
||||
pages.GET("/busi-group/:id/recording-rules", rt.auth(), rt.user(), rt.perm("/recording-rules"), rt.recordingRuleGets)
|
||||
pages.POST("/busi-group/:id/recording-rules", rt.auth(), rt.user(), rt.perm("/recording-rules/add"), rt.bgrw(), rt.recordingRuleAddByFE)
|
||||
pages.DELETE("/busi-group/:id/recording-rules", rt.auth(), rt.user(), rt.perm("/recording-rules/del"), rt.bgrw(), rt.recordingRuleDel)
|
||||
pages.PUT("/busi-group/:id/recording-rule/:rrid", rt.auth(), rt.user(), rt.perm("/recording-rules/put"), rt.bgrw(), rt.recordingRulePutByFE)
|
||||
pages.GET("/recording-rule/:rrid", rt.auth(), rt.user(), rt.perm("/recording-rules"), rt.recordingRuleGet)
|
||||
pages.PUT("/recording-rule/:rrid", rt.auth(), rt.user(), rt.perm("/recording-rules"), rt.recordingRulePutByFE)
|
||||
pages.PUT("/busi-group/:id/recording-rules/fields", rt.auth(), rt.user(), rt.perm("/recording-rules/put"), rt.recordingRulePutFields)
|
||||
|
||||
pages.GET("/busi-groups/alert-mutes", rt.auth(), rt.user(), rt.perm("/alert-mutes"), rt.alertMuteGetsByGids)
|
||||
@@ -558,6 +560,19 @@ func (rt *Router) Config(r *gin.Engine) {
|
||||
pages.POST("/event-pipeline-tryrun", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.tryRunEventPipeline)
|
||||
pages.POST("/event-processor-tryrun", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.tryRunEventProcessor)
|
||||
|
||||
// API 触发工作流
|
||||
pages.POST("/event-pipeline/:id/trigger", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.triggerEventPipelineByAPI)
|
||||
// SSE 流式执行工作流
|
||||
pages.POST("/event-pipeline/:id/stream", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.streamEventPipeline)
|
||||
|
||||
// 事件Pipeline执行记录路由
|
||||
pages.GET("/event-pipeline-executions", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.listAllEventPipelineExecutions)
|
||||
pages.GET("/event-pipeline/:id/executions", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.listEventPipelineExecutions)
|
||||
pages.GET("/event-pipeline/:id/execution/:exec_id", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.getEventPipelineExecution)
|
||||
pages.GET("/event-pipeline-execution/:exec_id", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.getEventPipelineExecution)
|
||||
pages.GET("/event-pipeline/:id/execution-stats", rt.auth(), rt.user(), rt.perm("/event-pipelines"), rt.getEventPipelineExecutionStats)
|
||||
pages.POST("/event-pipeline-executions/clean", rt.auth(), rt.user(), rt.admin(), rt.cleanEventPipelineExecutions)
|
||||
|
||||
pages.POST("/notify-channel-configs", rt.auth(), rt.user(), rt.perm("/notification-channels/add"), rt.notifyChannelsAdd)
|
||||
pages.DELETE("/notify-channel-configs", rt.auth(), rt.user(), rt.perm("/notification-channels/del"), rt.notifyChannelsDel)
|
||||
pages.PUT("/notify-channel-config/:id", rt.auth(), rt.user(), rt.perm("/notification-channels/put"), rt.notifyChannelPut)
|
||||
@@ -569,6 +584,14 @@ func (rt *Router) Config(r *gin.Engine) {
|
||||
pages.GET("/pagerduty-service-list/:id", rt.auth(), rt.user(), rt.pagerDutyNotifyServicesGet)
|
||||
pages.GET("/notify-channel-config", rt.auth(), rt.user(), rt.notifyChannelGetBy)
|
||||
pages.GET("/notify-channel-config/idents", rt.notifyChannelIdentsGet)
|
||||
|
||||
// saved view 查询条件保存相关路由
|
||||
pages.GET("/saved-views", rt.auth(), rt.user(), rt.savedViewGets)
|
||||
pages.POST("/saved-views", rt.auth(), rt.user(), rt.savedViewAdd)
|
||||
pages.PUT("/saved-view/:id", rt.auth(), rt.user(), rt.savedViewPut)
|
||||
pages.DELETE("/saved-view/:id", rt.auth(), rt.user(), rt.savedViewDel)
|
||||
pages.POST("/saved-view/:id/favorite", rt.auth(), rt.user(), rt.savedViewFavoriteAdd)
|
||||
pages.DELETE("/saved-view/:id/favorite", rt.auth(), rt.user(), rt.savedViewFavoriteDel)
|
||||
}
|
||||
|
||||
r.GET("/api/n9e/versions", func(c *gin.Context) {
|
||||
@@ -682,6 +705,9 @@ func (rt *Router) Config(r *gin.Engine) {
|
||||
service.GET("/message-templates", rt.messageTemplateGets)
|
||||
|
||||
service.GET("/event-pipelines", rt.eventPipelinesListByService)
|
||||
service.POST("/event-pipeline/:id/trigger", rt.triggerEventPipelineByService)
|
||||
service.POST("/event-pipeline/:id/stream", rt.streamEventPipelineByService)
|
||||
service.POST("/event-pipeline-execution", rt.eventPipelineExecutionAdd)
|
||||
|
||||
// 手机号加密存储配置接口
|
||||
service.POST("/users/phone/encrypt", rt.usersPhoneEncrypt)
|
||||
|
||||
@@ -276,7 +276,7 @@ func (rt *Router) datasourceUpsert(c *gin.Context) {
|
||||
}
|
||||
err = req.Add(rt.Ctx)
|
||||
} else {
|
||||
err = req.Update(rt.Ctx, "name", "identifier", "description", "cluster_name", "settings", "http", "auth", "updated_by", "updated_at", "is_default")
|
||||
err = req.Update(rt.Ctx, "name", "identifier", "description", "cluster_name", "settings", "http", "auth", "updated_by", "updated_at", "is_default", "weight")
|
||||
}
|
||||
|
||||
Render(c, nil, err)
|
||||
@@ -293,11 +293,15 @@ func DatasourceCheck(c *gin.Context, ds models.Datasource) error {
|
||||
}
|
||||
}
|
||||
|
||||
// 使用 TLS 配置(支持 mTLS)
|
||||
tlsConfig, err := ds.HTTPJson.TLS.TLSConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create TLS config: %v", err)
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: ds.HTTPJson.TLS.SkipTlsVerify,
|
||||
},
|
||||
TLSClientConfig: tlsConfig,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,19 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/alert/pipeline/engine"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/toolkits/pkg/ginx"
|
||||
"github.com/toolkits/pkg/i18n"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
)
|
||||
|
||||
// 获取事件Pipeline列表
|
||||
@@ -27,18 +32,37 @@ func (rt *Router) eventPipelinesList(c *gin.Context) {
|
||||
for _, tid := range pipeline.TeamIds {
|
||||
pipeline.TeamNames = append(pipeline.TeamNames, ugMap[tid])
|
||||
}
|
||||
// 兼容处理:自动填充工作流字段
|
||||
pipeline.FillWorkflowFields()
|
||||
}
|
||||
|
||||
gids, err := models.MyGroupIdsMap(rt.Ctx, me.Id)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
if me.IsAdmin() {
|
||||
for _, pipeline := range pipelines {
|
||||
if pipeline.TriggerMode == "" {
|
||||
pipeline.TriggerMode = models.TriggerModeEvent
|
||||
}
|
||||
|
||||
if pipeline.UseCase == "" {
|
||||
pipeline.UseCase = models.UseCaseEventPipeline
|
||||
}
|
||||
}
|
||||
ginx.NewRender(c).Data(pipelines, nil)
|
||||
return
|
||||
}
|
||||
|
||||
res := make([]*models.EventPipeline, 0)
|
||||
for _, pipeline := range pipelines {
|
||||
if pipeline.TriggerMode == "" {
|
||||
pipeline.TriggerMode = models.TriggerModeEvent
|
||||
}
|
||||
|
||||
if pipeline.UseCase == "" {
|
||||
pipeline.UseCase = models.UseCaseEventPipeline
|
||||
}
|
||||
|
||||
for _, tid := range pipeline.TeamIds {
|
||||
if _, ok := gids[tid]; ok {
|
||||
res = append(res, pipeline)
|
||||
@@ -61,6 +85,15 @@ func (rt *Router) getEventPipeline(c *gin.Context) {
|
||||
err = pipeline.FillTeamNames(rt.Ctx)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
// 兼容处理:自动填充工作流字段
|
||||
pipeline.FillWorkflowFields()
|
||||
if pipeline.TriggerMode == "" {
|
||||
pipeline.TriggerMode = models.TriggerModeEvent
|
||||
}
|
||||
if pipeline.UseCase == "" {
|
||||
pipeline.UseCase = models.UseCaseEventPipeline
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(pipeline, nil)
|
||||
}
|
||||
|
||||
@@ -131,7 +164,9 @@ func (rt *Router) tryRunEventPipeline(c *gin.Context) {
|
||||
var f struct {
|
||||
EventId int64 `json:"event_id"`
|
||||
PipelineConfig models.EventPipeline `json:"pipeline_config"`
|
||||
InputVariables map[string]string `json:"input_variables,omitempty"`
|
||||
}
|
||||
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
hisEvent, err := models.AlertHisEventGetById(rt.Ctx, f.EventId)
|
||||
@@ -141,30 +176,33 @@ func (rt *Router) tryRunEventPipeline(c *gin.Context) {
|
||||
event := hisEvent.ToCur()
|
||||
|
||||
lang := c.GetHeader("X-Language")
|
||||
var result string
|
||||
for _, p := range f.PipelineConfig.ProcessorConfigs {
|
||||
processor, err := models.GetProcessorByType(p.Typ, p.Config)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusBadRequest, "get processor: %+v err: %+v", p, err)
|
||||
}
|
||||
event, result, err = processor.Process(rt.Ctx, event)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusBadRequest, "processor: %+v err: %+v", p, err)
|
||||
}
|
||||
me := c.MustGet("user").(*models.User)
|
||||
|
||||
if event == nil {
|
||||
ginx.NewRender(c).Data(map[string]interface{}{
|
||||
"event": event,
|
||||
"result": i18n.Sprintf(lang, "event is dropped"),
|
||||
}, nil)
|
||||
return
|
||||
}
|
||||
// 统一使用工作流引擎执行(兼容线性模式和工作流模式)
|
||||
workflowEngine := engine.NewWorkflowEngine(rt.Ctx)
|
||||
|
||||
triggerCtx := &models.WorkflowTriggerContext{
|
||||
Mode: models.TriggerModeAPI,
|
||||
TriggerBy: me.Username,
|
||||
InputsOverrides: f.InputVariables,
|
||||
}
|
||||
|
||||
resultEvent, result, err := workflowEngine.Execute(&f.PipelineConfig, event, triggerCtx)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusBadRequest, "pipeline execute error: %v", err)
|
||||
}
|
||||
|
||||
m := map[string]interface{}{
|
||||
"event": event,
|
||||
"result": i18n.Sprintf(lang, result),
|
||||
"event": resultEvent,
|
||||
"result": i18n.Sprintf(lang, result.Message),
|
||||
"status": result.Status,
|
||||
"node_results": result.NodeResults,
|
||||
}
|
||||
|
||||
if resultEvent == nil {
|
||||
m["result"] = i18n.Sprintf(lang, "event is dropped")
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(m, nil)
|
||||
}
|
||||
|
||||
@@ -186,14 +224,18 @@ func (rt *Router) tryRunEventProcessor(c *gin.Context) {
|
||||
if err != nil {
|
||||
ginx.Bomb(200, "get processor err: %+v", err)
|
||||
}
|
||||
event, res, err := processor.Process(rt.Ctx, event)
|
||||
wfCtx := &models.WorkflowContext{
|
||||
Event: event,
|
||||
Vars: make(map[string]interface{}),
|
||||
}
|
||||
wfCtx, res, err := processor.Process(rt.Ctx, wfCtx)
|
||||
if err != nil {
|
||||
ginx.Bomb(200, "processor err: %+v", err)
|
||||
}
|
||||
|
||||
lang := c.GetHeader("X-Language")
|
||||
ginx.NewRender(c).Data(map[string]interface{}{
|
||||
"event": event,
|
||||
"event": wfCtx.Event,
|
||||
"result": i18n.Sprintf(lang, res),
|
||||
}, nil)
|
||||
}
|
||||
@@ -223,6 +265,10 @@ func (rt *Router) tryRunEventProcessorByNotifyRule(c *gin.Context) {
|
||||
ginx.Bomb(http.StatusBadRequest, "processors not found")
|
||||
}
|
||||
|
||||
wfCtx := &models.WorkflowContext{
|
||||
Event: event,
|
||||
Vars: make(map[string]interface{}),
|
||||
}
|
||||
for _, pl := range pipelines {
|
||||
for _, p := range pl.ProcessorConfigs {
|
||||
processor, err := models.GetProcessorByType(p.Typ, p.Config)
|
||||
@@ -230,14 +276,14 @@ func (rt *Router) tryRunEventProcessorByNotifyRule(c *gin.Context) {
|
||||
ginx.Bomb(http.StatusBadRequest, "get processor: %+v err: %+v", p, err)
|
||||
}
|
||||
|
||||
event, _, err := processor.Process(rt.Ctx, event)
|
||||
wfCtx, _, err = processor.Process(rt.Ctx, wfCtx)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusBadRequest, "processor: %+v err: %+v", p, err)
|
||||
}
|
||||
if event == nil {
|
||||
if wfCtx == nil || wfCtx.Event == nil {
|
||||
lang := c.GetHeader("X-Language")
|
||||
ginx.NewRender(c).Data(map[string]interface{}{
|
||||
"event": event,
|
||||
"event": nil,
|
||||
"result": i18n.Sprintf(lang, "event is dropped"),
|
||||
}, nil)
|
||||
return
|
||||
@@ -245,10 +291,348 @@ func (rt *Router) tryRunEventProcessorByNotifyRule(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(event, nil)
|
||||
ginx.NewRender(c).Data(wfCtx.Event, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) eventPipelinesListByService(c *gin.Context) {
|
||||
pipelines, err := models.ListEventPipelines(rt.Ctx)
|
||||
ginx.NewRender(c).Data(pipelines, err)
|
||||
}
|
||||
|
||||
type EventPipelineRequest struct {
|
||||
// 事件数据(可选,如果不传则使用空事件)
|
||||
Event *models.AlertCurEvent `json:"event,omitempty"`
|
||||
// 输入参数覆盖
|
||||
InputsOverrides map[string]string `json:"inputs_overrides,omitempty"`
|
||||
|
||||
Username string `json:"username,omitempty"`
|
||||
}
|
||||
|
||||
// executePipelineTrigger 执行 Pipeline 触发的公共逻辑
|
||||
func (rt *Router) executePipelineTrigger(pipeline *models.EventPipeline, req *EventPipelineRequest, triggerBy string) (string, error) {
|
||||
// 准备事件数据
|
||||
var event *models.AlertCurEvent
|
||||
if req.Event != nil {
|
||||
event = req.Event
|
||||
} else {
|
||||
// 创建空事件
|
||||
event = &models.AlertCurEvent{
|
||||
TriggerTime: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
// 生成执行ID
|
||||
executionID := uuid.New().String()
|
||||
|
||||
// 创建触发上下文
|
||||
triggerCtx := &models.WorkflowTriggerContext{
|
||||
Mode: models.TriggerModeAPI,
|
||||
TriggerBy: triggerBy,
|
||||
InputsOverrides: req.InputsOverrides,
|
||||
RequestID: executionID,
|
||||
}
|
||||
|
||||
// 异步执行工作流
|
||||
go func() {
|
||||
workflowEngine := engine.NewWorkflowEngine(rt.Ctx)
|
||||
_, _, err := workflowEngine.Execute(pipeline, event, triggerCtx)
|
||||
if err != nil {
|
||||
logger.Errorf("async workflow execute error: pipeline_id=%d execution_id=%s err=%v",
|
||||
pipeline.ID, executionID, err)
|
||||
}
|
||||
}()
|
||||
|
||||
return executionID, nil
|
||||
}
|
||||
|
||||
// triggerEventPipelineByService Service 调用触发工作流执行
|
||||
func (rt *Router) triggerEventPipelineByService(c *gin.Context) {
|
||||
pipelineID := ginx.UrlParamInt64(c, "id")
|
||||
var f EventPipelineRequest
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
// 获取 Pipeline
|
||||
pipeline, err := models.GetEventPipeline(rt.Ctx, pipelineID)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusNotFound, "pipeline not found: %v", err)
|
||||
}
|
||||
|
||||
executionID, err := rt.executePipelineTrigger(pipeline, &f, f.Username)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"execution_id": executionID,
|
||||
"message": "workflow execution started",
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// triggerEventPipelineByAPI API 触发工作流执行
|
||||
func (rt *Router) triggerEventPipelineByAPI(c *gin.Context) {
|
||||
pipelineID := ginx.UrlParamInt64(c, "id")
|
||||
var f EventPipelineRequest
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
// 获取 Pipeline
|
||||
pipeline, err := models.GetEventPipeline(rt.Ctx, pipelineID)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusNotFound, "pipeline not found: %v", err)
|
||||
}
|
||||
|
||||
// 检查权限
|
||||
me := c.MustGet("user").(*models.User)
|
||||
ginx.Dangerous(me.CheckGroupPermission(rt.Ctx, pipeline.TeamIds))
|
||||
|
||||
executionID, err := rt.executePipelineTrigger(pipeline, &f, me.Username)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusBadRequest, err.Error())
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"execution_id": executionID,
|
||||
"message": "workflow execution started",
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) listAllEventPipelineExecutions(c *gin.Context) {
|
||||
pipelineId := ginx.QueryInt64(c, "pipeline_id", 0)
|
||||
pipelineName := ginx.QueryStr(c, "pipeline_name", "")
|
||||
mode := ginx.QueryStr(c, "mode", "")
|
||||
status := ginx.QueryStr(c, "status", "")
|
||||
limit := ginx.QueryInt(c, "limit", 20)
|
||||
offset := ginx.QueryInt(c, "p", 1)
|
||||
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 20
|
||||
}
|
||||
if offset <= 0 {
|
||||
offset = 1
|
||||
}
|
||||
|
||||
executions, total, err := models.ListAllEventPipelineExecutions(rt.Ctx, pipelineId, pipelineName, mode, status, limit, (offset-1)*limit)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"list": executions,
|
||||
"total": total,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) listEventPipelineExecutions(c *gin.Context) {
|
||||
pipelineID := ginx.UrlParamInt64(c, "id")
|
||||
mode := ginx.QueryStr(c, "mode", "")
|
||||
status := ginx.QueryStr(c, "status", "")
|
||||
limit := ginx.QueryInt(c, "limit", 20)
|
||||
offset := ginx.QueryInt(c, "p", 1)
|
||||
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 20
|
||||
}
|
||||
if offset <= 0 {
|
||||
offset = 1
|
||||
}
|
||||
|
||||
executions, total, err := models.ListEventPipelineExecutions(rt.Ctx, pipelineID, mode, status, limit, (offset-1)*limit)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"list": executions,
|
||||
"total": total,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) getEventPipelineExecution(c *gin.Context) {
|
||||
execID := ginx.UrlParamStr(c, "exec_id")
|
||||
|
||||
detail, err := models.GetEventPipelineExecutionDetail(rt.Ctx, execID)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusNotFound, "execution not found: %v", err)
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(detail, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) getEventPipelineExecutionStats(c *gin.Context) {
|
||||
pipelineID := ginx.UrlParamInt64(c, "id")
|
||||
|
||||
stats, err := models.GetEventPipelineExecutionStatistics(rt.Ctx, pipelineID)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
ginx.NewRender(c).Data(stats, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) cleanEventPipelineExecutions(c *gin.Context) {
|
||||
var f struct {
|
||||
BeforeDays int `json:"before_days"`
|
||||
}
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
if f.BeforeDays <= 0 {
|
||||
f.BeforeDays = 30
|
||||
}
|
||||
|
||||
beforeTime := time.Now().AddDate(0, 0, -f.BeforeDays).Unix()
|
||||
affected, err := models.DeleteEventPipelineExecutions(rt.Ctx, beforeTime)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"deleted": affected,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) streamEventPipeline(c *gin.Context) {
|
||||
pipelineID := ginx.UrlParamInt64(c, "id")
|
||||
|
||||
var f EventPipelineRequest
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
pipeline, err := models.GetEventPipeline(rt.Ctx, pipelineID)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusNotFound, "pipeline not found: %v", err)
|
||||
}
|
||||
|
||||
me := c.MustGet("user").(*models.User)
|
||||
ginx.Dangerous(me.CheckGroupPermission(rt.Ctx, pipeline.TeamIds))
|
||||
|
||||
var event *models.AlertCurEvent
|
||||
if f.Event != nil {
|
||||
event = f.Event
|
||||
} else {
|
||||
event = &models.AlertCurEvent{
|
||||
TriggerTime: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
triggerCtx := &models.WorkflowTriggerContext{
|
||||
Mode: models.TriggerModeAPI,
|
||||
TriggerBy: me.Username,
|
||||
InputsOverrides: f.InputsOverrides,
|
||||
RequestID: uuid.New().String(),
|
||||
Stream: true, // 流式端点强制启用流式输出
|
||||
}
|
||||
|
||||
workflowEngine := engine.NewWorkflowEngine(rt.Ctx)
|
||||
_, result, err := workflowEngine.Execute(pipeline, event, triggerCtx)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusInternalServerError, "execute failed: %v", err)
|
||||
}
|
||||
|
||||
if result.Stream && result.StreamChan != nil {
|
||||
rt.handleStreamResponse(c, result, triggerCtx.RequestID)
|
||||
return
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(result, nil)
|
||||
}
|
||||
|
||||
func (rt *Router) handleStreamResponse(c *gin.Context, result *models.WorkflowResult, requestID string) {
|
||||
// 设置 SSE 响应头
|
||||
c.Header("Content-Type", "text/event-stream")
|
||||
c.Header("Cache-Control", "no-cache")
|
||||
c.Header("Connection", "keep-alive")
|
||||
c.Header("X-Accel-Buffering", "no") // 禁用 nginx 缓冲
|
||||
c.Header("X-Request-ID", requestID)
|
||||
|
||||
flusher, ok := c.Writer.(http.Flusher)
|
||||
if !ok {
|
||||
ginx.Bomb(http.StatusInternalServerError, "streaming not supported")
|
||||
return
|
||||
}
|
||||
|
||||
// 发送初始连接成功消息
|
||||
initData := fmt.Sprintf(`{"type":"connected","request_id":"%s","timestamp":%d}`, requestID, time.Now().UnixMilli())
|
||||
fmt.Fprintf(c.Writer, "data: %s\n\n", initData)
|
||||
flusher.Flush()
|
||||
|
||||
// 从 channel 读取并发送 SSE
|
||||
timeout := time.After(30 * time.Minute) // 最长流式输出时间
|
||||
for {
|
||||
select {
|
||||
case chunk, ok := <-result.StreamChan:
|
||||
if !ok {
|
||||
// channel 关闭,发送结束标记
|
||||
return
|
||||
}
|
||||
|
||||
data, err := json.Marshal(chunk)
|
||||
if err != nil {
|
||||
logger.Errorf("stream: failed to marshal chunk: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Fprintf(c.Writer, "data: %s\n\n", data)
|
||||
flusher.Flush()
|
||||
|
||||
if chunk.Done {
|
||||
return
|
||||
}
|
||||
|
||||
case <-c.Request.Context().Done():
|
||||
// 客户端断开连接
|
||||
logger.Infof("stream: client disconnected, request_id=%s", requestID)
|
||||
return
|
||||
case <-timeout:
|
||||
logger.Errorf("stream: timeout, request_id=%s", requestID)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rt *Router) streamEventPipelineByService(c *gin.Context) {
|
||||
pipelineID := ginx.UrlParamInt64(c, "id")
|
||||
|
||||
var f EventPipelineRequest
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
pipeline, err := models.GetEventPipeline(rt.Ctx, pipelineID)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusNotFound, "pipeline not found: %v", err)
|
||||
}
|
||||
|
||||
var event *models.AlertCurEvent
|
||||
if f.Event != nil {
|
||||
event = f.Event
|
||||
} else {
|
||||
event = &models.AlertCurEvent{
|
||||
TriggerTime: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
triggerCtx := &models.WorkflowTriggerContext{
|
||||
Mode: models.TriggerModeAPI,
|
||||
TriggerBy: f.Username,
|
||||
InputsOverrides: f.InputsOverrides,
|
||||
RequestID: uuid.New().String(),
|
||||
Stream: true, // 流式端点强制启用流式输出
|
||||
}
|
||||
|
||||
workflowEngine := engine.NewWorkflowEngine(rt.Ctx)
|
||||
_, result, err := workflowEngine.Execute(pipeline, event, triggerCtx)
|
||||
if err != nil {
|
||||
ginx.Bomb(http.StatusInternalServerError, "execute failed: %v", err)
|
||||
}
|
||||
|
||||
// 检查是否是流式输出
|
||||
if result.Stream && result.StreamChan != nil {
|
||||
rt.handleStreamResponse(c, result, triggerCtx.RequestID)
|
||||
return
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(result, nil)
|
||||
}
|
||||
|
||||
// eventPipelineExecutionAdd 接收 edge 节点同步的 Pipeline 执行记录
|
||||
func (rt *Router) eventPipelineExecutionAdd(c *gin.Context) {
|
||||
var execution models.EventPipelineExecution
|
||||
ginx.BindJSON(c, &execution)
|
||||
|
||||
if execution.ID == "" {
|
||||
ginx.Bomb(http.StatusBadRequest, "id is required")
|
||||
}
|
||||
if execution.PipelineID <= 0 {
|
||||
ginx.Bomb(http.StatusBadRequest, "pipeline_id is required")
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Message(models.DB(rt.Ctx).Create(&execution).Error)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/cas"
|
||||
"github.com/ccfos/nightingale/v6/pkg/dingtalk"
|
||||
"github.com/ccfos/nightingale/v6/pkg/feishu"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ldapx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/oauth2x"
|
||||
"github.com/ccfos/nightingale/v6/pkg/oidcx"
|
||||
@@ -519,6 +520,92 @@ func (rt *Router) loginCallbackDingTalk(c *gin.Context) {
|
||||
|
||||
}
|
||||
|
||||
func (rt *Router) loginRedirectFeiShu(c *gin.Context) {
|
||||
redirect := ginx.QueryStr(c, "redirect", "/")
|
||||
|
||||
v, exists := c.Get("userid")
|
||||
if exists {
|
||||
userid := v.(int64)
|
||||
user, err := models.UserGetById(rt.Ctx, userid)
|
||||
ginx.Dangerous(err)
|
||||
if user == nil {
|
||||
ginx.Bomb(200, "user not found")
|
||||
}
|
||||
|
||||
if user.Username != "" { // already login
|
||||
ginx.NewRender(c).Data(redirect, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if rt.Sso.FeiShu == nil || !rt.Sso.FeiShu.Enable {
|
||||
ginx.NewRender(c).Data("", nil)
|
||||
return
|
||||
}
|
||||
|
||||
redirect, err := rt.Sso.FeiShu.Authorize(rt.Redis, redirect)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
ginx.NewRender(c).Data(redirect, err)
|
||||
}
|
||||
|
||||
func (rt *Router) loginCallbackFeiShu(c *gin.Context) {
|
||||
code := ginx.QueryStr(c, "code", "")
|
||||
state := ginx.QueryStr(c, "state", "")
|
||||
|
||||
ret, err := rt.Sso.FeiShu.Callback(rt.Redis, c.Request.Context(), code, state)
|
||||
if err != nil {
|
||||
logger.Errorf("sso_callback FeiShu fail. code:%s, state:%s, get ret: %+v. error: %v", code, state, ret, err)
|
||||
ginx.NewRender(c).Data(CallbackOutput{}, err)
|
||||
return
|
||||
}
|
||||
|
||||
user, err := models.UserGet(rt.Ctx, "username=?", ret.Username)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
if user != nil {
|
||||
if rt.Sso.FeiShu != nil && rt.Sso.FeiShu.FeiShuConfig != nil && rt.Sso.FeiShu.FeiShuConfig.CoverAttributes {
|
||||
updatedFields := user.UpdateSsoFields(feishu.SsoTypeName, ret.Nickname, ret.Phone, ret.Email)
|
||||
ginx.Dangerous(user.Update(rt.Ctx, "update_at", updatedFields...))
|
||||
}
|
||||
} else {
|
||||
user = new(models.User)
|
||||
defaultRoles := []string{}
|
||||
defaultUserGroups := []int64{}
|
||||
if rt.Sso.FeiShu != nil && rt.Sso.FeiShu.FeiShuConfig != nil {
|
||||
defaultRoles = rt.Sso.FeiShu.FeiShuConfig.DefaultRoles
|
||||
defaultUserGroups = rt.Sso.FeiShu.FeiShuConfig.DefaultUserGroups
|
||||
}
|
||||
|
||||
user.FullSsoFields(feishu.SsoTypeName, ret.Username, ret.Nickname, ret.Phone, ret.Email, defaultRoles)
|
||||
ginx.Dangerous(user.Add(rt.Ctx))
|
||||
|
||||
if len(defaultUserGroups) > 0 {
|
||||
ginx.Dangerous(user.UpdateUserGroup(rt.Ctx, defaultUserGroups))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// set user login state
|
||||
userIdentity := fmt.Sprintf("%d-%s", user.Id, user.Username)
|
||||
ts, err := rt.createTokens(rt.HTTP.JWTAuth.SigningKey, userIdentity)
|
||||
ginx.Dangerous(err)
|
||||
ginx.Dangerous(rt.createAuth(c.Request.Context(), userIdentity, ts))
|
||||
|
||||
redirect := "/"
|
||||
if ret.Redirect != "/login" {
|
||||
redirect = ret.Redirect
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(CallbackOutput{
|
||||
Redirect: redirect,
|
||||
User: user,
|
||||
AccessToken: ts.AccessToken,
|
||||
RefreshToken: ts.RefreshToken,
|
||||
}, nil)
|
||||
|
||||
}
|
||||
|
||||
func (rt *Router) loginCallbackOAuth(c *gin.Context) {
|
||||
code := ginx.QueryStr(c, "code", "")
|
||||
state := ginx.QueryStr(c, "state", "")
|
||||
@@ -569,10 +656,11 @@ type SsoConfigOutput struct {
|
||||
CasDisplayName string `json:"casDisplayName"`
|
||||
OauthDisplayName string `json:"oauthDisplayName"`
|
||||
DingTalkDisplayName string `json:"dingTalkDisplayName"`
|
||||
FeiShuDisplayName string `json:"feishuDisplayName"`
|
||||
}
|
||||
|
||||
func (rt *Router) ssoConfigNameGet(c *gin.Context) {
|
||||
var oidcDisplayName, casDisplayName, oauthDisplayName, dingTalkDisplayName string
|
||||
var oidcDisplayName, casDisplayName, oauthDisplayName, dingTalkDisplayName, feiShuDisplayName string
|
||||
if rt.Sso.OIDC != nil {
|
||||
oidcDisplayName = rt.Sso.OIDC.GetDisplayName()
|
||||
}
|
||||
@@ -589,11 +677,16 @@ func (rt *Router) ssoConfigNameGet(c *gin.Context) {
|
||||
dingTalkDisplayName = rt.Sso.DingTalk.GetDisplayName()
|
||||
}
|
||||
|
||||
if rt.Sso.FeiShu != nil {
|
||||
feiShuDisplayName = rt.Sso.FeiShu.GetDisplayName()
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(SsoConfigOutput{
|
||||
OidcDisplayName: oidcDisplayName,
|
||||
CasDisplayName: casDisplayName,
|
||||
OauthDisplayName: oauthDisplayName,
|
||||
DingTalkDisplayName: dingTalkDisplayName,
|
||||
FeiShuDisplayName: feiShuDisplayName,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
@@ -608,6 +701,7 @@ func (rt *Router) ssoConfigGets(c *gin.Context) {
|
||||
|
||||
// TODO: dingTalkExist 为了兼容当前前端配置, 后期单点登陆统一调整后不在预先设置默认内容
|
||||
dingTalkExist := false
|
||||
feiShuExist := false
|
||||
for _, config := range lst {
|
||||
var ssoReqConfig models.SsoConfig
|
||||
ssoReqConfig.Id = config.Id
|
||||
@@ -618,6 +712,10 @@ func (rt *Router) ssoConfigGets(c *gin.Context) {
|
||||
dingTalkExist = true
|
||||
err := json.Unmarshal([]byte(config.Content), &ssoReqConfig.SettingJson)
|
||||
ginx.Dangerous(err)
|
||||
case feishu.SsoTypeName:
|
||||
feiShuExist = true
|
||||
err := json.Unmarshal([]byte(config.Content), &ssoReqConfig.SettingJson)
|
||||
ginx.Dangerous(err)
|
||||
default:
|
||||
ssoReqConfig.Content = config.Content
|
||||
}
|
||||
@@ -630,6 +728,11 @@ func (rt *Router) ssoConfigGets(c *gin.Context) {
|
||||
ssoConfig.Name = dingtalk.SsoTypeName
|
||||
ssoConfigs = append(ssoConfigs, ssoConfig)
|
||||
}
|
||||
if !feiShuExist {
|
||||
var ssoConfig models.SsoConfig
|
||||
ssoConfig.Name = feishu.SsoTypeName
|
||||
ssoConfigs = append(ssoConfigs, ssoConfig)
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(ssoConfigs, nil)
|
||||
}
|
||||
@@ -657,6 +760,23 @@ func (rt *Router) ssoConfigUpdate(c *gin.Context) {
|
||||
err = f.Update(rt.Ctx)
|
||||
}
|
||||
ginx.Dangerous(err)
|
||||
case feishu.SsoTypeName:
|
||||
f.Name = ssoConfig.Name
|
||||
setting, err := json.Marshal(ssoConfig.SettingJson)
|
||||
ginx.Dangerous(err)
|
||||
f.Content = string(setting)
|
||||
f.UpdateAt = time.Now().Unix()
|
||||
sso, err := f.Query(rt.Ctx)
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
ginx.Dangerous(err)
|
||||
}
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
err = f.Create(rt.Ctx)
|
||||
} else {
|
||||
f.Id = sso.Id
|
||||
err = f.Update(rt.Ctx)
|
||||
}
|
||||
ginx.Dangerous(err)
|
||||
default:
|
||||
f.Id = ssoConfig.Id
|
||||
f.Name = ssoConfig.Name
|
||||
@@ -695,6 +815,14 @@ func (rt *Router) ssoConfigUpdate(c *gin.Context) {
|
||||
rt.Sso.DingTalk = dingtalk.New(config)
|
||||
}
|
||||
rt.Sso.DingTalk.Reload(config)
|
||||
case feishu.SsoTypeName:
|
||||
var config feishu.Config
|
||||
err := json.Unmarshal([]byte(f.Content), &config)
|
||||
ginx.Dangerous(err)
|
||||
if rt.Sso.FeiShu == nil {
|
||||
rt.Sso.FeiShu = feishu.New(config)
|
||||
}
|
||||
rt.Sso.FeiShu.Reload(config)
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Message(nil)
|
||||
|
||||
@@ -2,7 +2,6 @@ package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -169,8 +168,15 @@ func (rt *Router) dsProxy(c *gin.Context) {
|
||||
|
||||
transport, has := transportGet(dsId, ds.UpdatedAt)
|
||||
if !has {
|
||||
// 使用 TLS 配置(支持 mTLS)
|
||||
tlsConfig, err := ds.HTTPJson.TLS.TLSConfig()
|
||||
if err != nil {
|
||||
c.String(http.StatusInternalServerError, "failed to create TLS config: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
transport = &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: ds.HTTPJson.TLS.SkipTlsVerify},
|
||||
TLSClientConfig: tlsConfig,
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: time.Duration(ds.HTTPJson.DialTimeout) * time.Millisecond,
|
||||
|
||||
@@ -1,18 +1,24 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/alert/eval"
|
||||
"github.com/ccfos/nightingale/v6/dscache"
|
||||
"github.com/ccfos/nightingale/v6/dskit/doris"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/toolkits/pkg/ginx"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
)
|
||||
|
||||
func CheckDsPerm(c *gin.Context, dsId int64, cate string, q interface{}) bool {
|
||||
type CheckDsPermFunc func(c *gin.Context, dsId int64, cate string, q interface{}) bool
|
||||
|
||||
var CheckDsPerm CheckDsPermFunc = func(c *gin.Context, dsId int64, cate string, q interface{}) bool {
|
||||
// todo: 后续需要根据 cate 判断是否需要权限
|
||||
return true
|
||||
}
|
||||
@@ -56,6 +62,13 @@ func QueryLogBatchConcurrently(anonymousAccess bool, ctx *gin.Context, f QueryFr
|
||||
return LogResp{}, fmt.Errorf("cluster not exists")
|
||||
}
|
||||
|
||||
// 根据数据源类型对 Query 进行模板渲染处理
|
||||
err := eval.ExecuteQueryTemplate(q.DsCate, q.Query, nil)
|
||||
if err != nil {
|
||||
logger.Warningf("query template execute error: %v", err)
|
||||
return LogResp{}, fmt.Errorf("query template execute error: %v", err)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(query Query) {
|
||||
defer wg.Done()
|
||||
@@ -107,10 +120,13 @@ func (rt *Router) QueryLogBatch(c *gin.Context) {
|
||||
}
|
||||
|
||||
func QueryDataConcurrently(anonymousAccess bool, ctx *gin.Context, f models.QueryParam) ([]models.DataResp, error) {
|
||||
var resp []models.DataResp
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
var errs []error
|
||||
var (
|
||||
resp []models.DataResp
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
errs []error
|
||||
rCtx = ctx.Request.Context()
|
||||
)
|
||||
|
||||
for _, q := range f.Queries {
|
||||
if !anonymousAccess && !CheckDsPerm(ctx, f.DatasourceId, f.Cate, q) {
|
||||
@@ -122,12 +138,17 @@ func QueryDataConcurrently(anonymousAccess bool, ctx *gin.Context, f models.Quer
|
||||
logger.Warningf("cluster:%d not exists", f.DatasourceId)
|
||||
return nil, fmt.Errorf("cluster not exists")
|
||||
}
|
||||
|
||||
vCtx := rCtx
|
||||
if f.Cate == models.DORIS {
|
||||
vCtx = context.WithValue(vCtx, doris.NoNeedCheckMaxRow, true)
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(query interface{}) {
|
||||
defer wg.Done()
|
||||
|
||||
data, err := plug.QueryData(ctx.Request.Context(), query)
|
||||
data, err := plug.QueryData(vCtx, query)
|
||||
if err != nil {
|
||||
logger.Warningf("query data error: req:%+v err:%v", query, err)
|
||||
mu.Lock()
|
||||
|
||||
@@ -112,6 +112,7 @@ func (rt *Router) recordingRulePutByFE(c *gin.Context) {
|
||||
}
|
||||
|
||||
rt.bgrwCheck(c, ar.GroupId)
|
||||
rt.bgroCheck(c, f.GroupId)
|
||||
|
||||
f.UpdateBy = c.MustGet("username").(string)
|
||||
ginx.NewRender(c).Message(ar.Update(rt.Ctx, f))
|
||||
|
||||
144
center/router/router_saved_view.go
Normal file
144
center/router/router_saved_view.go
Normal file
@@ -0,0 +1,144 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/slice"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/toolkits/pkg/ginx"
|
||||
)
|
||||
|
||||
func (rt *Router) savedViewGets(c *gin.Context) {
|
||||
page := ginx.QueryStr(c, "page", "")
|
||||
|
||||
me := c.MustGet("user").(*models.User)
|
||||
|
||||
lst, err := models.SavedViewGets(rt.Ctx, page)
|
||||
if err != nil {
|
||||
ginx.NewRender(c).Data(nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
userGids, err := models.MyGroupIds(rt.Ctx, me.Id)
|
||||
if err != nil {
|
||||
ginx.NewRender(c).Data(nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
favoriteMap, err := models.SavedViewFavoriteGetByUserId(rt.Ctx, me.Id)
|
||||
if err != nil {
|
||||
ginx.NewRender(c).Data(nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
favoriteViews := make([]models.SavedView, 0)
|
||||
normalViews := make([]models.SavedView, 0)
|
||||
|
||||
for _, view := range lst {
|
||||
visible := view.CreateBy == me.Username ||
|
||||
view.PublicCate == 2 ||
|
||||
(view.PublicCate == 1 && slice.HaveIntersection[int64](userGids, view.Gids))
|
||||
|
||||
if !visible {
|
||||
continue
|
||||
}
|
||||
|
||||
view.IsFavorite = favoriteMap[view.Id]
|
||||
|
||||
// 收藏的排前面
|
||||
if view.IsFavorite {
|
||||
favoriteViews = append(favoriteViews, view)
|
||||
} else {
|
||||
normalViews = append(normalViews, view)
|
||||
}
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(append(favoriteViews, normalViews...), nil)
|
||||
}
|
||||
|
||||
func (rt *Router) savedViewAdd(c *gin.Context) {
|
||||
var f models.SavedView
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
me := c.MustGet("user").(*models.User)
|
||||
f.Id = 0
|
||||
f.CreateBy = me.Username
|
||||
f.UpdateBy = me.Username
|
||||
|
||||
err := models.SavedViewAdd(rt.Ctx, &f)
|
||||
ginx.NewRender(c).Data(f.Id, err)
|
||||
}
|
||||
|
||||
func (rt *Router) savedViewPut(c *gin.Context) {
|
||||
id := ginx.UrlParamInt64(c, "id")
|
||||
|
||||
view, err := models.SavedViewGetById(rt.Ctx, id)
|
||||
if err != nil {
|
||||
ginx.NewRender(c).Data(nil, err)
|
||||
return
|
||||
}
|
||||
if view == nil {
|
||||
ginx.NewRender(c, http.StatusNotFound).Message("saved view not found")
|
||||
return
|
||||
}
|
||||
|
||||
me := c.MustGet("user").(*models.User)
|
||||
// 只有创建者可以更新
|
||||
if view.CreateBy != me.Username && !me.IsAdmin() {
|
||||
ginx.NewRender(c, http.StatusForbidden).Message("forbidden")
|
||||
return
|
||||
}
|
||||
|
||||
var f models.SavedView
|
||||
ginx.BindJSON(c, &f)
|
||||
|
||||
view.Name = f.Name
|
||||
view.Filter = f.Filter
|
||||
view.PublicCate = f.PublicCate
|
||||
view.Gids = f.Gids
|
||||
|
||||
err = models.SavedViewUpdate(rt.Ctx, view, me.Username)
|
||||
ginx.NewRender(c).Message(err)
|
||||
}
|
||||
|
||||
func (rt *Router) savedViewDel(c *gin.Context) {
|
||||
id := ginx.UrlParamInt64(c, "id")
|
||||
|
||||
view, err := models.SavedViewGetById(rt.Ctx, id)
|
||||
if err != nil {
|
||||
ginx.NewRender(c).Data(nil, err)
|
||||
return
|
||||
}
|
||||
if view == nil {
|
||||
ginx.NewRender(c, http.StatusNotFound).Message("saved view not found")
|
||||
return
|
||||
}
|
||||
|
||||
me := c.MustGet("user").(*models.User)
|
||||
// 只有创建者或管理员可以删除
|
||||
if view.CreateBy != me.Username && !me.IsAdmin() {
|
||||
ginx.NewRender(c, http.StatusForbidden).Message("forbidden")
|
||||
return
|
||||
}
|
||||
|
||||
err = models.SavedViewDel(rt.Ctx, id)
|
||||
ginx.NewRender(c).Message(err)
|
||||
}
|
||||
|
||||
func (rt *Router) savedViewFavoriteAdd(c *gin.Context) {
|
||||
id := ginx.UrlParamInt64(c, "id")
|
||||
me := c.MustGet("user").(*models.User)
|
||||
|
||||
err := models.UserViewFavoriteAdd(rt.Ctx, id, me.Id)
|
||||
ginx.NewRender(c).Message(err)
|
||||
}
|
||||
|
||||
func (rt *Router) savedViewFavoriteDel(c *gin.Context) {
|
||||
id := ginx.UrlParamInt64(c, "id")
|
||||
me := c.MustGet("user").(*models.User)
|
||||
|
||||
err := models.UserViewFavoriteDel(rt.Ctx, id, me.Id)
|
||||
ginx.NewRender(c).Message(err)
|
||||
}
|
||||
@@ -38,6 +38,16 @@ func (rt *Router) targetGetsByHostFilter(c *gin.Context) {
|
||||
total, err := models.TargetCountByFilter(rt.Ctx, query)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
models.FillTargetsBeatTime(rt.Redis, hosts)
|
||||
now := time.Now().Unix()
|
||||
for i := 0; i < len(hosts); i++ {
|
||||
if now-hosts[i].BeatTime < 60 {
|
||||
hosts[i].TargetUp = 2
|
||||
} else if now-hosts[i].BeatTime < 180 {
|
||||
hosts[i].TargetUp = 1
|
||||
}
|
||||
}
|
||||
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"list": hosts,
|
||||
"total": total,
|
||||
@@ -81,9 +91,24 @@ func (rt *Router) targetGets(c *gin.Context) {
|
||||
models.BuildTargetWhereWithBgids(bgids),
|
||||
models.BuildTargetWhereWithDsIds(dsIds),
|
||||
models.BuildTargetWhereWithQuery(query),
|
||||
models.BuildTargetWhereWithDowntime(downtime),
|
||||
models.BuildTargetWhereWithHosts(hosts),
|
||||
}
|
||||
|
||||
// downtime 筛选:从缓存获取心跳时间,选择较小的集合用 IN 或 NOT IN 过滤
|
||||
if downtime != 0 {
|
||||
downtimeOpt, hasMatch := rt.downtimeFilter(downtime)
|
||||
if !hasMatch {
|
||||
ginx.NewRender(c).Data(gin.H{
|
||||
"list": []*models.Target{},
|
||||
"total": 0,
|
||||
}, nil)
|
||||
return
|
||||
}
|
||||
if downtimeOpt != nil {
|
||||
options = append(options, downtimeOpt)
|
||||
}
|
||||
}
|
||||
|
||||
total, err := models.TargetTotal(rt.Ctx, options...)
|
||||
ginx.Dangerous(err)
|
||||
|
||||
@@ -102,14 +127,17 @@ func (rt *Router) targetGets(c *gin.Context) {
|
||||
now := time.Now()
|
||||
cache := make(map[int64]*models.BusiGroup)
|
||||
|
||||
// 从 Redis 补全 BeatTime
|
||||
models.FillTargetsBeatTime(rt.Redis, list)
|
||||
|
||||
var keys []string
|
||||
for i := 0; i < len(list); i++ {
|
||||
ginx.Dangerous(list[i].FillGroup(rt.Ctx, cache))
|
||||
keys = append(keys, models.WrapIdent(list[i].Ident))
|
||||
|
||||
if now.Unix()-list[i].UpdateAt < 60 {
|
||||
if now.Unix()-list[i].BeatTime < 60 {
|
||||
list[i].TargetUp = 2
|
||||
} else if now.Unix()-list[i].UpdateAt < 180 {
|
||||
} else if now.Unix()-list[i].BeatTime < 180 {
|
||||
list[i].TargetUp = 1
|
||||
}
|
||||
}
|
||||
@@ -148,6 +176,43 @@ func (rt *Router) targetGets(c *gin.Context) {
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// downtimeFilter 从缓存获取心跳时间,生成 downtime 筛选条件
|
||||
// 选择匹配集和非匹配集中较小的一方,用 IN 或 NOT IN 来减少 SQL 参数量
|
||||
// 返回值:
|
||||
// - option: 筛选条件,nil 表示所有 target 都符合条件(无需过滤)
|
||||
// - hasMatch: 是否有符合条件的 target,false 表示无匹配应返回空结果
|
||||
func (rt *Router) downtimeFilter(downtime int64) (option models.BuildTargetWhereOption, hasMatch bool) {
|
||||
now := time.Now().Unix()
|
||||
targets := rt.TargetCache.GetAll()
|
||||
var matchIdents, nonMatchIdents []string
|
||||
for _, target := range targets {
|
||||
matched := false
|
||||
if downtime > 0 {
|
||||
matched = target.BeatTime < now-downtime
|
||||
} else if downtime < 0 {
|
||||
matched = target.BeatTime > now+downtime
|
||||
}
|
||||
if matched {
|
||||
matchIdents = append(matchIdents, target.Ident)
|
||||
} else {
|
||||
nonMatchIdents = append(nonMatchIdents, target.Ident)
|
||||
}
|
||||
}
|
||||
|
||||
if len(matchIdents) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if len(nonMatchIdents) == 0 {
|
||||
return nil, true
|
||||
}
|
||||
|
||||
if len(matchIdents) <= len(nonMatchIdents) {
|
||||
return models.BuildTargetWhereWithIdents(matchIdents), true
|
||||
}
|
||||
return models.BuildTargetWhereExcludeIdents(nonMatchIdents), true
|
||||
}
|
||||
|
||||
func (rt *Router) targetExtendInfoByIdent(c *gin.Context) {
|
||||
ident := ginx.QueryStr(c, "ident", "")
|
||||
key := models.WrapExtendIdent(ident)
|
||||
|
||||
@@ -2,13 +2,14 @@ package router
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/center/cconf"
|
||||
"github.com/ccfos/nightingale/v6/datasource/tdengine"
|
||||
"github.com/ccfos/nightingale/v6/dscache"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/toolkits/pkg/ginx"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type databasesQueryForm struct {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/ccfos/nightingale/v6/pkg/cas"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/dingtalk"
|
||||
"github.com/ccfos/nightingale/v6/pkg/feishu"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ldapx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/oauth2x"
|
||||
"github.com/ccfos/nightingale/v6/pkg/oidcx"
|
||||
@@ -27,6 +28,7 @@ type SsoClient struct {
|
||||
CAS *cas.SsoClient
|
||||
OAuth2 *oauth2x.SsoClient
|
||||
DingTalk *dingtalk.SsoClient
|
||||
FeiShu *feishu.SsoClient
|
||||
LastUpdateTime int64
|
||||
configCache *memsto.ConfigCache
|
||||
configLastUpdateTime int64
|
||||
@@ -203,6 +205,13 @@ func Init(center cconf.Center, ctx *ctx.Context, configCache *memsto.ConfigCache
|
||||
log.Fatalf("init %s failed: %s", dingtalk.SsoTypeName, err)
|
||||
}
|
||||
ssoClient.DingTalk = dingtalk.New(config)
|
||||
case feishu.SsoTypeName:
|
||||
var config feishu.Config
|
||||
err := json.Unmarshal([]byte(cfg.Content), &config)
|
||||
if err != nil {
|
||||
log.Fatalf("init %s failed: %s", feishu.SsoTypeName, err)
|
||||
}
|
||||
ssoClient.FeiShu = feishu.New(config)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -291,6 +300,22 @@ func (s *SsoClient) reload(ctx *ctx.Context) error {
|
||||
s.DingTalk = nil
|
||||
}
|
||||
|
||||
if feiShuConfig, ok := ssoConfigMap[feishu.SsoTypeName]; ok {
|
||||
var config feishu.Config
|
||||
err := json.Unmarshal([]byte(feiShuConfig.Content), &config)
|
||||
if err != nil {
|
||||
logger.Warningf("reload %s failed: %s", feishu.SsoTypeName, err)
|
||||
} else {
|
||||
if s.FeiShu != nil {
|
||||
s.FeiShu.Reload(config)
|
||||
} else {
|
||||
s.FeiShu = feishu.New(config)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.FeiShu = nil
|
||||
}
|
||||
|
||||
s.LastUpdateTime = lastUpdateTime
|
||||
s.configLastUpdateTime = lastCacheUpdateTime
|
||||
return nil
|
||||
|
||||
@@ -71,7 +71,10 @@ CREATE TABLE `datasource`
|
||||
`updated_at` bigint not null default 0,
|
||||
`updated_by` varchar(64) not null default '',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
|
||||
|
||||
-- datasource add weight field
|
||||
alter table `datasource` add `weight` int not null default 0;
|
||||
|
||||
CREATE TABLE `builtin_cate` (
|
||||
`id` bigint unsigned not null auto_increment,
|
||||
|
||||
67
cron/clean_pipeline_execution.go
Normal file
67
cron/clean_pipeline_execution.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package cron
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBatchSize = 100 // 每批删除数量
|
||||
defaultSleepMs = 10 // 每批删除后休眠时间(毫秒)
|
||||
)
|
||||
|
||||
// cleanPipelineExecutionInBatches 分批删除执行记录,避免大批量删除影响数据库性能
|
||||
func cleanPipelineExecutionInBatches(ctx *ctx.Context, day int) {
|
||||
threshold := time.Now().Unix() - 86400*int64(day)
|
||||
var totalDeleted int64
|
||||
|
||||
for {
|
||||
deleted, err := models.DeleteEventPipelineExecutionsInBatches(ctx, threshold, defaultBatchSize)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to clean pipeline execution records in batch: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
totalDeleted += deleted
|
||||
|
||||
// 如果本批删除数量小于 batchSize,说明已删除完毕
|
||||
if deleted < int64(defaultBatchSize) {
|
||||
break
|
||||
}
|
||||
|
||||
// 休眠一段时间,降低数据库压力
|
||||
time.Sleep(time.Duration(defaultSleepMs) * time.Millisecond)
|
||||
}
|
||||
|
||||
if totalDeleted > 0 {
|
||||
logger.Infof("Cleaned %d pipeline execution records older than %d days", totalDeleted, day)
|
||||
}
|
||||
}
|
||||
|
||||
// CleanPipelineExecution starts a cron job to clean old pipeline execution records in batches
|
||||
// Runs daily at 6:00 AM
|
||||
// day: 数据保留天数,默认 7 天
|
||||
// 使用分批删除方式,每批 100 条,间隔 10ms,避免大批量删除影响数据库性能
|
||||
func CleanPipelineExecution(ctx *ctx.Context, day int) {
|
||||
c := cron.New()
|
||||
if day < 1 {
|
||||
day = 7 // default retention: 7 days
|
||||
}
|
||||
|
||||
_, err := c.AddFunc("0 6 * * *", func() {
|
||||
cleanPipelineExecutionInBatches(ctx, day)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to add clean pipeline execution cron job: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
c.Start()
|
||||
logger.Infof("Pipeline execution cleanup cron started, retention: %d days, batch_size: %d, sleep_ms: %d", day, defaultBatchSize, defaultSleepMs)
|
||||
}
|
||||
@@ -26,6 +26,10 @@ const (
|
||||
FieldId FixedField = "_id"
|
||||
)
|
||||
|
||||
// LabelSeparator 用于分隔多个标签的分隔符
|
||||
// 使用 ASCII 控制字符 Record Separator (0x1E),避免与用户数据中的 "--" 冲突
|
||||
const LabelSeparator = "\x1e"
|
||||
|
||||
type Query struct {
|
||||
Ref string `json:"ref" mapstructure:"ref"`
|
||||
IndexType string `json:"index_type" mapstructure:"index_type"` // 普通索引:index 索引模式:index_pattern
|
||||
@@ -128,7 +132,7 @@ func TransferData(metric, ref string, m map[string][][]float64) []models.DataRes
|
||||
}
|
||||
|
||||
data.Metric["__name__"] = model.LabelValue(metric)
|
||||
labels := strings.Split(k, "--")
|
||||
labels := strings.Split(k, LabelSeparator)
|
||||
for _, label := range labels {
|
||||
arr := strings.SplitN(label, "=", 2)
|
||||
if len(arr) == 2 {
|
||||
@@ -197,7 +201,7 @@ func GetBuckets(labelKey string, keys []string, arr []interface{}, metrics *Metr
|
||||
case json.Number, string:
|
||||
if !getTs {
|
||||
if labels != "" {
|
||||
newlabels = fmt.Sprintf("%s--%s=%v", labels, labelKey, keyValue)
|
||||
newlabels = fmt.Sprintf("%s%s%s=%v", labels, LabelSeparator, labelKey, keyValue)
|
||||
} else {
|
||||
newlabels = fmt.Sprintf("%s=%v", labelKey, keyValue)
|
||||
}
|
||||
@@ -561,6 +565,14 @@ func QueryData(ctx context.Context, queryParam interface{}, cliTimeout int64, ve
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 检查是否有 shard failures,有部分数据时仅记录警告继续处理
|
||||
if shardErr := checkShardFailures(result.Shards, "query_data", searchSourceString); shardErr != nil {
|
||||
if len(result.Aggregations["ts"]) == 0 {
|
||||
return nil, shardErr
|
||||
}
|
||||
// 有部分数据,checkShardFailures 已记录警告,继续处理
|
||||
}
|
||||
|
||||
logger.Debugf("query_data searchSource:%s resp:%s", string(jsonSearchSource), string(result.Aggregations["ts"]))
|
||||
|
||||
js, err := simplejson.NewJson(result.Aggregations["ts"])
|
||||
@@ -598,6 +610,40 @@ func QueryData(ctx context.Context, queryParam interface{}, cliTimeout int64, ve
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// checkShardFailures 检查 ES 查询结果中的 shard failures,返回格式化的错误信息
|
||||
func checkShardFailures(shards *elastic.ShardsInfo, logPrefix string, queryContext interface{}) error {
|
||||
if shards == nil || shards.Failed == 0 || len(shards.Failures) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var failureReasons []string
|
||||
for _, failure := range shards.Failures {
|
||||
reason := ""
|
||||
if failure.Reason != nil {
|
||||
if reasonType, ok := failure.Reason["type"].(string); ok {
|
||||
reason = reasonType
|
||||
}
|
||||
if reasonMsg, ok := failure.Reason["reason"].(string); ok {
|
||||
if reason != "" {
|
||||
reason += ": " + reasonMsg
|
||||
} else {
|
||||
reason = reasonMsg
|
||||
}
|
||||
}
|
||||
}
|
||||
if reason != "" {
|
||||
failureReasons = append(failureReasons, fmt.Sprintf("index=%s shard=%d: %s", failure.Index, failure.Shard, reason))
|
||||
}
|
||||
}
|
||||
|
||||
if len(failureReasons) > 0 {
|
||||
errMsg := fmt.Sprintf("elasticsearch shard failures (%d/%d failed): %s", shards.Failed, shards.Total, strings.Join(failureReasons, "; "))
|
||||
logger.Warningf("%s query:%v %s", logPrefix, queryContext, errMsg)
|
||||
return fmt.Errorf("%s", errMsg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func HitFilter(typ string) bool {
|
||||
switch typ {
|
||||
case "keyword", "date", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long":
|
||||
@@ -674,21 +720,27 @@ func QueryLog(ctx context.Context, queryParam interface{}, timeout int64, versio
|
||||
} else {
|
||||
source = source.From(param.P).Sort(param.DateField, param.Ascending)
|
||||
}
|
||||
sourceBytes, _ := json.Marshal(source)
|
||||
result, err := search(ctx, indexArr, source, param.Timeout, param.MaxShard)
|
||||
if err != nil {
|
||||
logger.Warningf("query data error:%v", err)
|
||||
logger.Warningf("query_log source:%s error:%v", string(sourceBytes), err)
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
// 检查是否有 shard failures,有部分数据时仅记录警告继续处理
|
||||
if shardErr := checkShardFailures(result.Shards, "query_log", string(sourceBytes)); shardErr != nil {
|
||||
if len(result.Hits.Hits) == 0 {
|
||||
return nil, 0, shardErr
|
||||
}
|
||||
// 有部分数据,checkShardFailures 已记录警告,继续处理
|
||||
}
|
||||
|
||||
total := result.TotalHits()
|
||||
|
||||
var ret []interface{}
|
||||
|
||||
b, _ := json.Marshal(source)
|
||||
logger.Debugf("query data result query source:%s len:%d total:%d", string(b), len(result.Hits.Hits), total)
|
||||
logger.Debugf("query_log source:%s len:%d total:%d", string(sourceBytes), len(result.Hits.Hits), total)
|
||||
|
||||
resultBytes, _ := json.Marshal(result)
|
||||
logger.Debugf("query data result query source:%s result:%s", string(b), string(resultBytes))
|
||||
logger.Debugf("query_log source:%s result:%s", string(sourceBytes), string(resultBytes))
|
||||
|
||||
if strings.HasPrefix(version, "6") {
|
||||
for i := 0; i < len(result.Hits.Hits); i++ {
|
||||
|
||||
@@ -133,4 +133,5 @@ type DatasourceInfo struct {
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
IsDefault bool `json:"is_default"`
|
||||
Weight int `json:"weight"`
|
||||
}
|
||||
|
||||
@@ -4,12 +4,13 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/datasource"
|
||||
"github.com/ccfos/nightingale/v6/dskit/doris"
|
||||
"github.com/ccfos/nightingale/v6/dskit/types"
|
||||
"github.com/ccfos/nightingale/v6/pkg/macros"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
"github.com/ccfos/nightingale/v6/pkg/macros"
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
@@ -38,6 +39,8 @@ type QueryParam struct {
|
||||
To int64 `json:"to" mapstructure:"to"`
|
||||
TimeField string `json:"time_field" mapstructure:"time_field"`
|
||||
TimeFormat string `json:"time_format" mapstructure:"time_format"`
|
||||
Interval int64 `json:"interval" mapstructure:"interval"` // 查询时间间隔(秒)
|
||||
Offset int `json:"offset" mapstructure:"offset"` // 延迟计算,不在使用通用配置delay
|
||||
}
|
||||
|
||||
func (d *Doris) InitClient() error {
|
||||
@@ -76,52 +79,19 @@ func (d *Doris) Equal(p datasource.Datasource) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// only compare first shard
|
||||
if d.Addr != newest.Addr {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.User != newest.User {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.Password != newest.Password {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.EnableWrite != newest.EnableWrite {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.FeAddr != newest.FeAddr {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.MaxQueryRows != newest.MaxQueryRows {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.Timeout != newest.Timeout {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.MaxIdleConns != newest.MaxIdleConns {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.MaxOpenConns != newest.MaxOpenConns {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.ConnMaxLifetime != newest.ConnMaxLifetime {
|
||||
return false
|
||||
}
|
||||
|
||||
if d.ClusterName != newest.ClusterName {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
return d.Addr == newest.Addr &&
|
||||
d.FeAddr == newest.FeAddr &&
|
||||
d.User == newest.User &&
|
||||
d.Password == newest.Password &&
|
||||
d.EnableWrite == newest.EnableWrite &&
|
||||
d.UserWrite == newest.UserWrite &&
|
||||
d.PasswordWrite == newest.PasswordWrite &&
|
||||
d.MaxQueryRows == newest.MaxQueryRows &&
|
||||
d.Timeout == newest.Timeout &&
|
||||
d.MaxIdleConns == newest.MaxIdleConns &&
|
||||
d.MaxOpenConns == newest.MaxOpenConns &&
|
||||
d.ConnMaxLifetime == newest.ConnMaxLifetime &&
|
||||
d.ClusterName == newest.ClusterName
|
||||
}
|
||||
|
||||
func (d *Doris) MakeLogQuery(ctx context.Context, query interface{}, eventTags []string, start, end int64) (interface{}, error) {
|
||||
@@ -146,6 +116,30 @@ func (d *Doris) QueryData(ctx context.Context, query interface{}) ([]models.Data
|
||||
return nil, fmt.Errorf("valueKey is required")
|
||||
}
|
||||
|
||||
// 设置默认 interval
|
||||
if dorisQueryParam.Interval == 0 {
|
||||
dorisQueryParam.Interval = 60
|
||||
}
|
||||
|
||||
// 计算时间范围
|
||||
now := time.Now().Unix()
|
||||
var start, end int64
|
||||
if dorisQueryParam.To != 0 && dorisQueryParam.From != 0 {
|
||||
end = dorisQueryParam.To
|
||||
start = dorisQueryParam.From
|
||||
} else {
|
||||
end = now
|
||||
start = end - dorisQueryParam.Interval
|
||||
}
|
||||
|
||||
if dorisQueryParam.Offset != 0 {
|
||||
end -= int64(dorisQueryParam.Offset)
|
||||
start -= int64(dorisQueryParam.Offset)
|
||||
}
|
||||
|
||||
dorisQueryParam.From = start
|
||||
dorisQueryParam.To = end
|
||||
|
||||
if strings.Contains(dorisQueryParam.SQL, "$__") {
|
||||
var err error
|
||||
dorisQueryParam.SQL, err = macros.Macro(dorisQueryParam.SQL, dorisQueryParam.From, dorisQueryParam.To)
|
||||
@@ -154,13 +148,14 @@ func (d *Doris) QueryData(ctx context.Context, query interface{}) ([]models.Data
|
||||
}
|
||||
}
|
||||
|
||||
items, err := d.QueryTimeseries(context.TODO(), &doris.QueryParam{
|
||||
items, err := d.QueryTimeseries(ctx, &doris.QueryParam{
|
||||
Database: dorisQueryParam.Database,
|
||||
Sql: dorisQueryParam.SQL,
|
||||
Keys: types.Keys{
|
||||
ValueKey: dorisQueryParam.Keys.ValueKey,
|
||||
LabelKey: dorisQueryParam.Keys.LabelKey,
|
||||
TimeKey: dorisQueryParam.Keys.TimeKey,
|
||||
Offset: dorisQueryParam.Offset,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -188,6 +183,18 @@ func (d *Doris) QueryLog(ctx context.Context, query interface{}) ([]interface{},
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
// 记录规则预览场景下,只传了interval, 没有传From和To
|
||||
now := time.Now().Unix()
|
||||
if dorisQueryParam.To == 0 && dorisQueryParam.From == 0 && dorisQueryParam.Interval != 0 {
|
||||
dorisQueryParam.To = now
|
||||
dorisQueryParam.From = now - dorisQueryParam.Interval
|
||||
}
|
||||
|
||||
if dorisQueryParam.Offset != 0 {
|
||||
dorisQueryParam.To -= int64(dorisQueryParam.Offset)
|
||||
dorisQueryParam.From -= int64(dorisQueryParam.Offset)
|
||||
}
|
||||
|
||||
if strings.Contains(dorisQueryParam.SQL, "$__") {
|
||||
var err error
|
||||
dorisQueryParam.SQL, err = macros.Macro(dorisQueryParam.SQL, dorisQueryParam.From, dorisQueryParam.To)
|
||||
|
||||
@@ -33,6 +33,7 @@ type Query struct {
|
||||
Time int64 `json:"time" mapstructure:"time"` // 单点时间(秒)- 用于告警
|
||||
Step string `json:"step" mapstructure:"step"` // 步长,如 "1m", "5m"
|
||||
Limit int `json:"limit" mapstructure:"limit"` // 限制返回数量
|
||||
Ref string `json:"ref" mapstructure:"ref"` // 变量引用名(如 A、B)
|
||||
}
|
||||
|
||||
// IsInstantQuery 判断是否为即时查询(告警场景)
|
||||
@@ -162,7 +163,7 @@ func (vl *VictoriaLogs) queryDataInstant(ctx context.Context, param *Query) ([]m
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return convertPrometheusInstantToDataResp(result), nil
|
||||
return convertPrometheusInstantToDataResp(result, param.Ref), nil
|
||||
}
|
||||
|
||||
// queryDataRange 看图场景,调用 /select/logsql/stats_query_range
|
||||
@@ -185,15 +186,17 @@ func (vl *VictoriaLogs) queryDataRange(ctx context.Context, param *Query) ([]mod
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return convertPrometheusRangeToDataResp(result), nil
|
||||
return convertPrometheusRangeToDataResp(result, param.Ref), nil
|
||||
}
|
||||
|
||||
// convertPrometheusInstantToDataResp 将 Prometheus Instant Query 格式转换为 DataResp
|
||||
func convertPrometheusInstantToDataResp(resp *victorialogs.PrometheusResponse) []models.DataResp {
|
||||
func convertPrometheusInstantToDataResp(resp *victorialogs.PrometheusResponse, ref string) []models.DataResp {
|
||||
var dataResps []models.DataResp
|
||||
|
||||
for _, item := range resp.Data.Result {
|
||||
dataResp := models.DataResp{}
|
||||
dataResp := models.DataResp{
|
||||
Ref: ref,
|
||||
}
|
||||
|
||||
// 转换 Metric
|
||||
dataResp.Metric = make(model.Metric)
|
||||
@@ -218,11 +221,13 @@ func convertPrometheusInstantToDataResp(resp *victorialogs.PrometheusResponse) [
|
||||
}
|
||||
|
||||
// convertPrometheusRangeToDataResp 将 Prometheus Range Query 格式转换为 DataResp
|
||||
func convertPrometheusRangeToDataResp(resp *victorialogs.PrometheusResponse) []models.DataResp {
|
||||
func convertPrometheusRangeToDataResp(resp *victorialogs.PrometheusResponse, ref string) []models.DataResp {
|
||||
var dataResps []models.DataResp
|
||||
|
||||
for _, item := range resp.Data.Result {
|
||||
dataResp := models.DataResp{}
|
||||
dataResp := models.DataResp{
|
||||
Ref: ref,
|
||||
}
|
||||
|
||||
// 转换 Metric
|
||||
dataResp.Metric = make(model.Metric)
|
||||
|
||||
@@ -87,8 +87,8 @@ services:
|
||||
- mysql
|
||||
- redis
|
||||
- victoriametrics
|
||||
command: >
|
||||
sh -c "/app/n9e"
|
||||
command:
|
||||
- /app/n9e
|
||||
|
||||
categraf:
|
||||
image: "flashcatcloud/categraf:latest"
|
||||
|
||||
@@ -59,8 +59,8 @@ services:
|
||||
- mysql
|
||||
- redis
|
||||
- prometheus
|
||||
command: >
|
||||
sh -c "/app/n9e"
|
||||
command:
|
||||
- /app/n9e
|
||||
|
||||
categraf:
|
||||
image: "flashcatcloud/categraf:latest"
|
||||
|
||||
@@ -58,8 +58,8 @@ services:
|
||||
- mysql
|
||||
- redis
|
||||
- prometheus
|
||||
command: >
|
||||
sh -c "/app/n9e"
|
||||
command:
|
||||
- /app/n9e
|
||||
|
||||
categraf:
|
||||
image: "flashcatcloud/categraf:latest"
|
||||
|
||||
@@ -74,8 +74,8 @@ services:
|
||||
- postgres:postgres
|
||||
- redis:redis
|
||||
- victoriametrics:victoriametrics
|
||||
command: >
|
||||
sh -c "/app/n9e"
|
||||
command:
|
||||
- /app/n9e
|
||||
|
||||
categraf:
|
||||
image: "flashcatcloud/categraf:latest"
|
||||
|
||||
@@ -738,6 +738,7 @@ CREATE TABLE datasource
|
||||
http varchar(4096) not null default '',
|
||||
auth varchar(8192) not null default '',
|
||||
is_default boolean not null default false,
|
||||
weight int not null default 0,
|
||||
created_at bigint not null default 0,
|
||||
created_by varchar(64) not null default '',
|
||||
updated_at bigint not null default 0,
|
||||
@@ -804,6 +805,9 @@ CREATE TABLE builtin_metrics (
|
||||
lang varchar(191) NOT NULL DEFAULT '',
|
||||
note varchar(4096) NOT NULL,
|
||||
expression varchar(4096) NOT NULL,
|
||||
expression_type varchar(32) NOT NULL DEFAULT 'promql',
|
||||
metric_type varchar(191) NOT NULL DEFAULT '',
|
||||
extra_fields text,
|
||||
created_at bigint NOT NULL DEFAULT 0,
|
||||
created_by varchar(191) NOT NULL DEFAULT '',
|
||||
updated_at bigint NOT NULL DEFAULT 0,
|
||||
@@ -826,6 +830,9 @@ COMMENT ON COLUMN builtin_metrics.unit IS 'unit of metric';
|
||||
COMMENT ON COLUMN builtin_metrics.lang IS 'language of metric';
|
||||
COMMENT ON COLUMN builtin_metrics.note IS 'description of metric in Chinese';
|
||||
COMMENT ON COLUMN builtin_metrics.expression IS 'expression of metric';
|
||||
COMMENT ON COLUMN builtin_metrics.expression_type IS 'expression type: metric_name or promql';
|
||||
COMMENT ON COLUMN builtin_metrics.metric_type IS 'metric type like counter/gauge';
|
||||
COMMENT ON COLUMN builtin_metrics.extra_fields IS 'custom extra fields';
|
||||
COMMENT ON COLUMN builtin_metrics.created_at IS 'create time';
|
||||
COMMENT ON COLUMN builtin_metrics.created_by IS 'creator';
|
||||
COMMENT ON COLUMN builtin_metrics.updated_at IS 'update time';
|
||||
|
||||
@@ -655,6 +655,7 @@ CREATE TABLE `datasource`
|
||||
`http` varchar(4096) not null default '',
|
||||
`auth` varchar(8192) not null default '',
|
||||
`is_default` boolean COMMENT 'is default datasource',
|
||||
`weight` int not null default 0,
|
||||
`created_at` bigint not null default 0,
|
||||
`created_by` varchar(64) not null default '',
|
||||
`updated_at` bigint not null default 0,
|
||||
@@ -719,6 +720,9 @@ CREATE TABLE `builtin_metrics` (
|
||||
`lang` varchar(191) NOT NULL DEFAULT 'zh' COMMENT '''language''',
|
||||
`note` varchar(4096) NOT NULL COMMENT '''description of metric''',
|
||||
`expression` varchar(4096) NOT NULL COMMENT '''expression of metric''',
|
||||
`expression_type` varchar(32) NOT NULL DEFAULT 'promql' COMMENT '''expression type: metric_name or promql''',
|
||||
`metric_type` varchar(191) NOT NULL DEFAULT '' COMMENT '''metric type like counter/gauge''',
|
||||
`extra_fields` text COMMENT '''custom extra fields''',
|
||||
`created_at` bigint NOT NULL DEFAULT 0 COMMENT '''create time''',
|
||||
`created_by` varchar(191) NOT NULL DEFAULT '' COMMENT '''creator''',
|
||||
`updated_at` bigint NOT NULL DEFAULT 0 COMMENT '''update time''',
|
||||
|
||||
@@ -292,4 +292,72 @@ ALTER TABLE `alert_rule` ADD COLUMN `pipeline_configs` text COMMENT 'pipeline co
|
||||
|
||||
/* v8.4.2 2025-11-13 */
|
||||
ALTER TABLE `board` ADD COLUMN `note` varchar(1024) not null default '' comment 'note';
|
||||
ALTER TABLE `builtin_payloads` ADD COLUMN `note` varchar(1024) not null default '' comment 'note of payload';
|
||||
ALTER TABLE `builtin_payloads` ADD COLUMN `note` varchar(1024) not null default '' comment 'note of payload';
|
||||
|
||||
/* v9 2026-01-09 */
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `typ` varchar(128) NOT NULL DEFAULT '' COMMENT 'pipeline type: builtin, user-defined';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `use_case` varchar(128) NOT NULL DEFAULT '' COMMENT 'use case: metric_explorer, event_summary, event_pipeline';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `trigger_mode` varchar(128) NOT NULL DEFAULT 'event' COMMENT 'trigger mode: event, api, cron';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `disabled` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'disabled flag';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `nodes` text COMMENT 'workflow nodes (JSON)';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `connections` text COMMENT 'node connections (JSON)';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `input_variables` text COMMENT 'input variables (JSON)';
|
||||
ALTER TABLE `event_pipeline` ADD COLUMN `label_filters` text COMMENT 'label filters (JSON)';
|
||||
|
||||
CREATE TABLE `event_pipeline_execution` (
|
||||
`id` varchar(36) NOT NULL COMMENT 'execution id',
|
||||
`pipeline_id` bigint NOT NULL COMMENT 'pipeline id',
|
||||
`pipeline_name` varchar(128) DEFAULT '' COMMENT 'pipeline name snapshot',
|
||||
`event_id` bigint DEFAULT 0 COMMENT 'related alert event id',
|
||||
`mode` varchar(16) NOT NULL DEFAULT 'event' COMMENT 'trigger mode: event/api/cron',
|
||||
`status` varchar(16) NOT NULL DEFAULT 'running' COMMENT 'status: running/success/failed',
|
||||
`node_results` mediumtext COMMENT 'node execution results (JSON)',
|
||||
`error_message` varchar(1024) DEFAULT '' COMMENT 'error message',
|
||||
`error_node` varchar(36) DEFAULT '' COMMENT 'error node id',
|
||||
`created_at` bigint NOT NULL DEFAULT 0 COMMENT 'start timestamp',
|
||||
`finished_at` bigint DEFAULT 0 COMMENT 'finish timestamp',
|
||||
`duration_ms` bigint DEFAULT 0 COMMENT 'duration in milliseconds',
|
||||
`trigger_by` varchar(64) DEFAULT '' COMMENT 'trigger by',
|
||||
`inputs_snapshot` text COMMENT 'inputs snapshot',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_pipeline_id` (`pipeline_id`),
|
||||
KEY `idx_event_id` (`event_id`),
|
||||
KEY `idx_mode` (`mode`),
|
||||
KEY `idx_status` (`status`),
|
||||
KEY `idx_created_at` (`created_at`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='event pipeline execution records';
|
||||
|
||||
/* v8.5.0 builtin_metrics new fields */
|
||||
ALTER TABLE `builtin_metrics` ADD COLUMN `expression_type` varchar(32) NOT NULL DEFAULT 'promql' COMMENT 'expression type: metric_name or promql';
|
||||
ALTER TABLE `builtin_metrics` ADD COLUMN `metric_type` varchar(191) NOT NULL DEFAULT '' COMMENT 'metric type like counter/gauge';
|
||||
ALTER TABLE `builtin_metrics` ADD COLUMN `extra_fields` text COMMENT 'custom extra fields';
|
||||
|
||||
/* v9 2026-01-16 saved_view */
|
||||
CREATE TABLE `saved_view` (
|
||||
`id` bigint NOT NULL AUTO_INCREMENT,
|
||||
`name` varchar(255) NOT NULL COMMENT 'view name',
|
||||
`page` varchar(64) NOT NULL COMMENT 'page identifier',
|
||||
`filter` text COMMENT 'filter config (JSON)',
|
||||
`public_cate` int NOT NULL DEFAULT 0 COMMENT 'public category: 0-self, 1-team, 2-all',
|
||||
`gids` text COMMENT 'team group ids (JSON)',
|
||||
`create_at` bigint NOT NULL DEFAULT 0 COMMENT 'create timestamp',
|
||||
`create_by` varchar(64) NOT NULL DEFAULT '' COMMENT 'creator',
|
||||
`update_at` bigint NOT NULL DEFAULT 0 COMMENT 'update timestamp',
|
||||
`update_by` varchar(64) NOT NULL DEFAULT '' COMMENT 'updater',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_page` (`page`),
|
||||
KEY `idx_create_by` (`create_by`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='saved views for pages';
|
||||
|
||||
CREATE TABLE `user_view_favorite` (
|
||||
`id` bigint NOT NULL AUTO_INCREMENT,
|
||||
`view_id` bigint NOT NULL COMMENT 'saved view id',
|
||||
`user_id` bigint NOT NULL COMMENT 'user id',
|
||||
`create_at` bigint NOT NULL DEFAULT 0 COMMENT 'create timestamp',
|
||||
PRIMARY KEY (`id`),
|
||||
KEY `idx_view_id` (`view_id`),
|
||||
KEY `idx_user_id` (`user_id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='user favorite views';
|
||||
|
||||
/* v9 2026-01-20 datasource weight */
|
||||
ALTER TABLE `datasource` ADD COLUMN `weight` int not null default 0 COMMENT 'weight for sorting';
|
||||
|
||||
@@ -589,6 +589,7 @@ CREATE TABLE `datasource`
|
||||
`http` varchar(4096) not null default '',
|
||||
`auth` varchar(8192) not null default '',
|
||||
`is_default` tinyint not null default 0,
|
||||
`weight` int not null default 0,
|
||||
`created_at` bigint not null default 0,
|
||||
`created_by` varchar(64) not null default '',
|
||||
`updated_at` bigint not null default 0,
|
||||
@@ -651,6 +652,9 @@ CREATE TABLE `builtin_metrics` (
|
||||
`lang` varchar(191) NOT NULL DEFAULT '',
|
||||
`note` varchar(4096) NOT NULL,
|
||||
`expression` varchar(4096) NOT NULL,
|
||||
`expression_type` varchar(32) NOT NULL DEFAULT 'promql',
|
||||
`metric_type` varchar(191) NOT NULL DEFAULT '',
|
||||
`extra_fields` text,
|
||||
`created_at` bigint NOT NULL DEFAULT 0,
|
||||
`created_by` varchar(191) NOT NULL DEFAULT '',
|
||||
`updated_at` bigint NOT NULL DEFAULT 0,
|
||||
|
||||
@@ -57,3 +57,29 @@ func (cs *Cache) Get(cate string, dsId int64) (datasource.Datasource, bool) {
|
||||
|
||||
return cs.datas[cate][dsId], true
|
||||
}
|
||||
|
||||
func (cs *Cache) Delete(cate string, dsId int64) {
|
||||
cs.mutex.Lock()
|
||||
defer cs.mutex.Unlock()
|
||||
if _, found := cs.datas[cate]; !found {
|
||||
return
|
||||
}
|
||||
delete(cs.datas[cate], dsId)
|
||||
|
||||
logger.Debugf("delete plugin:%s %d from cache", cate, dsId)
|
||||
}
|
||||
|
||||
// GetAllIds 返回缓存中所有数据源的 ID,按类型分组
|
||||
func (cs *Cache) GetAllIds() map[string][]int64 {
|
||||
cs.mutex.RLock()
|
||||
defer cs.mutex.RUnlock()
|
||||
result := make(map[string][]int64)
|
||||
for cate, dsMap := range cs.datas {
|
||||
ids := make([]int64, 0, len(dsMap))
|
||||
for dsId := range dsMap {
|
||||
ids = append(ids, dsId)
|
||||
}
|
||||
result[cate] = ids
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ func getDatasourcesFromDBLoop(ctx *ctx.Context, fromAPI bool) {
|
||||
foundDefaultDatasource = true
|
||||
}
|
||||
|
||||
logger.Debugf("get datasource: %+v", item)
|
||||
// logger.Debugf("get datasource: %+v", item)
|
||||
ds := datasource.DatasourceInfo{
|
||||
Id: item.Id,
|
||||
Name: item.Name,
|
||||
@@ -104,6 +104,7 @@ func getDatasourcesFromDBLoop(ctx *ctx.Context, fromAPI bool) {
|
||||
AuthJson: item.AuthJson,
|
||||
Status: item.Status,
|
||||
IsDefault: item.IsDefault,
|
||||
Weight: item.Weight,
|
||||
}
|
||||
|
||||
if item.PluginType == "elasticsearch" {
|
||||
@@ -173,7 +174,10 @@ func esN9eToDatasourceInfo(ds *datasource.DatasourceInfo, item models.Datasource
|
||||
}
|
||||
|
||||
func PutDatasources(items []datasource.DatasourceInfo) {
|
||||
// 记录当前有效的数据源 ID,按类型分组
|
||||
validIds := make(map[string]map[int64]struct{})
|
||||
ids := make([]int64, 0)
|
||||
|
||||
for _, item := range items {
|
||||
if item.Type == "prometheus" {
|
||||
continue
|
||||
@@ -202,6 +206,12 @@ func PutDatasources(items []datasource.DatasourceInfo) {
|
||||
}
|
||||
ids = append(ids, item.Id)
|
||||
|
||||
// 记录有效的数据源 ID
|
||||
if _, ok := validIds[typ]; !ok {
|
||||
validIds[typ] = make(map[int64]struct{})
|
||||
}
|
||||
validIds[typ][item.Id] = struct{}{}
|
||||
|
||||
// 异步初始化 client 不然数据源同步的会很慢
|
||||
go func() {
|
||||
defer func() {
|
||||
@@ -213,5 +223,19 @@ func PutDatasources(items []datasource.DatasourceInfo) {
|
||||
}()
|
||||
}
|
||||
|
||||
logger.Debugf("get plugin by type success Ids:%v", ids)
|
||||
// 删除 items 中不存在但 DsCache 中存在的数据源
|
||||
cachedIds := DsCache.GetAllIds()
|
||||
for cate, dsIds := range cachedIds {
|
||||
for _, dsId := range dsIds {
|
||||
if _, ok := validIds[cate]; !ok {
|
||||
// 该类型在 items 中完全不存在,删除缓存中的所有该类型数据源
|
||||
DsCache.Delete(cate, dsId)
|
||||
} else if _, ok := validIds[cate][dsId]; !ok {
|
||||
// 该数据源 ID 在 items 中不存在,删除
|
||||
DsCache.Delete(cate, dsId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logger.Debugf("get plugin by type success Ids:%v", ids)
|
||||
}
|
||||
|
||||
@@ -18,19 +18,30 @@ import (
|
||||
"github.com/mitchellh/mapstructure"
|
||||
)
|
||||
|
||||
const (
|
||||
ShowIndexFieldIndexType = "index_type"
|
||||
ShowIndexFieldColumnName = "column_name"
|
||||
ShowIndexKeyName = "key_name"
|
||||
|
||||
SQLShowIndex = "SHOW INDEX FROM "
|
||||
)
|
||||
|
||||
// Doris struct to hold connection details and the connection object
|
||||
type Doris struct {
|
||||
Addr string `json:"doris.addr" mapstructure:"doris.addr"` // fe mysql endpoint
|
||||
FeAddr string `json:"doris.fe_addr" mapstructure:"doris.fe_addr"` // fe http endpoint
|
||||
User string `json:"doris.user" mapstructure:"doris.user"` //
|
||||
Password string `json:"doris.password" mapstructure:"doris.password"` //
|
||||
Timeout int `json:"doris.timeout" mapstructure:"doris.timeout"`
|
||||
Timeout int `json:"doris.timeout" mapstructure:"doris.timeout"` // ms
|
||||
MaxIdleConns int `json:"doris.max_idle_conns" mapstructure:"doris.max_idle_conns"`
|
||||
MaxOpenConns int `json:"doris.max_open_conns" mapstructure:"doris.max_open_conns"`
|
||||
ConnMaxLifetime int `json:"doris.conn_max_lifetime" mapstructure:"doris.conn_max_lifetime"`
|
||||
MaxQueryRows int `json:"doris.max_query_rows" mapstructure:"doris.max_query_rows"`
|
||||
ClusterName string `json:"doris.cluster_name" mapstructure:"doris.cluster_name"`
|
||||
EnableWrite bool `json:"doris.enable_write" mapstructure:"doris.enable_write"`
|
||||
// 写用户,用来区分读写用户,减少数据源
|
||||
UserWrite string `json:"doris.user_write" mapstructure:"doris.user_write"`
|
||||
PasswordWrite string `json:"doris.password_write" mapstructure:"doris.password_write"`
|
||||
}
|
||||
|
||||
// NewDorisWithSettings initializes a new Doris instance with the given settings
|
||||
@@ -63,7 +74,7 @@ func (d *Doris) NewConn(ctx context.Context, database string) (*sql.DB, error) {
|
||||
|
||||
// Set default values similar to postgres implementation
|
||||
if d.Timeout == 0 {
|
||||
d.Timeout = 60
|
||||
d.Timeout = 60000
|
||||
}
|
||||
if d.MaxIdleConns == 0 {
|
||||
d.MaxIdleConns = 10
|
||||
@@ -80,13 +91,13 @@ func (d *Doris) NewConn(ctx context.Context, database string) (*sql.DB, error) {
|
||||
|
||||
var keys []string
|
||||
keys = append(keys, d.Addr)
|
||||
keys = append(keys, d.Password, d.User)
|
||||
keys = append(keys, d.User, d.Password)
|
||||
if len(database) > 0 {
|
||||
keys = append(keys, database)
|
||||
}
|
||||
cachedkey := strings.Join(keys, ":")
|
||||
cachedKey := strings.Join(keys, ":")
|
||||
// cache conn with database
|
||||
conn, ok := pool.PoolClient.Load(cachedkey)
|
||||
conn, ok := pool.PoolClient.Load(cachedKey)
|
||||
if ok {
|
||||
return conn.(*sql.DB), nil
|
||||
}
|
||||
@@ -94,7 +105,7 @@ func (d *Doris) NewConn(ctx context.Context, database string) (*sql.DB, error) {
|
||||
var err error
|
||||
defer func() {
|
||||
if db != nil && err == nil {
|
||||
pool.PoolClient.Store(cachedkey, db)
|
||||
pool.PoolClient.Store(cachedKey, db)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -113,13 +124,86 @@ func (d *Doris) NewConn(ctx context.Context, database string) (*sql.DB, error) {
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// NewWriteConn establishes a new connection to Doris for write operations
|
||||
// When EnableWrite is true and UserWrite is configured, it uses the write user credentials
|
||||
// Otherwise, it reuses the read connection from NewConn
|
||||
func (d *Doris) NewWriteConn(ctx context.Context, database string) (*sql.DB, error) {
|
||||
// If write user is not configured, reuse the read connection
|
||||
if !d.EnableWrite || len(d.UserWrite) == 0 {
|
||||
return d.NewConn(ctx, database)
|
||||
}
|
||||
|
||||
if len(d.Addr) == 0 {
|
||||
return nil, errors.New("empty fe-node addr")
|
||||
}
|
||||
|
||||
// Set default values similar to postgres implementation
|
||||
if d.Timeout == 0 {
|
||||
d.Timeout = 60000
|
||||
}
|
||||
if d.MaxIdleConns == 0 {
|
||||
d.MaxIdleConns = 10
|
||||
}
|
||||
if d.MaxOpenConns == 0 {
|
||||
d.MaxOpenConns = 100
|
||||
}
|
||||
if d.ConnMaxLifetime == 0 {
|
||||
d.ConnMaxLifetime = 14400
|
||||
}
|
||||
if d.MaxQueryRows == 0 {
|
||||
d.MaxQueryRows = 500
|
||||
}
|
||||
|
||||
// Use write user credentials
|
||||
user := d.UserWrite
|
||||
password := d.PasswordWrite
|
||||
|
||||
var keys []string
|
||||
keys = append(keys, d.Addr)
|
||||
keys = append(keys, user, password)
|
||||
if len(database) > 0 {
|
||||
keys = append(keys, database)
|
||||
}
|
||||
cachedKey := strings.Join(keys, ":")
|
||||
// cache conn with database
|
||||
conn, ok := pool.PoolClient.Load(cachedKey)
|
||||
if ok {
|
||||
return conn.(*sql.DB), nil
|
||||
}
|
||||
var db *sql.DB
|
||||
var err error
|
||||
defer func() {
|
||||
if db != nil && err == nil {
|
||||
pool.PoolClient.Store(cachedKey, db)
|
||||
}
|
||||
}()
|
||||
|
||||
// Simplified connection logic for Doris using MySQL driver
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8", user, password, d.Addr, database)
|
||||
db, err = sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set connection pool configuration for write connections
|
||||
// Use more conservative values since write operations are typically less frequent
|
||||
writeMaxIdleConns := max(d.MaxIdleConns/5, 2)
|
||||
writeMaxOpenConns := max(d.MaxOpenConns/10, 5)
|
||||
|
||||
db.SetMaxIdleConns(writeMaxIdleConns)
|
||||
db.SetMaxOpenConns(writeMaxOpenConns)
|
||||
db.SetConnMaxLifetime(time.Duration(d.ConnMaxLifetime) * time.Second)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// createTimeoutContext creates a context with timeout based on Doris configuration
|
||||
func (d *Doris) createTimeoutContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
timeout := d.Timeout
|
||||
if timeout == 0 {
|
||||
timeout = 60
|
||||
}
|
||||
return context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
|
||||
return context.WithTimeout(ctx, time.Duration(timeout)*time.Millisecond)
|
||||
}
|
||||
|
||||
// ShowDatabases lists all databases in Doris
|
||||
@@ -312,6 +396,88 @@ func (d *Doris) DescTable(ctx context.Context, database, table string) ([]*types
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
type TableIndexInfo struct {
|
||||
ColumnName string `json:"column_name"`
|
||||
IndexName string `json:"index_name"`
|
||||
IndexType string `json:"index_type"`
|
||||
}
|
||||
|
||||
// ShowIndexes 查询表的所有索引信息
|
||||
func (d *Doris) ShowIndexes(ctx context.Context, database, table string) ([]TableIndexInfo, error) {
|
||||
if database == "" || table == "" {
|
||||
return nil, fmt.Errorf("database and table names cannot be empty")
|
||||
}
|
||||
|
||||
tCtx, cancel := d.createTimeoutContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
db, err := d.NewConn(tCtx, database)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
querySQL := fmt.Sprintf("%s `%s`.`%s`", SQLShowIndex, database, table)
|
||||
rows, err := db.QueryContext(tCtx, querySQL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query indexes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get columns: %w", err)
|
||||
}
|
||||
count := len(columns)
|
||||
|
||||
// 预映射列索引
|
||||
colIdx := map[string]int{
|
||||
ShowIndexKeyName: -1,
|
||||
ShowIndexFieldColumnName: -1,
|
||||
ShowIndexFieldIndexType: -1,
|
||||
}
|
||||
for i, col := range columns {
|
||||
lCol := strings.ToLower(col)
|
||||
if lCol == ShowIndexKeyName || lCol == ShowIndexFieldColumnName || lCol == ShowIndexFieldIndexType {
|
||||
colIdx[lCol] = i
|
||||
}
|
||||
}
|
||||
|
||||
var result []TableIndexInfo
|
||||
for rows.Next() {
|
||||
// 使用 sql.RawBytes 可以接受任何类型并转为 string,避免复杂的类型断言
|
||||
scanArgs := make([]interface{}, count)
|
||||
values := make([]sql.RawBytes, count)
|
||||
for i := range values {
|
||||
scanArgs[i] = &values[i]
|
||||
}
|
||||
|
||||
if err = rows.Scan(scanArgs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := TableIndexInfo{}
|
||||
if i := colIdx[ShowIndexFieldColumnName]; i != -1 && i < count {
|
||||
info.ColumnName = string(values[i])
|
||||
}
|
||||
if i := colIdx[ShowIndexKeyName]; i != -1 && i < count {
|
||||
info.IndexName = string(values[i])
|
||||
}
|
||||
if i := colIdx[ShowIndexFieldIndexType]; i != -1 && i < count {
|
||||
info.IndexType = string(values[i])
|
||||
}
|
||||
|
||||
if info.ColumnName != "" {
|
||||
result = append(result, info)
|
||||
}
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error iterating rows: %w", err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// SelectRows selects rows from a specified table in Doris based on a given query with MaxQueryRows check
|
||||
func (d *Doris) SelectRows(ctx context.Context, database, table, query string) ([]map[string]interface{}, error) {
|
||||
sql := fmt.Sprintf("SELECT * FROM %s.%s", database, table)
|
||||
@@ -382,7 +548,7 @@ func (d *Doris) ExecContext(ctx context.Context, database string, sql string) er
|
||||
timeoutCtx, cancel := d.createTimeoutContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
db, err := d.NewConn(timeoutCtx, database)
|
||||
db, err := d.NewWriteConn(timeoutCtx, database)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -10,13 +10,14 @@ const (
|
||||
TimeseriesAggregationTimestamp = "__ts__"
|
||||
)
|
||||
|
||||
// QueryLogs 查询日志
|
||||
// TODO: 待测试, MAP/ARRAY/STRUCT/JSON 等类型能否处理
|
||||
func (d *Doris) QueryLogs(ctx context.Context, query *QueryParam) ([]map[string]interface{}, error) {
|
||||
// 等同于 Query()
|
||||
return d.Query(ctx, query)
|
||||
return d.Query(ctx, query, true)
|
||||
}
|
||||
|
||||
// 本质是查询时序数据, 取第一组, SQL由上层封装, 不再做复杂的解析和截断
|
||||
// QueryHistogram 本质是查询时序数据, 取第一组, SQL由上层封装, 不再做复杂的解析和截断
|
||||
func (d *Doris) QueryHistogram(ctx context.Context, query *QueryParam) ([][]float64, error) {
|
||||
values, err := d.QueryTimeseries(ctx, query)
|
||||
if err != nil {
|
||||
|
||||
324
dskit/doris/sql_analyzer.go
Normal file
324
dskit/doris/sql_analyzer.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/pingcap/tidb/pkg/parser"
|
||||
"github.com/pingcap/tidb/pkg/parser/ast"
|
||||
_ "github.com/pingcap/tidb/pkg/parser/test_driver" // required for parser
|
||||
)
|
||||
|
||||
// mapAccessPattern matches Doris map/array access syntax like `col['key']` or col["key"]
|
||||
var mapAccessPattern = regexp.MustCompile(`\[['"]\w+['"]\]`)
|
||||
|
||||
// castStringPattern matches Doris CAST(... AS STRING) syntax
|
||||
var castStringPattern = regexp.MustCompile(`(?i)\bAS\s+STRING\b`)
|
||||
|
||||
// macro patterns
|
||||
var timeGroupPattern = regexp.MustCompile(`\$__timeGroup\([^)]+\)`)
|
||||
var timeFilterPattern = regexp.MustCompile(`\$__timeFilter\([^)]+\)`)
|
||||
var intervalPattern = regexp.MustCompile(`\$__interval`)
|
||||
|
||||
// SQLAnalyzeResult holds the analysis result of a SQL statement
|
||||
type SQLAnalyzeResult struct {
|
||||
IsSelectLike bool // whether the statement is a SELECT-like query
|
||||
HasTopAgg bool // whether the top-level query has aggregate functions
|
||||
LimitConst *int64 // top-level LIMIT constant value (nil if no LIMIT or non-constant)
|
||||
}
|
||||
|
||||
// AnalyzeSQL analyzes a SQL statement and extracts top-level features
|
||||
func AnalyzeSQL(sql string) (*SQLAnalyzeResult, error) {
|
||||
// Preprocess SQL to remove Doris-specific syntax that TiDB parser doesn't support
|
||||
preprocessedSQL := preprocessDorisSQL(sql)
|
||||
|
||||
p := parser.New()
|
||||
stmtNodes, _, err := p.Parse(preprocessedSQL, "", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(stmtNodes) == 0 {
|
||||
return &SQLAnalyzeResult{}, nil
|
||||
}
|
||||
|
||||
result := &SQLAnalyzeResult{}
|
||||
stmt := stmtNodes[0]
|
||||
|
||||
switch s := stmt.(type) {
|
||||
case *ast.SelectStmt:
|
||||
result.IsSelectLike = true
|
||||
analyzeSelectStmt(s, result)
|
||||
case *ast.SetOprStmt: // UNION / INTERSECT / EXCEPT
|
||||
result.IsSelectLike = true
|
||||
analyzeSetOprStmt(s, result)
|
||||
default:
|
||||
result.IsSelectLike = false
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// analyzeSelectStmt analyzes a SELECT statement
|
||||
func analyzeSelectStmt(sel *ast.SelectStmt, result *SQLAnalyzeResult) {
|
||||
// Check if top-level SELECT has aggregate functions
|
||||
if sel.Fields != nil {
|
||||
for _, field := range sel.Fields.Fields {
|
||||
if field.Expr != nil && hasAggregateFunc(field.Expr) {
|
||||
result.HasTopAgg = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if any CTE has aggregate functions
|
||||
if !result.HasTopAgg && sel.With != nil {
|
||||
for _, cte := range sel.With.CTEs {
|
||||
if selectHasAggregate(cte.Query) {
|
||||
result.HasTopAgg = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract top-level LIMIT
|
||||
if sel.Limit != nil && sel.Limit.Count != nil {
|
||||
if val, ok := extractConstValue(sel.Limit.Count); ok {
|
||||
result.LimitConst = &val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// selectHasAggregate checks if a node (SELECT, UNION, or SubqueryExpr) has aggregate functions
|
||||
func selectHasAggregate(node ast.Node) bool {
|
||||
switch n := node.(type) {
|
||||
case *ast.SelectStmt:
|
||||
if n.Fields != nil {
|
||||
for _, field := range n.Fields.Fields {
|
||||
if field.Expr != nil && hasAggregateFunc(field.Expr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
case *ast.SetOprStmt:
|
||||
// For UNION, check all branches
|
||||
if n.SelectList != nil {
|
||||
for _, sel := range n.SelectList.Selects {
|
||||
if selectHasAggregate(sel) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
case *ast.SubqueryExpr:
|
||||
// CTE query is wrapped in SubqueryExpr
|
||||
if n.Query != nil {
|
||||
return selectHasAggregate(n.Query)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// analyzeSetOprStmt analyzes UNION/INTERSECT/EXCEPT statements
|
||||
func analyzeSetOprStmt(setOpr *ast.SetOprStmt, result *SQLAnalyzeResult) {
|
||||
// UNION's LIMIT is at the outermost level
|
||||
if setOpr.Limit != nil && setOpr.Limit.Count != nil {
|
||||
if val, ok := extractConstValue(setOpr.Limit.Count); ok {
|
||||
result.LimitConst = &val
|
||||
}
|
||||
}
|
||||
|
||||
// Check if all branches are aggregates (conservative: if any is non-aggregate, don't skip)
|
||||
if setOpr.SelectList == nil || len(setOpr.SelectList.Selects) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
allAgg := true
|
||||
for _, sel := range setOpr.SelectList.Selects {
|
||||
if selectStmt, ok := sel.(*ast.SelectStmt); ok {
|
||||
if selectStmt.Fields != nil {
|
||||
hasAgg := false
|
||||
for _, field := range selectStmt.Fields.Fields {
|
||||
if field.Expr != nil && hasAggregateFunc(field.Expr) {
|
||||
hasAgg = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasAgg {
|
||||
allAgg = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.HasTopAgg = allAgg
|
||||
}
|
||||
|
||||
// hasAggregateFunc checks if an expression contains aggregate functions (without entering subqueries)
|
||||
func hasAggregateFunc(expr ast.ExprNode) bool {
|
||||
checker := &aggregateChecker{}
|
||||
expr.Accept(checker)
|
||||
return checker.found
|
||||
}
|
||||
|
||||
// aggregateChecker implements ast.Visitor to find aggregate functions
|
||||
type aggregateChecker struct {
|
||||
found bool
|
||||
}
|
||||
|
||||
func (c *aggregateChecker) Enter(n ast.Node) (ast.Node, bool) {
|
||||
if c.found {
|
||||
return n, true // stop traversal
|
||||
}
|
||||
|
||||
switch node := n.(type) {
|
||||
case *ast.SubqueryExpr:
|
||||
return n, true // don't enter subquery
|
||||
case *ast.AggregateFuncExpr:
|
||||
c.found = true
|
||||
return n, true
|
||||
case *ast.FuncCallExpr:
|
||||
// Check for Doris-specific aggregate/statistic functions
|
||||
funcName := strings.ToUpper(node.FnName.L)
|
||||
if isDorisAggregateFunc(funcName) {
|
||||
c.found = true
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
return n, false // continue traversal
|
||||
}
|
||||
|
||||
func (c *aggregateChecker) Leave(n ast.Node) (ast.Node, bool) {
|
||||
return n, true
|
||||
}
|
||||
|
||||
// isDorisAggregateFunc checks if a function is a Doris-specific aggregate/statistic function
|
||||
func isDorisAggregateFunc(funcName string) bool {
|
||||
dorisAggFuncs := map[string]bool{
|
||||
// Standard aggregates (in case parser doesn't recognize them)
|
||||
"COUNT": true,
|
||||
"SUM": true,
|
||||
"AVG": true,
|
||||
"MIN": true,
|
||||
"MAX": true,
|
||||
"ANY": true,
|
||||
"ANY_VALUE": true,
|
||||
|
||||
// HLL related
|
||||
"HLL_UNION_AGG": true,
|
||||
"HLL_RAW_AGG": true,
|
||||
"HLL_CARDINALITY": true,
|
||||
"HLL_UNION": true,
|
||||
"HLL_HASH": true,
|
||||
|
||||
// Bitmap related
|
||||
"BITMAP_UNION": true,
|
||||
"BITMAP_UNION_COUNT": true,
|
||||
"BITMAP_INTERSECT": true,
|
||||
"BITMAP_COUNT": true,
|
||||
"BITMAP_AND_COUNT": true,
|
||||
"BITMAP_OR_COUNT": true,
|
||||
"BITMAP_XOR_COUNT": true,
|
||||
"BITMAP_AND_NOT_COUNT": true,
|
||||
|
||||
// Other aggregates
|
||||
"PERCENTILE": true,
|
||||
"PERCENTILE_APPROX": true,
|
||||
"APPROX_COUNT_DISTINCT": true,
|
||||
"NDV": true,
|
||||
"COLLECT_LIST": true,
|
||||
"COLLECT_SET": true,
|
||||
"GROUP_CONCAT": true,
|
||||
"GROUP_BIT_AND": true,
|
||||
"GROUP_BIT_OR": true,
|
||||
"GROUP_BIT_XOR": true,
|
||||
"GROUPING": true,
|
||||
"GROUPING_ID": true,
|
||||
|
||||
// Statistical functions
|
||||
"STDDEV": true,
|
||||
"STDDEV_POP": true,
|
||||
"STDDEV_SAMP": true,
|
||||
"STD": true,
|
||||
"VARIANCE": true,
|
||||
"VAR_POP": true,
|
||||
"VAR_SAMP": true,
|
||||
"COVAR_POP": true,
|
||||
"COVAR_SAMP": true,
|
||||
"CORR": true,
|
||||
|
||||
// Window functions that are also aggregates
|
||||
"FIRST_VALUE": true,
|
||||
"LAST_VALUE": true,
|
||||
"LAG": true,
|
||||
"LEAD": true,
|
||||
"ROW_NUMBER": true,
|
||||
"RANK": true,
|
||||
"DENSE_RANK": true,
|
||||
"NTILE": true,
|
||||
"CUME_DIST": true,
|
||||
"PERCENT_RANK": true,
|
||||
}
|
||||
return dorisAggFuncs[funcName]
|
||||
}
|
||||
|
||||
// extractConstValue extracts constant integer value from an expression
|
||||
func extractConstValue(expr ast.ExprNode) (int64, bool) {
|
||||
switch v := expr.(type) {
|
||||
case ast.ValueExpr:
|
||||
switch val := v.GetValue().(type) {
|
||||
case int64:
|
||||
return val, true
|
||||
case uint64:
|
||||
return int64(val), true
|
||||
case float64:
|
||||
return int64(val), true
|
||||
case int:
|
||||
return int64(val), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// preprocessDorisSQL removes Doris-specific syntax that TiDB parser doesn't support
|
||||
func preprocessDorisSQL(sql string) string {
|
||||
// Remove map/array access syntax like ['key'] or ["key"]
|
||||
// This is used in Doris for accessing map/variant/json fields
|
||||
sql = mapAccessPattern.ReplaceAllString(sql, "")
|
||||
|
||||
// Replace Doris CAST(... AS STRING) with CAST(... AS CHAR)
|
||||
sql = castStringPattern.ReplaceAllString(sql, "AS CHAR")
|
||||
|
||||
// Replace macros with valid SQL equivalents
|
||||
sql = timeGroupPattern.ReplaceAllString(sql, "ts")
|
||||
sql = timeFilterPattern.ReplaceAllString(sql, "1=1")
|
||||
sql = intervalPattern.ReplaceAllString(sql, "60")
|
||||
|
||||
return sql
|
||||
}
|
||||
|
||||
// NeedsRowCountCheck determines if a SQL query needs row count checking
|
||||
// Returns: needsCheck bool, directReject bool, rejectReason string
|
||||
func NeedsRowCountCheck(sql string, maxQueryRows int) (bool, bool, string) {
|
||||
result, err := AnalyzeSQL(sql)
|
||||
if err != nil {
|
||||
// Parse failed, fall back to probe check
|
||||
return true, false, ""
|
||||
}
|
||||
|
||||
if !result.IsSelectLike {
|
||||
// Not a SELECT query, skip check
|
||||
return false, false, ""
|
||||
}
|
||||
|
||||
// Rule 1: Top-level has aggregate functions -> skip check
|
||||
if result.HasTopAgg {
|
||||
return false, false, ""
|
||||
}
|
||||
|
||||
// Rule 2: Top-level LIMIT <= maxRows -> skip check
|
||||
if result.LimitConst != nil && *result.LimitConst <= int64(maxQueryRows) {
|
||||
return false, false, ""
|
||||
}
|
||||
|
||||
// Otherwise, needs probe check (including LIMIT > maxRows, since actual result may be smaller)
|
||||
return true, false, ""
|
||||
}
|
||||
784
dskit/doris/sql_analyzer_test.go
Normal file
784
dskit/doris/sql_analyzer_test.go
Normal file
@@ -0,0 +1,784 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAnalyzeSQL_AggregateQueries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantHasAgg bool
|
||||
wantIsSelect bool
|
||||
}{
|
||||
// Standard aggregate functions - should skip check
|
||||
{
|
||||
name: "COUNT(*)",
|
||||
sql: "SELECT COUNT(*) AS `cnt`, FLOOR(UNIX_TIMESTAMP(event_date) DIV 10) * 10 AS `time`, CAST(`labels`['event'] AS STRING) AS `labels.event` FROM `db_insight_doris`.`ewall_event` WHERE `event_date` BETWEEN FROM_UNIXTIME(1768965669) AND FROM_UNIXTIME(1768965969) GROUP BY `time`, `labels.event` ORDER BY `time` ASC",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "COUNT with column",
|
||||
sql: "SELECT COUNT(id) FROM users",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "SUM function",
|
||||
sql: "SELECT SUM(amount) FROM orders",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "AVG function",
|
||||
sql: "SELECT AVG(price) FROM products",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "MIN function",
|
||||
sql: "SELECT MIN(created_at) FROM logs",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "MAX function",
|
||||
sql: "SELECT MAX(score) FROM results",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "Multiple aggregates",
|
||||
sql: "SELECT COUNT(*), SUM(amount), AVG(price) FROM orders",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "Aggregate with GROUP BY",
|
||||
sql: "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "Aggregate with WHERE and GROUP BY",
|
||||
sql: "SELECT category, SUM(sales) FROM products WHERE status = 'active' GROUP BY category",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "Aggregate with HAVING",
|
||||
sql: "SELECT user_id, COUNT(*) as cnt FROM orders GROUP BY user_id HAVING cnt > 10",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
// macro queries with aggregates
|
||||
{
|
||||
name: "COUNT with timeGroup",
|
||||
sql: "SELECT COUNT(*) AS `cnt`, $__timeGroup(timestamp,$__interval) AS `time` FROM `apm`.`traces_span` WHERE (`service_name` = 'demo-logic-server') AND $__timeFilter(`timestamp`) GROUP BY `time` ORDER BY `time` ASC",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "CTE with ratio calculation",
|
||||
sql: "WITH `time_totals` AS (SELECT $__timeGroup(timestamp,$__interval) AS `time`, COUNT(*) AS `total_count` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `time`), `time_counts` AS (SELECT ANY_VALUE(`service_name`) AS `service_name`, $__timeGroup(timestamp,$__interval) AS `time`, COUNT(*) AS `count` FROM `apm`.`traces_span` WHERE (`service_name` = 'demo-logic-server') AND $__timeFilter(`timestamp`) GROUP BY `time`) SELECT tc.`service_name`, tc.`time`, ROUND(tc.`count` * 100.0 / tt.`total_count`, 2) AS `ratio` FROM `time_counts` tc JOIN `time_totals` tt ON tc.`time` = tt.`time` ORDER BY tc.`time` ASC",
|
||||
wantHasAgg: true, // CTE has aggregate functions
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "CTE with top values and ratio",
|
||||
sql: "WITH `top_values` AS (SELECT `service_name` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `service_name` ORDER BY COUNT(*) DESC LIMIT 5), `time_totals` AS (SELECT $__timeGroup(timestamp,$__interval) AS `time`, COUNT(*) AS `total_count` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `time`), `time_counts` AS (SELECT `service_name`, $__timeGroup(timestamp,$__interval) AS `time`, COUNT(*) AS `count` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) AND `service_name` IN (SELECT `service_name` FROM `top_values`) GROUP BY `service_name`, `time`) SELECT tc.`service_name`, tc.`time`, ROUND(tc.`count` * 100.0 / tt.`total_count`, 2) AS `ratio` FROM `time_counts` tc JOIN `time_totals` tt ON tc.`time` = tt.`time` ORDER BY tc.`time` ASC",
|
||||
wantHasAgg: true, // CTE has aggregate functions
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "PERCENTILE_APPROX with timeGroup",
|
||||
sql: "SELECT PERCENTILE_APPROX(`duration`, 0.95) AS `p95`, $__timeGroup(timestamp,$__interval) AS `time` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `time` ORDER BY `time` ASC",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "COUNT DISTINCT with timeGroup",
|
||||
sql: "SELECT COUNT(DISTINCT `duration`) AS `unique_count`, $__timeGroup(timestamp,$__interval) AS `time` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `time` ORDER BY `time` ASC",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "CASE WHEN with COUNT and ROUND",
|
||||
sql: "SELECT ROUND(COUNT(CASE WHEN `duration` IS NOT NULL THEN 1 END) * 100.0 / COUNT(*), 2) AS `exist_ratio`, $__timeGroup(timestamp,$__interval) AS `time` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `time` ORDER BY `time` ASC",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "AVG with timeGroup",
|
||||
sql: "SELECT AVG(`duration`) AS `avg`, $__timeGroup(timestamp,$__interval) AS `time` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`) GROUP BY `time` ORDER BY `time` ASC",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "Simple COUNT with timeFilter",
|
||||
sql: "SELECT COUNT(*) AS `cnt` FROM `apm`.`traces_span` WHERE (`span_name` = 'GET /backend/detail') AND $__timeFilter(`timestamp`)",
|
||||
wantHasAgg: true,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "CTE with CROSS JOIN ratio",
|
||||
sql: "WITH `total` AS (SELECT COUNT(*) AS `total_count` FROM `apm`.`traces_span` WHERE $__timeFilter(`timestamp`)), `value_counts` AS (SELECT ANY_VALUE(`span_kind`) AS `span_kind`, COUNT(*) AS `count` FROM `apm`.`traces_span` WHERE (`span_kind` = 'SPAN_KIND_SERVER') AND $__timeFilter(`timestamp`)) SELECT vc.`span_kind`, vc.`count` AS `count`, ROUND(vc.`count` * 100.0 / t.`total_count`, 2) AS `ratio` FROM `value_counts` vc CROSS JOIN `total` t ORDER BY vc.`count` DESC;",
|
||||
wantHasAgg: true, // CTE has aggregate functions
|
||||
wantIsSelect: true,
|
||||
},
|
||||
// Non-aggregate queries - should not skip check
|
||||
{
|
||||
name: "Simple SELECT *",
|
||||
sql: "SELECT * FROM users",
|
||||
wantHasAgg: false,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "SELECT with columns",
|
||||
sql: "SELECT id, name, email FROM users",
|
||||
wantHasAgg: false,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "SELECT with WHERE",
|
||||
sql: "SELECT * FROM users WHERE status = 'active'",
|
||||
wantHasAgg: false,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "SELECT with JOIN",
|
||||
sql: "SELECT u.name, o.amount FROM users u JOIN orders o ON u.id = o.user_id",
|
||||
wantHasAgg: false,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := AnalyzeSQL(tt.sql)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeSQL() error = %v", err)
|
||||
}
|
||||
if result.HasTopAgg != tt.wantHasAgg {
|
||||
t.Errorf("name: %s, HasTopAgg = %v, want %v", tt.name, result.HasTopAgg, tt.wantHasAgg)
|
||||
}
|
||||
if result.IsSelectLike != tt.wantIsSelect {
|
||||
t.Errorf("IsSelectLike = %v, want %v", result.IsSelectLike, tt.wantIsSelect)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeSQL_SubqueryWithAggregate(t *testing.T) {
|
||||
// Aggregate in subquery should NOT skip check for main query
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantHasAgg bool
|
||||
}{
|
||||
{
|
||||
name: "Aggregate in subquery only",
|
||||
sql: "SELECT * FROM (SELECT user_id, COUNT(*) as cnt FROM orders GROUP BY user_id) t",
|
||||
wantHasAgg: false, // top-level has no aggregate
|
||||
},
|
||||
{
|
||||
name: "Aggregate in WHERE subquery",
|
||||
sql: "SELECT * FROM users WHERE id IN (SELECT user_id FROM orders GROUP BY user_id HAVING COUNT(*) > 5)",
|
||||
wantHasAgg: false, // top-level has no aggregate
|
||||
},
|
||||
{
|
||||
name: "Both top-level and subquery aggregates",
|
||||
sql: "SELECT COUNT(*) FROM (SELECT user_id FROM orders GROUP BY user_id) t",
|
||||
wantHasAgg: true, // top-level has aggregate
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := AnalyzeSQL(tt.sql)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeSQL() error = %v", err)
|
||||
}
|
||||
if result.HasTopAgg != tt.wantHasAgg {
|
||||
t.Errorf("HasTopAgg = %v, want %v", result.HasTopAgg, tt.wantHasAgg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeSQL_LimitQueries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantLimit *int64
|
||||
wantIsSelect bool
|
||||
}{
|
||||
{
|
||||
name: "LIMIT 10",
|
||||
sql: "SELECT * FROM users LIMIT 10",
|
||||
wantLimit: ptr(int64(10)),
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 100",
|
||||
sql: "SELECT * FROM users LIMIT 100",
|
||||
wantLimit: ptr(int64(100)),
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 1000",
|
||||
sql: "SELECT * FROM users LIMIT 1000",
|
||||
wantLimit: ptr(int64(1000)),
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "LIMIT with OFFSET",
|
||||
sql: "SELECT * FROM users LIMIT 50 OFFSET 100",
|
||||
wantLimit: ptr(int64(50)),
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "No LIMIT",
|
||||
sql: "SELECT * FROM users",
|
||||
wantLimit: nil,
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 0",
|
||||
sql: "SELECT * FROM users LIMIT 0",
|
||||
wantLimit: ptr(int64(0)),
|
||||
wantIsSelect: true,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 1",
|
||||
sql: "SELECT * FROM users LIMIT 1",
|
||||
wantLimit: ptr(int64(1)),
|
||||
wantIsSelect: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := AnalyzeSQL(tt.sql)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeSQL() error = %v", err)
|
||||
}
|
||||
if result.IsSelectLike != tt.wantIsSelect {
|
||||
t.Errorf("IsSelectLike = %v, want %v", result.IsSelectLike, tt.wantIsSelect)
|
||||
}
|
||||
if tt.wantLimit == nil {
|
||||
if result.LimitConst != nil {
|
||||
t.Errorf("LimitConst = %v, want nil", *result.LimitConst)
|
||||
}
|
||||
} else {
|
||||
if result.LimitConst == nil {
|
||||
t.Errorf("LimitConst = nil, want %v", *tt.wantLimit)
|
||||
} else if *result.LimitConst != *tt.wantLimit {
|
||||
t.Errorf("LimitConst = %v, want %v", *result.LimitConst, *tt.wantLimit)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeSQL_UnionQueries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantHasAgg bool
|
||||
wantLimit *int64
|
||||
}{
|
||||
{
|
||||
name: "UNION without aggregate",
|
||||
sql: "SELECT id, name FROM users UNION SELECT id, name FROM admins",
|
||||
wantHasAgg: false,
|
||||
wantLimit: nil,
|
||||
},
|
||||
{
|
||||
name: "UNION ALL without aggregate",
|
||||
sql: "SELECT * FROM users UNION ALL SELECT * FROM admins",
|
||||
wantHasAgg: false,
|
||||
wantLimit: nil,
|
||||
},
|
||||
{
|
||||
name: "UNION with aggregate in all branches",
|
||||
sql: "SELECT COUNT(*) FROM users UNION SELECT COUNT(*) FROM admins",
|
||||
wantHasAgg: true,
|
||||
wantLimit: nil,
|
||||
},
|
||||
{
|
||||
name: "UNION with aggregate in one branch only",
|
||||
sql: "SELECT COUNT(*) FROM users UNION SELECT id FROM admins",
|
||||
wantHasAgg: false, // not all branches have aggregate
|
||||
wantLimit: nil,
|
||||
},
|
||||
{
|
||||
name: "UNION with outer LIMIT",
|
||||
sql: "SELECT * FROM users UNION SELECT * FROM admins LIMIT 100",
|
||||
wantHasAgg: false,
|
||||
wantLimit: ptr(int64(100)),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := AnalyzeSQL(tt.sql)
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeSQL() error = %v", err)
|
||||
}
|
||||
if result.HasTopAgg != tt.wantHasAgg {
|
||||
t.Errorf("HasTopAgg = %v, want %v", result.HasTopAgg, tt.wantHasAgg)
|
||||
}
|
||||
if tt.wantLimit == nil {
|
||||
if result.LimitConst != nil {
|
||||
t.Errorf("LimitConst = %v, want nil", *result.LimitConst)
|
||||
}
|
||||
} else {
|
||||
if result.LimitConst == nil {
|
||||
t.Errorf("LimitConst = nil, want %v", *tt.wantLimit)
|
||||
} else if *result.LimitConst != *tt.wantLimit {
|
||||
t.Errorf("LimitConst = %v, want %v", *result.LimitConst, *tt.wantLimit)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeSQL_NonSelectStatements(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantIsSelect bool
|
||||
}{
|
||||
{
|
||||
name: "SHOW DATABASES",
|
||||
sql: "SHOW DATABASES",
|
||||
wantIsSelect: false,
|
||||
},
|
||||
{
|
||||
name: "SHOW TABLES",
|
||||
sql: "SHOW TABLES",
|
||||
wantIsSelect: false,
|
||||
},
|
||||
{
|
||||
name: "DESCRIBE table",
|
||||
sql: "DESCRIBE users",
|
||||
wantIsSelect: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := AnalyzeSQL(tt.sql)
|
||||
if err != nil {
|
||||
// Some statements may not be parseable, which is fine
|
||||
return
|
||||
}
|
||||
if result.IsSelectLike != tt.wantIsSelect {
|
||||
t.Errorf("IsSelectLike = %v, want %v", result.IsSelectLike, tt.wantIsSelect)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsRowCountCheck(t *testing.T) {
|
||||
maxRows := 500
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantNeedCheck bool
|
||||
wantReject bool
|
||||
}{
|
||||
// Should skip check (needsCheck = false)
|
||||
{
|
||||
name: "Aggregate COUNT(*)",
|
||||
sql: "SELECT COUNT(*) FROM users",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Aggregate SUM",
|
||||
sql: "SELECT SUM(amount) FROM orders",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Aggregate with GROUP BY",
|
||||
sql: "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT equal to max",
|
||||
sql: "SELECT * FROM users LIMIT 500",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT less than max",
|
||||
sql: "SELECT * FROM users LIMIT 100",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 1",
|
||||
sql: "SELECT * FROM users LIMIT 1",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
|
||||
// LIMIT > maxRows still needs probe check (actual result might be smaller)
|
||||
{
|
||||
name: "LIMIT exceeds max",
|
||||
sql: "SELECT * FROM users LIMIT 1000",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT much larger than max",
|
||||
sql: "SELECT * FROM users LIMIT 10000",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
|
||||
// Should execute probe check (needsCheck = true)
|
||||
{
|
||||
name: "No LIMIT no aggregate",
|
||||
sql: "SELECT * FROM users",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "SELECT with WHERE no LIMIT",
|
||||
sql: "SELECT * FROM users WHERE status = 'active'",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "SELECT with JOIN no LIMIT",
|
||||
sql: "SELECT u.*, o.* FROM users u JOIN orders o ON u.id = o.user_id",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Aggregate in subquery only",
|
||||
sql: "SELECT * FROM (SELECT user_id, COUNT(*) as cnt FROM orders GROUP BY user_id) t",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
needsCheck, directReject, _ := NeedsRowCountCheck(tt.sql, maxRows)
|
||||
if needsCheck != tt.wantNeedCheck {
|
||||
t.Errorf("needsCheck = %v, want %v", needsCheck, tt.wantNeedCheck)
|
||||
}
|
||||
if directReject != tt.wantReject {
|
||||
t.Errorf("directReject = %v, want %v", directReject, tt.wantReject)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsRowCountCheck_DorisSpecificFunctions(t *testing.T) {
|
||||
maxRows := 500
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantNeedCheck bool
|
||||
}{
|
||||
// Doris HLL functions
|
||||
{
|
||||
name: "HLL_UNION_AGG",
|
||||
sql: "SELECT HLL_UNION_AGG(hll_col) FROM user_stats",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
{
|
||||
name: "HLL_CARDINALITY",
|
||||
sql: "SELECT HLL_CARDINALITY(hll_col) FROM user_stats",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
// Doris Bitmap functions
|
||||
{
|
||||
name: "BITMAP_UNION_COUNT",
|
||||
sql: "SELECT BITMAP_UNION_COUNT(bitmap_col) FROM user_tags",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
{
|
||||
name: "BITMAP_UNION",
|
||||
sql: "SELECT BITMAP_UNION(bitmap_col) FROM user_tags GROUP BY category",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
// Other Doris aggregate functions
|
||||
{
|
||||
name: "APPROX_COUNT_DISTINCT",
|
||||
sql: "SELECT APPROX_COUNT_DISTINCT(user_id) FROM events",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
{
|
||||
name: "GROUP_CONCAT",
|
||||
sql: "SELECT GROUP_CONCAT(name) FROM users GROUP BY department",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
{
|
||||
name: "PERCENTILE_APPROX",
|
||||
sql: "SELECT PERCENTILE_APPROX(latency, 0.99) FROM requests",
|
||||
wantNeedCheck: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
needsCheck, _, _ := NeedsRowCountCheck(tt.sql, maxRows)
|
||||
if needsCheck != tt.wantNeedCheck {
|
||||
t.Errorf("needsCheck = %v, want %v (should skip check for Doris aggregate functions)", needsCheck, tt.wantNeedCheck)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsRowCountCheck_ComplexQueries(t *testing.T) {
|
||||
maxRows := 500
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantNeedCheck bool
|
||||
wantReject bool
|
||||
}{
|
||||
{
|
||||
name: "CTE with aggregate",
|
||||
sql: "WITH user_counts AS (SELECT user_id, COUNT(*) as cnt FROM orders GROUP BY user_id) SELECT * FROM user_counts",
|
||||
wantNeedCheck: false, // CTE has aggregate, skip check
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Complex JOIN with aggregate",
|
||||
sql: "SELECT u.department, COUNT(*) FROM users u JOIN orders o ON u.id = o.user_id GROUP BY u.department",
|
||||
wantNeedCheck: false, // has aggregate
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Nested subquery",
|
||||
sql: "SELECT * FROM users WHERE id IN (SELECT user_id FROM orders WHERE amount > 100)",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "DISTINCT query",
|
||||
sql: "SELECT DISTINCT category FROM products",
|
||||
wantNeedCheck: true, // DISTINCT is not aggregate
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "ORDER BY with LIMIT",
|
||||
sql: "SELECT * FROM users ORDER BY created_at DESC LIMIT 100",
|
||||
wantNeedCheck: false, // has valid LIMIT
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Multiple aggregates in single query",
|
||||
sql: "SELECT COUNT(*), SUM(amount), AVG(amount), MIN(amount), MAX(amount) FROM orders",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
needsCheck, directReject, _ := NeedsRowCountCheck(tt.sql, maxRows)
|
||||
if needsCheck != tt.wantNeedCheck {
|
||||
t.Errorf("needsCheck = %v, want %v", needsCheck, tt.wantNeedCheck)
|
||||
}
|
||||
if directReject != tt.wantReject {
|
||||
t.Errorf("directReject = %v, want %v", directReject, tt.wantReject)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsRowCountCheck_EdgeCases(t *testing.T) {
|
||||
maxRows := 500
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
wantNeedCheck bool
|
||||
wantReject bool
|
||||
}{
|
||||
{
|
||||
name: "Empty-ish LIMIT 0",
|
||||
sql: "SELECT * FROM users LIMIT 0",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT at boundary",
|
||||
sql: "SELECT * FROM users LIMIT 501",
|
||||
wantNeedCheck: true, // 501 > 500, needs probe check
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "SELECT with trailing semicolon",
|
||||
sql: "SELECT * FROM users;",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "SELECT with extra whitespace",
|
||||
sql: " SELECT * FROM users ",
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Lowercase keywords",
|
||||
sql: "select count(*) from users",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "Mixed case keywords",
|
||||
sql: "Select Count(*) From users",
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
needsCheck, directReject, _ := NeedsRowCountCheck(tt.sql, maxRows)
|
||||
if needsCheck != tt.wantNeedCheck {
|
||||
t.Errorf("needsCheck = %v, want %v", needsCheck, tt.wantNeedCheck)
|
||||
}
|
||||
if directReject != tt.wantReject {
|
||||
t.Errorf("directReject = %v, want %v", directReject, tt.wantReject)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeedsRowCountCheck_DifferentMaxRows(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sql string
|
||||
maxRows int
|
||||
wantNeedCheck bool
|
||||
wantReject bool
|
||||
}{
|
||||
{
|
||||
name: "LIMIT 100 with maxRows 50",
|
||||
sql: "SELECT * FROM users LIMIT 100",
|
||||
maxRows: 50,
|
||||
wantNeedCheck: true, // LIMIT > maxRows, needs probe check
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 100 with maxRows 100",
|
||||
sql: "SELECT * FROM users LIMIT 100",
|
||||
maxRows: 100,
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "LIMIT 100 with maxRows 200",
|
||||
sql: "SELECT * FROM users LIMIT 100",
|
||||
maxRows: 200,
|
||||
wantNeedCheck: false,
|
||||
wantReject: false,
|
||||
},
|
||||
{
|
||||
name: "No LIMIT with maxRows 1000",
|
||||
sql: "SELECT * FROM users",
|
||||
maxRows: 1000,
|
||||
wantNeedCheck: true,
|
||||
wantReject: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
needsCheck, directReject, _ := NeedsRowCountCheck(tt.sql, tt.maxRows)
|
||||
if needsCheck != tt.wantNeedCheck {
|
||||
t.Errorf("needsCheck = %v, want %v", needsCheck, tt.wantNeedCheck)
|
||||
}
|
||||
if directReject != tt.wantReject {
|
||||
t.Errorf("directReject = %v, want %v", directReject, tt.wantReject)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSummary_SkipProbeCheck prints a summary of which SQL patterns skip the probe check
|
||||
func TestSummary_SkipProbeCheck(t *testing.T) {
|
||||
maxRows := 500
|
||||
|
||||
skipCheckCases := []struct {
|
||||
category string
|
||||
sql string
|
||||
}{
|
||||
// Aggregate functions
|
||||
{"Aggregate - COUNT(*)", "SELECT COUNT(*) FROM users"},
|
||||
{"Aggregate - COUNT(col)", "SELECT COUNT(id) FROM users"},
|
||||
{"Aggregate - SUM", "SELECT SUM(amount) FROM orders"},
|
||||
{"Aggregate - AVG", "SELECT AVG(price) FROM products"},
|
||||
{"Aggregate - MIN", "SELECT MIN(created_at) FROM logs"},
|
||||
{"Aggregate - MAX", "SELECT MAX(score) FROM results"},
|
||||
{"Aggregate - GROUP BY", "SELECT user_id, COUNT(*) FROM orders GROUP BY user_id"},
|
||||
{"Aggregate - HAVING", "SELECT user_id, SUM(amount) FROM orders GROUP BY user_id HAVING SUM(amount) > 1000"},
|
||||
|
||||
// Doris specific aggregates
|
||||
{"Doris - HLL_UNION_AGG", "SELECT HLL_UNION_AGG(hll_col) FROM stats"},
|
||||
{"Doris - BITMAP_UNION_COUNT", "SELECT BITMAP_UNION_COUNT(bitmap_col) FROM tags"},
|
||||
{"Doris - APPROX_COUNT_DISTINCT", "SELECT APPROX_COUNT_DISTINCT(user_id) FROM events"},
|
||||
{"Doris - GROUP_CONCAT", "SELECT GROUP_CONCAT(name) FROM users GROUP BY dept"},
|
||||
|
||||
// LIMIT <= maxRows
|
||||
{"LIMIT - Equal to max", "SELECT * FROM users LIMIT 500"},
|
||||
{"LIMIT - Less than max", "SELECT * FROM users LIMIT 100"},
|
||||
{"LIMIT - With OFFSET", "SELECT * FROM users LIMIT 100 OFFSET 50"},
|
||||
{"LIMIT - Value 1", "SELECT * FROM users LIMIT 1"},
|
||||
{"LIMIT - Value 0", "SELECT * FROM users LIMIT 0"},
|
||||
}
|
||||
|
||||
t.Log("=== SQL patterns that SKIP probe check (no extra query needed) ===")
|
||||
for _, tc := range skipCheckCases {
|
||||
needsCheck, _, _ := NeedsRowCountCheck(tc.sql, maxRows)
|
||||
status := "✓ SKIP"
|
||||
if needsCheck {
|
||||
status = "✗ NEEDS CHECK (unexpected)"
|
||||
}
|
||||
t.Logf(" %s: %s\n SQL: %s", status, tc.category, tc.sql)
|
||||
}
|
||||
|
||||
needsCheckCases := []struct {
|
||||
category string
|
||||
sql string
|
||||
}{
|
||||
{"No LIMIT - Simple SELECT", "SELECT * FROM users"},
|
||||
{"No LIMIT - With WHERE", "SELECT * FROM users WHERE status = 'active'"},
|
||||
{"No LIMIT - With JOIN", "SELECT u.*, o.* FROM users u JOIN orders o ON u.id = o.user_id"},
|
||||
{"No LIMIT - Subquery with agg", "SELECT * FROM (SELECT user_id, COUNT(*) FROM orders GROUP BY user_id) t"},
|
||||
{"No LIMIT - DISTINCT", "SELECT DISTINCT category FROM products"},
|
||||
{"LIMIT > max (actual may be smaller)", "SELECT * FROM users LIMIT 1000"},
|
||||
{"LIMIT >> max", "SELECT * FROM users LIMIT 10000"},
|
||||
}
|
||||
|
||||
t.Log("\n=== SQL patterns that NEED probe check ===")
|
||||
for _, tc := range needsCheckCases {
|
||||
needsCheck, _, _ := NeedsRowCountCheck(tc.sql, maxRows)
|
||||
status := "✓ NEEDS CHECK"
|
||||
if !needsCheck {
|
||||
status = "✗ SKIP (unexpected)"
|
||||
}
|
||||
t.Logf(" %s: %s\n SQL: %s", status, tc.category, tc.sql)
|
||||
}
|
||||
}
|
||||
|
||||
// ptr is a helper function to create a pointer to int64
|
||||
func ptr(v int64) *int64 {
|
||||
return &v
|
||||
}
|
||||
@@ -15,6 +15,10 @@ const (
|
||||
TimeFieldFormatDateTime = "datetime"
|
||||
)
|
||||
|
||||
type noNeedCheckMaxRowKey struct{}
|
||||
|
||||
var NoNeedCheckMaxRow = noNeedCheckMaxRowKey{}
|
||||
|
||||
// 不再拼接SQL, 完全信赖用户的输入
|
||||
type QueryParam struct {
|
||||
Database string `json:"database"`
|
||||
@@ -39,7 +43,7 @@ var (
|
||||
)
|
||||
|
||||
// Query executes a given SQL query in Doris and returns the results with MaxQueryRows check
|
||||
func (d *Doris) Query(ctx context.Context, query *QueryParam) ([]map[string]interface{}, error) {
|
||||
func (d *Doris) Query(ctx context.Context, query *QueryParam, checkMaxRow bool) ([]map[string]interface{}, error) {
|
||||
// 校验SQL的合法性, 过滤掉 write请求
|
||||
sqlItem := strings.Split(strings.ToUpper(query.Sql), " ")
|
||||
for _, item := range sqlItem {
|
||||
@@ -48,10 +52,12 @@ func (d *Doris) Query(ctx context.Context, query *QueryParam) ([]map[string]inte
|
||||
}
|
||||
}
|
||||
|
||||
// 检查查询结果行数
|
||||
err := d.CheckMaxQueryRows(ctx, query.Database, query.Sql)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if checkMaxRow {
|
||||
// 检查查询结果行数
|
||||
err := d.CheckMaxQueryRows(ctx, query.Database, query.Sql)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := d.ExecQuery(ctx, query.Database, query.Sql)
|
||||
@@ -63,8 +69,12 @@ func (d *Doris) Query(ctx context.Context, query *QueryParam) ([]map[string]inte
|
||||
|
||||
// QueryTimeseries executes a time series data query using the given parameters with MaxQueryRows check
|
||||
func (d *Doris) QueryTimeseries(ctx context.Context, query *QueryParam) ([]types.MetricValues, error) {
|
||||
// 使用 Query 方法执行查询,Query方法内部已包含MaxQueryRows检查
|
||||
rows, err := d.Query(ctx, query)
|
||||
// 默认需要检查,除非调用方声明不需要检查
|
||||
checkMaxRow := true
|
||||
if noCheck, ok := ctx.Value(NoNeedCheckMaxRow).(bool); ok && noCheck {
|
||||
checkMaxRow = false
|
||||
}
|
||||
rows, err := d.Query(ctx, query, checkMaxRow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -73,35 +83,44 @@ func (d *Doris) QueryTimeseries(ctx context.Context, query *QueryParam) ([]types
|
||||
}
|
||||
|
||||
// CheckMaxQueryRows checks if the query result exceeds the maximum allowed rows
|
||||
// It uses SQL analysis to skip unnecessary checks for aggregate queries or queries with LIMIT <= maxRows
|
||||
// For queries that need checking, it uses probe approach (LIMIT maxRows+1) instead of COUNT(*) for better performance
|
||||
func (d *Doris) CheckMaxQueryRows(ctx context.Context, database, sql string) error {
|
||||
maxQueryRows := d.MaxQueryRows
|
||||
if maxQueryRows == 0 {
|
||||
maxQueryRows = 500
|
||||
}
|
||||
|
||||
cleanedSQL := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sql), ";"))
|
||||
|
||||
// Step 1: Analyze SQL to determine if check is needed
|
||||
needsCheck, _, _ := NeedsRowCountCheck(cleanedSQL, maxQueryRows)
|
||||
if !needsCheck {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Step 2: Execute probe query (more efficient than COUNT(*))
|
||||
return d.probeRowCount(ctx, database, cleanedSQL, maxQueryRows)
|
||||
}
|
||||
|
||||
// probeRowCount uses threshold probing to check row count
|
||||
// It reads at most maxRows+1 rows, which is O(maxRows) instead of O(totalRows) for COUNT(*)
|
||||
// Doris optimizes LIMIT queries by stopping scan early once limit is reached
|
||||
func (d *Doris) probeRowCount(ctx context.Context, database, sql string, maxRows int) error {
|
||||
timeoutCtx, cancel := d.createTimeoutContext(ctx)
|
||||
defer cancel()
|
||||
|
||||
cleanedSQL := strings.ReplaceAll(sql, ";", "")
|
||||
checkQuery := fmt.Sprintf("SELECT COUNT(*) as count FROM (%s) AS subquery;", cleanedSQL)
|
||||
// Probe SQL: only need to check if exceeds threshold, not actual data
|
||||
probeSQL := fmt.Sprintf("SELECT 1 FROM (%s) AS __probe_chk LIMIT %d", sql, maxRows+1)
|
||||
|
||||
// 执行计数查询
|
||||
results, err := d.ExecQuery(timeoutCtx, database, checkQuery)
|
||||
results, err := d.ExecQuery(timeoutCtx, database, probeSQL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(results) > 0 {
|
||||
if count, exists := results[0]["count"]; exists {
|
||||
v, err := sqlbase.ParseFloat64Value(count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
maxQueryRows := d.MaxQueryRows
|
||||
if maxQueryRows == 0 {
|
||||
maxQueryRows = 500
|
||||
}
|
||||
|
||||
if v > float64(maxQueryRows) {
|
||||
return fmt.Errorf("query result rows count %d exceeds the maximum limit %d", int(v), maxQueryRows)
|
||||
}
|
||||
}
|
||||
// If returned rows > maxRows, it exceeds the limit
|
||||
if len(results) > maxRows {
|
||||
return fmt.Errorf("query result rows count exceeds the maximum limit %d", maxRows)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -158,7 +158,10 @@ func FormatMetricValues(keys types.Keys, rows []map[string]interface{}, ignoreDe
|
||||
}
|
||||
|
||||
if !exists {
|
||||
ts = float64(time.Now().Unix()) // Default to current time if not specified
|
||||
// Default to current time if not specified
|
||||
// 大多数情况下offset为空
|
||||
// 对于记录规则延迟计算的情况,统计值的时间戳需要有偏移,以便跟统计值对应
|
||||
ts = float64(time.Now().Unix()) - float64(keys.Offset)
|
||||
}
|
||||
|
||||
valuePair := []float64{ts, value}
|
||||
|
||||
@@ -48,4 +48,5 @@ type Keys struct {
|
||||
LabelKey string `json:"labelKey" mapstructure:"labelKey"` // 多个用空格分隔
|
||||
TimeKey string `json:"timeKey" mapstructure:"timeKey"`
|
||||
TimeFormat string `json:"timeFormat" mapstructure:"timeFormat"` // not used anymore
|
||||
Offset int `json:"offset" mapstructure:"offset"`
|
||||
}
|
||||
|
||||
12
go.mod
12
go.mod
@@ -33,6 +33,7 @@ require (
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.5.1
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/mailru/easyjson v0.7.7
|
||||
github.com/mattn/go-isatty v0.0.19
|
||||
@@ -42,6 +43,7 @@ require (
|
||||
github.com/opensearch-project/opensearch-go/v2 v2.3.0
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pelletier/go-toml/v2 v2.0.8
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20260120034856-e15515e804da
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.20.5
|
||||
github.com/prometheus/common v0.60.1
|
||||
@@ -101,6 +103,9 @@ require (
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20250523034308-74f78ae071ee // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect
|
||||
github.com/pingcap/log v1.1.0 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rogpeppe/go-internal v1.13.1 // indirect
|
||||
@@ -108,10 +113,13 @@ require (
|
||||
github.com/valyala/fastrand v1.1.0 // indirect
|
||||
github.com/valyala/histogram v1.2.0 // indirect
|
||||
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
modernc.org/libc v1.22.5 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/sqlite v1.23.1 // indirect
|
||||
)
|
||||
@@ -135,7 +143,7 @@ require (
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.14.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
|
||||
|
||||
39
go.sum
39
go.sum
@@ -101,6 +101,7 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3w
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8=
|
||||
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
|
||||
@@ -195,8 +196,9 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
|
||||
github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
|
||||
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
|
||||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
|
||||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
@@ -243,6 +245,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
|
||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww=
|
||||
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
@@ -315,6 +318,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.5.1 h1:gX4dz92YU70inuIX+ug+PBe64eHToIN9rHB4Vupv5Eg=
|
||||
github.com/larksuite/oapi-sdk-go/v3 v3.5.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
@@ -361,9 +366,19 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ
|
||||
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pingcap/errors v0.11.5-0.20250523034308-74f78ae071ee h1:/IDPbpzkzA97t1/Z1+C3KlxbevjMeaI6BQYxvivu4u8=
|
||||
github.com/pingcap/errors v0.11.5-0.20250523034308-74f78ae071ee/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
|
||||
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
|
||||
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
|
||||
github.com/pingcap/log v1.1.0 h1:ELiPxACz7vdo1qAvvaWJg1NrYFoY6gqAh/+Uo6aXdD8=
|
||||
github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20260120034856-e15515e804da h1:PhkRZgMWdq9kTsu7vtVbcDs+SBXjHfFj84027WVZCzI=
|
||||
github.com/pingcap/tidb/pkg/parser v0.0.0-20260120034856-e15515e804da/go.mod h1:oHE+ub2QaDERd+UNHe4z2BhFV2jZrm7VNOe6atR9AF4=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
@@ -392,7 +407,6 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE=
|
||||
github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
@@ -467,13 +481,24 @@ go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
|
||||
go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
|
||||
go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
|
||||
go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
|
||||
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
|
||||
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
|
||||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
@@ -507,6 +532,7 @@ golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
@@ -623,6 +649,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
@@ -668,6 +696,9 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkp
|
||||
gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
|
||||
gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
@@ -693,8 +724,8 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
|
||||
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
|
||||
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
||||
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
|
||||
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
[
|
||||
{
|
||||
{
|
||||
"name": "JMX - Kubernetes",
|
||||
"tags": "Prometheus JMX Kubernetes",
|
||||
"configs": {
|
||||
@@ -1871,5 +1870,4 @@
|
||||
"version": "3.0.0"
|
||||
},
|
||||
"uuid": 1755595969673000
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -125,6 +125,7 @@ type SiteInfo struct {
|
||||
PrintBodyPaths []string `json:"print_body_paths"`
|
||||
PrintAccessLog bool `json:"print_access_log"`
|
||||
SiteUrl string `json:"site_url"`
|
||||
ReportHostNIC bool `json:"report_host_nic"`
|
||||
}
|
||||
|
||||
func (c *CvalCache) GetSiteInfo() *SiteInfo {
|
||||
|
||||
@@ -68,18 +68,6 @@ func (epc *EventProcessorCacheType) Get(processorId int64) *models.EventPipeline
|
||||
return epc.eventPipelines[processorId]
|
||||
}
|
||||
|
||||
func (epc *EventProcessorCacheType) GetProcessorsById(processorId int64) []models.Processor {
|
||||
epc.RLock()
|
||||
defer epc.RUnlock()
|
||||
|
||||
eventPipeline, ok := epc.eventPipelines[processorId]
|
||||
if !ok {
|
||||
return []models.Processor{}
|
||||
}
|
||||
|
||||
return eventPipeline.Processors
|
||||
}
|
||||
|
||||
func (epc *EventProcessorCacheType) GetProcessorIds() []int64 {
|
||||
epc.RLock()
|
||||
defer epc.RUnlock()
|
||||
@@ -137,18 +125,7 @@ func (epc *EventProcessorCacheType) syncEventProcessors() error {
|
||||
|
||||
m := make(map[int64]*models.EventPipeline)
|
||||
for i := 0; i < len(lst); i++ {
|
||||
eventPipeline := lst[i]
|
||||
for _, p := range eventPipeline.ProcessorConfigs {
|
||||
processor, err := models.GetProcessorByType(p.Typ, p.Config)
|
||||
if err != nil {
|
||||
logger.Warningf("event_pipeline_id: %d, event:%+v, processor:%+v get processor err: %+v", eventPipeline.ID, eventPipeline, p, err)
|
||||
continue
|
||||
}
|
||||
|
||||
eventPipeline.Processors = append(eventPipeline.Processors, processor)
|
||||
}
|
||||
|
||||
m[lst[i].ID] = eventPipeline
|
||||
m[lst[i].ID] = lst[i]
|
||||
}
|
||||
|
||||
epc.Set(m, stat.Total, stat.LastUpdated)
|
||||
|
||||
@@ -283,7 +283,7 @@ func (ncc *NotifyChannelCacheType) processNotifyTask(task *NotifyTask) {
|
||||
if len(task.Sendtos) == 0 || ncc.needBatchContacts(task.NotifyChannel.RequestConfig.HTTPRequestConfig) {
|
||||
start := time.Now()
|
||||
resp, err := task.NotifyChannel.SendHTTP(task.Events, task.TplContent, task.CustomParams, task.Sendtos, httpClient)
|
||||
resp = fmt.Sprintf("duration: %d ms %s", time.Since(start).Milliseconds(), resp)
|
||||
resp = fmt.Sprintf("send_time: %s duration: %d ms %s", time.Now().Format("2006-01-02 15:04:05"), time.Since(start).Milliseconds(), resp)
|
||||
logger.Infof("http_sendernotify_id: %d, channel_name: %v, event:%+v, tplContent:%v, customParams:%v, userInfo:%+v, respBody: %v, err: %v",
|
||||
task.NotifyRuleId, task.NotifyChannel.Name, task.Events[0], task.TplContent, task.CustomParams, task.Sendtos, resp, err)
|
||||
|
||||
|
||||
@@ -27,7 +27,8 @@ type TargetCacheType struct {
|
||||
redis storage.Redis
|
||||
|
||||
sync.RWMutex
|
||||
targets map[string]*models.Target // key: ident
|
||||
targets map[string]*models.Target // key: ident
|
||||
targetsIndex map[string][]string // key: ip, value: ident list
|
||||
}
|
||||
|
||||
func NewTargetCache(ctx *ctx.Context, stats *Stats, redis storage.Redis) *TargetCacheType {
|
||||
@@ -38,6 +39,7 @@ func NewTargetCache(ctx *ctx.Context, stats *Stats, redis storage.Redis) *Target
|
||||
stats: stats,
|
||||
redis: redis,
|
||||
targets: make(map[string]*models.Target),
|
||||
targetsIndex: make(map[string][]string),
|
||||
}
|
||||
|
||||
tc.SyncTargets()
|
||||
@@ -51,6 +53,7 @@ func (tc *TargetCacheType) Reset() {
|
||||
tc.statTotal = -1
|
||||
tc.statLastUpdated = -1
|
||||
tc.targets = make(map[string]*models.Target)
|
||||
tc.targetsIndex = make(map[string][]string)
|
||||
}
|
||||
|
||||
func (tc *TargetCacheType) StatChanged(total, lastUpdated int64) bool {
|
||||
@@ -62,8 +65,17 @@ func (tc *TargetCacheType) StatChanged(total, lastUpdated int64) bool {
|
||||
}
|
||||
|
||||
func (tc *TargetCacheType) Set(m map[string]*models.Target, total, lastUpdated int64) {
|
||||
idx := make(map[string][]string, len(m))
|
||||
for ident, target := range m {
|
||||
if _, ok := idx[target.HostIp]; !ok {
|
||||
idx[target.HostIp] = []string{}
|
||||
}
|
||||
idx[target.HostIp] = append(idx[target.HostIp], ident)
|
||||
}
|
||||
|
||||
tc.Lock()
|
||||
tc.targets = m
|
||||
tc.targetsIndex = idx
|
||||
tc.Unlock()
|
||||
|
||||
// only one goroutine used, so no need lock
|
||||
@@ -78,6 +90,75 @@ func (tc *TargetCacheType) Get(ident string) (*models.Target, bool) {
|
||||
return val, has
|
||||
}
|
||||
|
||||
func (tc *TargetCacheType) GetByIp(ip string) ([]*models.Target, bool) {
|
||||
tc.RLock()
|
||||
defer tc.RUnlock()
|
||||
idents, has := tc.targetsIndex[ip]
|
||||
if !has {
|
||||
return nil, false
|
||||
}
|
||||
targs := make([]*models.Target, 0, len(idents))
|
||||
for _, ident := range idents {
|
||||
if val, has := tc.targets[ident]; has {
|
||||
targs = append(targs, val)
|
||||
}
|
||||
}
|
||||
return targs, len(targs) > 0
|
||||
}
|
||||
|
||||
func (tc *TargetCacheType) GetAll() []*models.Target {
|
||||
tc.RLock()
|
||||
defer tc.RUnlock()
|
||||
lst := make([]*models.Target, 0, len(tc.targets))
|
||||
for _, target := range tc.targets {
|
||||
lst = append(lst, target)
|
||||
}
|
||||
return lst
|
||||
}
|
||||
|
||||
// GetAllBeatTime 返回所有 target 的心跳时间 map,key 为 ident,value 为 BeatTime
|
||||
func (tc *TargetCacheType) GetAllBeatTime() map[string]int64 {
|
||||
tc.RLock()
|
||||
defer tc.RUnlock()
|
||||
beatTimeMap := make(map[string]int64, len(tc.targets))
|
||||
for ident, target := range tc.targets {
|
||||
beatTimeMap[ident] = target.BeatTime
|
||||
}
|
||||
return beatTimeMap
|
||||
}
|
||||
|
||||
// refreshBeatTime 从 Redis 刷新缓存中所有 target 的 BeatTime
|
||||
func (tc *TargetCacheType) refreshBeatTime() {
|
||||
if tc.redis == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 快照 ident 列表,避免持锁访问 Redis
|
||||
tc.RLock()
|
||||
idents := make([]string, 0, len(tc.targets))
|
||||
for ident := range tc.targets {
|
||||
idents = append(idents, ident)
|
||||
}
|
||||
tc.RUnlock()
|
||||
|
||||
if len(idents) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
beatTimes := models.FetchBeatTimesFromRedis(tc.redis, idents)
|
||||
if len(beatTimes) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
tc.Lock()
|
||||
for ident, ts := range beatTimes {
|
||||
if target, ok := tc.targets[ident]; ok {
|
||||
target.BeatTime = ts
|
||||
}
|
||||
}
|
||||
tc.Unlock()
|
||||
}
|
||||
|
||||
func (tc *TargetCacheType) Gets(idents []string) []*models.Target {
|
||||
tc.RLock()
|
||||
defer tc.RUnlock()
|
||||
@@ -105,7 +186,7 @@ func (tc *TargetCacheType) GetOffsetHost(targets []*models.Target, now, offset i
|
||||
continue
|
||||
}
|
||||
|
||||
if now-target.UpdateAt > 120 {
|
||||
if now-target.BeatTime > 120 {
|
||||
// means this target is not a active host, do not check offset
|
||||
continue
|
||||
}
|
||||
@@ -147,6 +228,7 @@ func (tc *TargetCacheType) syncTargets() error {
|
||||
}
|
||||
|
||||
if !tc.StatChanged(stat.Total, stat.LastUpdated) {
|
||||
tc.refreshBeatTime()
|
||||
tc.stats.GaugeCronDuration.WithLabelValues("sync_targets").Set(0)
|
||||
tc.stats.GaugeSyncNumber.WithLabelValues("sync_targets").Set(0)
|
||||
dumper.PutSyncRecord("targets", start.Unix(), -1, -1, "not changed")
|
||||
@@ -170,6 +252,9 @@ func (tc *TargetCacheType) syncTargets() error {
|
||||
}
|
||||
}
|
||||
|
||||
// 从 Redis 批量获取心跳时间填充 BeatTime
|
||||
models.FillTargetsBeatTime(tc.redis, lst)
|
||||
|
||||
for i := 0; i < len(lst); i++ {
|
||||
m[lst[i].Ident] = lst[i]
|
||||
}
|
||||
@@ -186,57 +271,18 @@ func (tc *TargetCacheType) syncTargets() error {
|
||||
|
||||
// get host update time
|
||||
func (tc *TargetCacheType) GetHostUpdateTime(targets []string) map[string]int64 {
|
||||
metaMap := make(map[string]int64)
|
||||
if tc.redis == nil {
|
||||
return metaMap
|
||||
return make(map[string]int64)
|
||||
}
|
||||
|
||||
num := 0
|
||||
var keys []string
|
||||
for i := 0; i < len(targets); i++ {
|
||||
keys = append(keys, models.WrapIdentUpdateTime(targets[i]))
|
||||
num++
|
||||
if num == 100 {
|
||||
vals := storage.MGet(context.Background(), tc.redis, keys)
|
||||
for _, value := range vals {
|
||||
var hostUpdateTime models.HostUpdateTime
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := json.Unmarshal(value, &hostUpdateTime)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to unmarshal host meta: %s value:%v", err, value)
|
||||
continue
|
||||
}
|
||||
metaMap[hostUpdateTime.Ident] = hostUpdateTime.UpdateTime
|
||||
}
|
||||
keys = keys[:0]
|
||||
num = 0
|
||||
}
|
||||
}
|
||||
|
||||
vals := storage.MGet(context.Background(), tc.redis, keys)
|
||||
for _, value := range vals {
|
||||
var hostUpdateTime models.HostUpdateTime
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := json.Unmarshal(value, &hostUpdateTime)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to unmarshal host err:%v value:%s", err, string(value))
|
||||
continue
|
||||
}
|
||||
metaMap[hostUpdateTime.Ident] = hostUpdateTime.UpdateTime
|
||||
}
|
||||
metaMap := models.FetchBeatTimesFromRedis(tc.redis, targets)
|
||||
|
||||
for _, ident := range targets {
|
||||
if _, ok := metaMap[ident]; !ok {
|
||||
// if not exists, get from cache
|
||||
target, exists := tc.Get(ident)
|
||||
if exists {
|
||||
metaMap[ident] = target.UpdateAt
|
||||
metaMap[ident] = target.BeatTime
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,10 +509,16 @@ func (ar *AlertRule) Verify() error {
|
||||
|
||||
ar.AppendTags = strings.TrimSpace(ar.AppendTags)
|
||||
arr := strings.Fields(ar.AppendTags)
|
||||
appendTagKeys := make(map[string]struct{})
|
||||
for i := 0; i < len(arr); i++ {
|
||||
if !strings.Contains(arr[i], "=") {
|
||||
return fmt.Errorf("AppendTags(%s) invalid", arr[i])
|
||||
}
|
||||
pair := strings.SplitN(arr[i], "=", 2)
|
||||
if _, exists := appendTagKeys[pair[0]]; exists {
|
||||
return fmt.Errorf("AppendTags has duplicate key: %s", pair[0])
|
||||
}
|
||||
appendTagKeys[pair[0]] = struct{}{}
|
||||
}
|
||||
|
||||
gids := strings.Fields(ar.NotifyGroups)
|
||||
|
||||
@@ -46,6 +46,12 @@ func NewAnomalyPoint(key string, labels map[string]string, ts int64, value float
|
||||
}
|
||||
|
||||
func (v *AnomalyPoint) ReadableValue() string {
|
||||
if len(v.ValuesUnit) > 0 {
|
||||
for _, unit := range v.ValuesUnit { // 配置了单位,优先用配置了单位的值
|
||||
return unit.Text
|
||||
}
|
||||
}
|
||||
|
||||
ret := fmt.Sprintf("%.5f", v.Value)
|
||||
ret = strings.TrimRight(ret, "0")
|
||||
return strings.TrimRight(ret, ".")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -12,20 +13,23 @@ import (
|
||||
|
||||
// BuiltinMetric represents a metric along with its metadata.
|
||||
type BuiltinMetric struct {
|
||||
ID int64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
|
||||
UUID int64 `json:"uuid" gorm:"type:bigint;not null;default:0;comment:'uuid'"`
|
||||
Collector string `json:"collector" gorm:"type:varchar(191);not null;index:idx_collector,sort:asc;comment:'type of collector'"`
|
||||
Typ string `json:"typ" gorm:"type:varchar(191);not null;index:idx_typ,sort:asc;comment:'type of metric'"`
|
||||
Name string `json:"name" gorm:"type:varchar(191);not null;index:idx_builtinmetric_name,sort:asc;comment:'name of metric'"`
|
||||
Unit string `json:"unit" gorm:"type:varchar(191);not null;comment:'unit of metric'"`
|
||||
Note string `json:"note" gorm:"type:varchar(4096);not null;comment:'description of metric'"`
|
||||
Lang string `json:"lang" gorm:"type:varchar(191);not null;default:'zh';index:idx_lang,sort:asc;comment:'language'"`
|
||||
Translation []Translation `json:"translation" gorm:"type:text;serializer:json;comment:'translation of metric'"`
|
||||
Expression string `json:"expression" gorm:"type:varchar(4096);not null;comment:'expression of metric'"`
|
||||
CreatedAt int64 `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
|
||||
CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
|
||||
UpdatedAt int64 `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
|
||||
UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
|
||||
ID int64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
|
||||
UUID int64 `json:"uuid" gorm:"type:bigint;not null;default:0;comment:'uuid'"`
|
||||
Collector string `json:"collector" gorm:"type:varchar(191);not null;index:idx_collector,sort:asc;comment:'type of collector'"`
|
||||
Typ string `json:"typ" gorm:"type:varchar(191);not null;index:idx_typ,sort:asc;comment:'type of metric'"`
|
||||
Name string `json:"name" gorm:"type:varchar(191);not null;index:idx_builtinmetric_name,sort:asc;comment:'name of metric'"`
|
||||
Unit string `json:"unit" gorm:"type:varchar(191);not null;comment:'unit of metric'"`
|
||||
Note string `json:"note" gorm:"type:varchar(4096);not null;comment:'description of metric'"`
|
||||
Lang string `json:"lang" gorm:"type:varchar(191);not null;default:'zh';index:idx_lang,sort:asc;comment:'language'"`
|
||||
Translation []Translation `json:"translation" gorm:"type:text;serializer:json;comment:'translation of metric'"`
|
||||
Expression string `json:"expression" gorm:"type:varchar(4096);not null;comment:'expression of metric'"`
|
||||
ExpressionType string `json:"expression_type" gorm:"type:varchar(32);not null;default:'promql';comment:'expression type: metric_name or promql'"`
|
||||
MetricType string `json:"metric_type" gorm:"type:varchar(191);not null;default:'';comment:'metric type like counter/gauge'"`
|
||||
ExtraFields json.RawMessage `json:"extra_fields" gorm:"type:text;serializer:json;comment:'custom extra fields'"`
|
||||
CreatedAt int64 `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
|
||||
CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
|
||||
UpdatedAt int64 `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
|
||||
UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
|
||||
}
|
||||
|
||||
type Translation struct {
|
||||
|
||||
@@ -29,6 +29,27 @@ func (bp *BuiltinPayload) TableName() string {
|
||||
return "builtin_payloads"
|
||||
}
|
||||
|
||||
type PostgresBuiltinPayload struct {
|
||||
ID int64 `json:"id" gorm:"primaryKey;type:bigint;autoIncrement;comment:'unique identifier'"`
|
||||
Type string `json:"type" gorm:"type:varchar(191);not null;index:idx_type,sort:asc;comment:'type of payload'"`
|
||||
Component string `json:"component" gorm:"type:varchar(191);not null;index:idx_component,sort:asc;comment:'component of payload'"`
|
||||
ComponentID uint64 `json:"component_id" gorm:"type:bigint;index:idx_component,sort:asc;comment:'component_id of payload'"`
|
||||
Cate string `json:"cate" gorm:"type:varchar(191);not null;comment:'category of payload'"`
|
||||
Name string `json:"name" gorm:"type:varchar(191);not null;index:idx_buildinpayload_name,sort:asc;comment:'name of payload'"`
|
||||
Tags string `json:"tags" gorm:"type:varchar(191);not null;default:'';comment:'tags of payload'"`
|
||||
Content string `json:"content" gorm:"type:text;not null;comment:'content of payload'"`
|
||||
UUID int64 `json:"uuid" gorm:"type:bigint;not null;index:idx_uuid;comment:'uuid of payload'"`
|
||||
Note string `json:"note" gorm:"type:varchar(1024);not null;default:'';comment:'note of payload'"`
|
||||
CreatedAt int64 `json:"created_at" gorm:"type:bigint;not null;default:0;comment:'create time'"`
|
||||
CreatedBy string `json:"created_by" gorm:"type:varchar(191);not null;default:'';comment:'creator'"`
|
||||
UpdatedAt int64 `json:"updated_at" gorm:"type:bigint;not null;default:0;comment:'update time'"`
|
||||
UpdatedBy string `json:"updated_by" gorm:"type:varchar(191);not null;default:'';comment:'updater'"`
|
||||
}
|
||||
|
||||
func (bp *PostgresBuiltinPayload) TableName() string {
|
||||
return "builtin_payloads"
|
||||
}
|
||||
|
||||
func (bp *BuiltinPayload) Verify() error {
|
||||
bp.Type = strings.TrimSpace(bp.Type)
|
||||
if bp.Type == "" {
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -42,6 +45,7 @@ type Datasource struct {
|
||||
CreatedBy string `json:"created_by"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
IsDefault bool `json:"is_default"`
|
||||
Weight int `json:"weight"`
|
||||
Transport *http.Transport `json:"-" gorm:"-"`
|
||||
ForceSave bool `json:"force_save" gorm:"-"`
|
||||
}
|
||||
@@ -144,6 +148,72 @@ func (h HTTP) ParseUrl() (target *url.URL, err error) {
|
||||
|
||||
type TLS struct {
|
||||
SkipTlsVerify bool `json:"skip_tls_verify"`
|
||||
// mTLS 配置
|
||||
CACert string `json:"ca_cert"` // CA 证书内容 (PEM 格式)
|
||||
ClientCert string `json:"client_cert"` // 客户端证书内容 (PEM 格式)
|
||||
ClientKey string `json:"client_key"` // 客户端密钥内容 (PEM 格式)
|
||||
ClientKeyPassword string `json:"client_key_password"` // 密钥密码(可选)
|
||||
ServerName string `json:"server_name"` // TLS ServerName(可选,用于证书验证)
|
||||
MinVersion string `json:"min_version"` // TLS 最小版本 (1.0, 1.1, 1.2, 1.3)
|
||||
MaxVersion string `json:"max_version"` // TLS 最大版本
|
||||
}
|
||||
|
||||
// TLSConfig 从证书内容创建 tls.Config
|
||||
// 证书内容为 PEM 格式字符串
|
||||
func (t *TLS) TLSConfig() (*tls.Config, error) {
|
||||
tlsConfig := &tls.Config{
|
||||
InsecureSkipVerify: t.SkipTlsVerify,
|
||||
}
|
||||
|
||||
// 设置 ServerName
|
||||
if t.ServerName != "" {
|
||||
tlsConfig.ServerName = t.ServerName
|
||||
}
|
||||
|
||||
// 设置 TLS 版本
|
||||
if t.MinVersion != "" {
|
||||
if v, ok := tlsVersionMap[t.MinVersion]; ok {
|
||||
tlsConfig.MinVersion = v
|
||||
}
|
||||
}
|
||||
if t.MaxVersion != "" {
|
||||
if v, ok := tlsVersionMap[t.MaxVersion]; ok {
|
||||
tlsConfig.MaxVersion = v
|
||||
}
|
||||
}
|
||||
|
||||
// 如果配置了客户端证书,则加载 mTLS 配置
|
||||
clientCert := strings.TrimSpace(t.ClientCert)
|
||||
clientKey := strings.TrimSpace(t.ClientKey)
|
||||
caCert := strings.TrimSpace(t.CACert)
|
||||
|
||||
if clientCert != "" && clientKey != "" {
|
||||
// 加载客户端证书和密钥
|
||||
cert, err := tls.X509KeyPair([]byte(clientCert), []byte(clientKey))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load client certificate: %w", err)
|
||||
}
|
||||
tlsConfig.Certificates = []tls.Certificate{cert}
|
||||
}
|
||||
|
||||
// 加载 CA 证书
|
||||
if caCert != "" {
|
||||
caCertPool := x509.NewCertPool()
|
||||
if !caCertPool.AppendCertsFromPEM([]byte(caCert)) {
|
||||
return nil, fmt.Errorf("failed to parse CA certificate")
|
||||
}
|
||||
tlsConfig.RootCAs = caCertPool
|
||||
}
|
||||
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
// tlsVersionMap TLS 版本映射
|
||||
var tlsVersionMap = map[string]uint16{
|
||||
"1.0": tls.VersionTLS10,
|
||||
"1.1": tls.VersionTLS11,
|
||||
"1.2": tls.VersionTLS12,
|
||||
"1.3": tls.VersionTLS13,
|
||||
}
|
||||
|
||||
func (ds *Datasource) TableName() string {
|
||||
@@ -448,7 +518,8 @@ func (ds *Datasource) Encrypt(openRsa bool, publicKeyData []byte) error {
|
||||
// Decrypt 用于 edge 将从中心同步的数据源解密,中心不可调用
|
||||
func (ds *Datasource) Decrypt() error {
|
||||
if rsaConfig == nil {
|
||||
return errors.New("rsa config is nil")
|
||||
logger.Debugf("datasource %s rsa config is nil", ds.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
if !rsaConfig.OpenRSA {
|
||||
|
||||
@@ -13,6 +13,10 @@ import (
|
||||
type EventPipeline struct {
|
||||
ID int64 `json:"id" gorm:"primaryKey"`
|
||||
Name string `json:"name" gorm:"type:varchar(128)"`
|
||||
Typ string `json:"typ" gorm:"type:varchar(128)"` // builtin, user-defined // event_pipeline, event_summary, metric_explorer
|
||||
UseCase string `json:"use_case" gorm:"type:varchar(128)"` // metric_explorer, event_summary, event_pipeline
|
||||
TriggerMode string `json:"trigger_mode" gorm:"type:varchar(128)"` // event, api, cron
|
||||
Disabled bool `json:"disabled" gorm:"type:boolean"`
|
||||
TeamIds []int64 `json:"team_ids" gorm:"type:text;serializer:json"`
|
||||
TeamNames []string `json:"team_names" gorm:"-"`
|
||||
Description string `json:"description" gorm:"type:varchar(255)"`
|
||||
@@ -20,12 +24,18 @@ type EventPipeline struct {
|
||||
LabelFilters []TagFilter `json:"label_filters" gorm:"type:text;serializer:json"`
|
||||
AttrFilters []TagFilter `json:"attribute_filters" gorm:"type:text;serializer:json"`
|
||||
ProcessorConfigs []ProcessorConfig `json:"processors" gorm:"type:text;serializer:json"`
|
||||
CreateAt int64 `json:"create_at" gorm:"type:bigint"`
|
||||
CreateBy string `json:"create_by" gorm:"type:varchar(64)"`
|
||||
UpdateAt int64 `json:"update_at" gorm:"type:bigint"`
|
||||
UpdateBy string `json:"update_by" gorm:"type:varchar(64)"`
|
||||
|
||||
Processors []Processor `json:"-" gorm:"-"`
|
||||
// 工作流节点列表
|
||||
Nodes []WorkflowNode `json:"nodes,omitempty" gorm:"type:text;serializer:json"`
|
||||
// 节点连接关系
|
||||
Connections Connections `json:"connections,omitempty" gorm:"type:text;serializer:json"`
|
||||
// 输入参数(工作流级别的配置变量)
|
||||
Inputs []InputVariable `json:"inputs,omitempty" gorm:"type:text;serializer:json"`
|
||||
|
||||
CreateAt int64 `json:"create_at" gorm:"type:bigint"`
|
||||
CreateBy string `json:"create_by" gorm:"type:varchar(64)"`
|
||||
UpdateAt int64 `json:"update_at" gorm:"type:bigint"`
|
||||
UpdateBy string `json:"update_by" gorm:"type:varchar(64)"`
|
||||
}
|
||||
|
||||
type ProcessorConfig struct {
|
||||
@@ -46,9 +56,6 @@ func (e *EventPipeline) Verify() error {
|
||||
return errors.New("team_ids cannot be empty")
|
||||
}
|
||||
|
||||
if len(e.TeamIds) == 0 {
|
||||
e.TeamIds = make([]int64, 0)
|
||||
}
|
||||
if len(e.LabelFilters) == 0 {
|
||||
e.LabelFilters = make([]TagFilter, 0)
|
||||
}
|
||||
@@ -59,6 +66,17 @@ func (e *EventPipeline) Verify() error {
|
||||
e.ProcessorConfigs = make([]ProcessorConfig, 0)
|
||||
}
|
||||
|
||||
// 初始化空数组,避免 null
|
||||
if e.Nodes == nil {
|
||||
e.Nodes = make([]WorkflowNode, 0)
|
||||
}
|
||||
if e.Connections == nil {
|
||||
e.Connections = make(Connections)
|
||||
}
|
||||
if e.Inputs == nil {
|
||||
e.Inputs = make([]InputVariable, 0)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -176,3 +194,61 @@ func EventPipelineStatistics(ctx *ctx.Context) (*Statistics, error) {
|
||||
|
||||
return stats[0], nil
|
||||
}
|
||||
|
||||
// 无论是新格式还是旧格式,都返回统一的 []WorkflowNode
|
||||
func (e *EventPipeline) GetWorkflowNodes() []WorkflowNode {
|
||||
// 优先使用新格式
|
||||
if len(e.Nodes) > 0 {
|
||||
return e.Nodes
|
||||
}
|
||||
|
||||
// 兼容旧格式:将 ProcessorConfigs 转换为 WorkflowNode
|
||||
nodes := make([]WorkflowNode, len(e.ProcessorConfigs))
|
||||
for i, pc := range e.ProcessorConfigs {
|
||||
nodeID := fmt.Sprintf("node_%d", i)
|
||||
nodeName := pc.Typ
|
||||
|
||||
nodes[i] = WorkflowNode{
|
||||
ID: nodeID,
|
||||
Name: nodeName,
|
||||
Type: pc.Typ,
|
||||
Config: pc.Config,
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
func (e *EventPipeline) GetWorkflowConnections() Connections {
|
||||
// 优先使用显式定义的连接
|
||||
if len(e.Connections) > 0 {
|
||||
return e.Connections
|
||||
}
|
||||
|
||||
// 自动生成线性连接:node_0 → node_1 → node_2 → ...
|
||||
nodes := e.GetWorkflowNodes()
|
||||
conns := make(Connections)
|
||||
|
||||
for i := 0; i < len(nodes)-1; i++ {
|
||||
conns[nodes[i].ID] = NodeConnections{
|
||||
Main: [][]ConnectionTarget{
|
||||
{{Node: nodes[i+1].ID, Type: "main", Index: 0}},
|
||||
},
|
||||
}
|
||||
}
|
||||
return conns
|
||||
}
|
||||
|
||||
func (e *EventPipeline) FillWorkflowFields() {
|
||||
if len(e.Nodes) == 0 && len(e.ProcessorConfigs) > 0 {
|
||||
e.Nodes = e.GetWorkflowNodes()
|
||||
e.Connections = e.GetWorkflowConnections()
|
||||
}
|
||||
}
|
||||
|
||||
func (e *EventPipeline) GetInputsMap() map[string]string {
|
||||
inputsMap := make(map[string]string)
|
||||
for _, v := range e.Inputs {
|
||||
inputsMap[v.Key] = v.Value
|
||||
}
|
||||
return inputsMap
|
||||
}
|
||||
|
||||
320
models/event_pipeline_execution.go
Normal file
320
models/event_pipeline_execution.go
Normal file
@@ -0,0 +1,320 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/poster"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// 执行状态常量
|
||||
const (
|
||||
ExecutionStatusRunning = "running"
|
||||
ExecutionStatusSuccess = "success"
|
||||
ExecutionStatusFailed = "failed"
|
||||
)
|
||||
|
||||
// EventPipelineExecution 工作流执行记录
|
||||
type EventPipelineExecution struct {
|
||||
ID string `json:"id" gorm:"primaryKey;type:varchar(36)"`
|
||||
PipelineID int64 `json:"pipeline_id" gorm:"index"`
|
||||
PipelineName string `json:"pipeline_name" gorm:"type:varchar(128)"`
|
||||
EventID int64 `json:"event_id" gorm:"index"`
|
||||
|
||||
// 触发模式:event(告警触发)、api(API触发)、cron(定时触发)
|
||||
Mode string `json:"mode" gorm:"type:varchar(16);index"`
|
||||
|
||||
// 状态:running、success、failed
|
||||
Status string `json:"status" gorm:"type:varchar(16);index"`
|
||||
|
||||
// 各节点执行结果(JSON)
|
||||
NodeResults string `json:"node_results" gorm:"type:mediumtext"`
|
||||
|
||||
// 错误信息
|
||||
ErrorMessage string `json:"error_message" gorm:"type:varchar(1024)"`
|
||||
ErrorNode string `json:"error_node" gorm:"type:varchar(36)"`
|
||||
|
||||
// 时间
|
||||
CreatedAt int64 `json:"created_at" gorm:"index"`
|
||||
FinishedAt int64 `json:"finished_at"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
|
||||
// 触发者信息
|
||||
TriggerBy string `json:"trigger_by" gorm:"type:varchar(64)"`
|
||||
|
||||
// 输入参数快照(脱敏后存储)
|
||||
InputsSnapshot string `json:"inputs_snapshot,omitempty" gorm:"type:text"`
|
||||
}
|
||||
|
||||
func (e *EventPipelineExecution) TableName() string {
|
||||
return "event_pipeline_execution"
|
||||
}
|
||||
|
||||
// SetNodeResults 设置节点执行结果(序列化为 JSON)
|
||||
func (e *EventPipelineExecution) SetNodeResults(results []*NodeExecutionResult) error {
|
||||
data, err := json.Marshal(results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.NodeResults = string(data)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetNodeResults 获取节点执行结果(反序列化)
|
||||
func (e *EventPipelineExecution) GetNodeResults() ([]*NodeExecutionResult, error) {
|
||||
if e.NodeResults == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var results []*NodeExecutionResult
|
||||
err := json.Unmarshal([]byte(e.NodeResults), &results)
|
||||
return results, err
|
||||
}
|
||||
|
||||
// SetInputsSnapshot 设置输入参数快照(脱敏后存储)
|
||||
func (e *EventPipelineExecution) SetInputsSnapshot(inputs map[string]string) error {
|
||||
data, err := json.Marshal(inputs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.InputsSnapshot = string(data)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetInputsSnapshot 获取输入参数快照
|
||||
func (e *EventPipelineExecution) GetInputsSnapshot() (map[string]string, error) {
|
||||
if e.InputsSnapshot == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var inputs map[string]string
|
||||
err := json.Unmarshal([]byte(e.InputsSnapshot), &inputs)
|
||||
return inputs, err
|
||||
}
|
||||
|
||||
// CreateEventPipelineExecution 创建执行记录
|
||||
func CreateEventPipelineExecution(c *ctx.Context, execution *EventPipelineExecution) error {
|
||||
if !c.IsCenter {
|
||||
return poster.PostByUrls(c, "/v1/n9e/event-pipeline-execution", execution)
|
||||
}
|
||||
return DB(c).Create(execution).Error
|
||||
}
|
||||
|
||||
// UpdateEventPipelineExecution 更新执行记录
|
||||
func UpdateEventPipelineExecution(c *ctx.Context, execution *EventPipelineExecution) error {
|
||||
return DB(c).Save(execution).Error
|
||||
}
|
||||
|
||||
// GetEventPipelineExecution 获取单条执行记录
|
||||
func GetEventPipelineExecution(c *ctx.Context, id string) (*EventPipelineExecution, error) {
|
||||
var execution EventPipelineExecution
|
||||
err := DB(c).Where("id = ?", id).First(&execution).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &execution, nil
|
||||
}
|
||||
|
||||
// ListEventPipelineExecutions 获取 Pipeline 的执行记录列表
|
||||
func ListEventPipelineExecutions(c *ctx.Context, pipelineID int64, mode, status string, limit, offset int) ([]*EventPipelineExecution, int64, error) {
|
||||
var executions []*EventPipelineExecution
|
||||
var total int64
|
||||
|
||||
session := DB(c).Model(&EventPipelineExecution{}).Where("pipeline_id = ?", pipelineID)
|
||||
|
||||
if mode != "" {
|
||||
session = session.Where("mode = ?", mode)
|
||||
}
|
||||
if status != "" {
|
||||
session = session.Where("status = ?", status)
|
||||
}
|
||||
|
||||
err := session.Count(&total).Error
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
err = session.Order("created_at desc").Limit(limit).Offset(offset).Find(&executions).Error
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return executions, total, nil
|
||||
}
|
||||
|
||||
// ListEventPipelineExecutionsByEventID 根据事件ID获取执行记录
|
||||
func ListEventPipelineExecutionsByEventID(c *ctx.Context, eventID int64) ([]*EventPipelineExecution, error) {
|
||||
var executions []*EventPipelineExecution
|
||||
err := DB(c).Where("event_id = ?", eventID).Order("created_at desc").Find(&executions).Error
|
||||
return executions, err
|
||||
}
|
||||
|
||||
// ListAllEventPipelineExecutions 获取所有 Pipeline 的执行记录列表
|
||||
func ListAllEventPipelineExecutions(c *ctx.Context, pipelineId int64, pipelineName, mode, status string, limit, offset int) ([]*EventPipelineExecution, int64, error) {
|
||||
var executions []*EventPipelineExecution
|
||||
var total int64
|
||||
|
||||
session := DB(c).Model(&EventPipelineExecution{})
|
||||
|
||||
if pipelineId > 0 {
|
||||
session = session.Where("pipeline_id = ?", pipelineId)
|
||||
}
|
||||
if pipelineName != "" {
|
||||
session = session.Where("pipeline_name LIKE ?", "%"+pipelineName+"%")
|
||||
}
|
||||
if mode != "" {
|
||||
session = session.Where("mode = ?", mode)
|
||||
}
|
||||
if status != "" {
|
||||
session = session.Where("status = ?", status)
|
||||
}
|
||||
|
||||
err := session.Count(&total).Error
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
err = session.Order("created_at desc").Limit(limit).Offset(offset).Find(&executions).Error
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return executions, total, nil
|
||||
}
|
||||
|
||||
// DeleteEventPipelineExecutions 批量删除执行记录(按时间)
|
||||
func DeleteEventPipelineExecutions(c *ctx.Context, beforeTime int64) (int64, error) {
|
||||
result := DB(c).Where("created_at < ?", beforeTime).Delete(&EventPipelineExecution{})
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
// DeleteEventPipelineExecutionsInBatches 分批删除执行记录(按时间)
|
||||
// 每次删除 limit 条记录,返回本次删除的数量
|
||||
// 使用子查询方式实现,兼容 MySQL、PostgreSQL、SQLite
|
||||
func DeleteEventPipelineExecutionsInBatches(c *ctx.Context, beforeTime int64, limit int) (int64, error) {
|
||||
// 先查询要删除的 ID
|
||||
var ids []string
|
||||
err := DB(c).Model(&EventPipelineExecution{}).
|
||||
Where("created_at < ?", beforeTime).
|
||||
Limit(limit).
|
||||
Pluck("id", &ids).Error
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
// 按 ID 删除
|
||||
result := DB(c).Where("id IN ?", ids).Delete(&EventPipelineExecution{})
|
||||
return result.RowsAffected, result.Error
|
||||
}
|
||||
|
||||
// DeleteEventPipelineExecutionsByPipelineID 删除指定 Pipeline 的所有执行记录
|
||||
func DeleteEventPipelineExecutionsByPipelineID(c *ctx.Context, pipelineID int64) error {
|
||||
return DB(c).Where("pipeline_id = ?", pipelineID).Delete(&EventPipelineExecution{}).Error
|
||||
}
|
||||
|
||||
// EventPipelineExecutionStatistics 执行统计
|
||||
type EventPipelineExecutionStatistics struct {
|
||||
Total int64 `json:"total"`
|
||||
Success int64 `json:"success"`
|
||||
Failed int64 `json:"failed"`
|
||||
Running int64 `json:"running"`
|
||||
AvgDurMs int64 `json:"avg_duration_ms"`
|
||||
LastRunAt int64 `json:"last_run_at"`
|
||||
}
|
||||
|
||||
// GetEventPipelineExecutionStatistics 获取执行统计信息
|
||||
func GetEventPipelineExecutionStatistics(c *ctx.Context, pipelineID int64) (*EventPipelineExecutionStatistics, error) {
|
||||
var stats EventPipelineExecutionStatistics
|
||||
|
||||
// 总数
|
||||
err := DB(c).Model(&EventPipelineExecution{}).Where("pipeline_id = ?", pipelineID).Count(&stats.Total).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 成功数
|
||||
err = DB(c).Model(&EventPipelineExecution{}).Where("pipeline_id = ? AND status = ?", pipelineID, ExecutionStatusSuccess).Count(&stats.Success).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 失败数
|
||||
err = DB(c).Model(&EventPipelineExecution{}).Where("pipeline_id = ? AND status = ?", pipelineID, ExecutionStatusFailed).Count(&stats.Failed).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 运行中
|
||||
err = DB(c).Model(&EventPipelineExecution{}).Where("pipeline_id = ? AND status = ?", pipelineID, ExecutionStatusRunning).Count(&stats.Running).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 平均耗时
|
||||
var avgDur struct {
|
||||
AvgDur float64 `gorm:"column:avg_dur"`
|
||||
}
|
||||
err = DB(c).Model(&EventPipelineExecution{}).
|
||||
Select("AVG(duration_ms) as avg_dur").
|
||||
Where("pipeline_id = ? AND status = ?", pipelineID, ExecutionStatusSuccess).
|
||||
Scan(&avgDur).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats.AvgDurMs = int64(avgDur.AvgDur)
|
||||
|
||||
// 最后执行时间
|
||||
var lastExec EventPipelineExecution
|
||||
err = DB(c).Where("pipeline_id = ?", pipelineID).Order("created_at desc").First(&lastExec).Error
|
||||
if err == nil {
|
||||
stats.LastRunAt = lastExec.CreatedAt
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
// EventPipelineExecutionDetail 执行详情(包含解析后的节点结果)
|
||||
type EventPipelineExecutionDetail struct {
|
||||
EventPipelineExecution
|
||||
NodeResultsParsed []*NodeExecutionResult `json:"node_results_parsed"`
|
||||
InputsSnapshotParsed map[string]string `json:"inputs_snapshot_parsed"`
|
||||
}
|
||||
|
||||
// GetEventPipelineExecutionDetail 获取执行详情
|
||||
func GetEventPipelineExecutionDetail(c *ctx.Context, id string) (*EventPipelineExecutionDetail, error) {
|
||||
execution, err := GetEventPipelineExecution(c, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if execution == nil {
|
||||
return &EventPipelineExecutionDetail{}, nil
|
||||
}
|
||||
|
||||
detail := &EventPipelineExecutionDetail{
|
||||
EventPipelineExecution: *execution,
|
||||
}
|
||||
|
||||
// 解析节点结果
|
||||
nodeResults, err := execution.GetNodeResults()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse node results error: %w", err)
|
||||
}
|
||||
detail.NodeResultsParsed = nodeResults
|
||||
|
||||
// 解析输入参数快照
|
||||
inputsSnapshot, err := execution.GetInputsSnapshot()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse inputs snapshot error: %w", err)
|
||||
}
|
||||
detail.InputsSnapshotParsed = inputsSnapshot
|
||||
|
||||
return detail, nil
|
||||
}
|
||||
@@ -9,11 +9,21 @@ import (
|
||||
|
||||
type Processor interface {
|
||||
Init(settings interface{}) (Processor, error) // 初始化配置
|
||||
Process(ctx *ctx.Context, event *AlertCurEvent) (*AlertCurEvent, string, error)
|
||||
Process(ctx *ctx.Context, wfCtx *WorkflowContext) (*WorkflowContext, string, error)
|
||||
// 处理器有三种情况:
|
||||
// 1. 处理成功,返回处理后的事件
|
||||
// 2. 处理成功,不需要返回处理后端事件,只返回处理结果,将处理结果放到 string 中,比如 eventdrop callback 处理器
|
||||
// 1. 处理成功,返回处理后的 WorkflowContext
|
||||
// 2. 处理成功,不需要返回处理后的上下文,只返回处理结果,将处理结果放到 string 中,比如 eventdrop callback 处理器
|
||||
// 3. 处理失败,返回错误,将错误放到 error 中
|
||||
// WorkflowContext 包含:Event(事件)、Env(环境变量/输入参数)、Metadata(执行元数据)
|
||||
}
|
||||
|
||||
// BranchProcessor 分支处理器接口
|
||||
// 用于 if、switch、foreach 等需要返回分支索引或特殊输出的处理器
|
||||
type BranchProcessor interface {
|
||||
Processor
|
||||
// ProcessWithBranch 处理事件并返回 NodeOutput
|
||||
// NodeOutput 包含:处理后的上下文、消息、是否终止、分支索引
|
||||
ProcessWithBranch(ctx *ctx.Context, wfCtx *WorkflowContext) (*NodeOutput, error)
|
||||
}
|
||||
|
||||
type NewProcessorFn func(settings interface{}) (Processor, error)
|
||||
|
||||
@@ -622,9 +622,26 @@ var NewTplMap = map[string]string{
|
||||
{{if $event.RuleNote }}**Alarm description:** **{{$event.RuleNote}}**{{end}}
|
||||
{{- end -}}
|
||||
[Event Details]({{.domain}}/share/alert-his-events/{{$event.Id}})|[Block for 1 hour]({{.domain}}/alert-mutes/add?__event_id={{$event.Id}})|[View Curve]({{.domain}}/metric/explorer?__event_id={{$event.Id}}&mode=graph)`,
|
||||
|
||||
// Jira and JSMAlert share the same template format
|
||||
Jira: `Severity: S{{$event.Severity}} {{if $event.IsRecovered}}Recovered{{else}}Triggered{{end}}
|
||||
Rule Name: {{$event.RuleName}}{{if $event.RuleNote}}
|
||||
Rule Notes: {{$event.RuleNote}}{{end}}
|
||||
Metrics: {{$event.TagsJSON}}
|
||||
Annotations:
|
||||
{{- range $key, $val := $event.AnnotationsJSON}}
|
||||
{{$key}}: {{$val}}
|
||||
{{- end}}\n{{if $event.IsRecovered}}Recovery Time: {{timeformat $event.LastEvalTime}}{{else}}Trigger Time: {{timeformat $event.TriggerTime}}
|
||||
Trigger Value: {{$event.TriggerValue}}{{end}}
|
||||
Send Time: {{timestamp}}
|
||||
Event Details: {{.domain}}/share/alert-his-events/{{$event.Id}}
|
||||
Mute for 1 Hour: {{.domain}}/alert-mutes/add?__event_id={{$event.Id}}`,
|
||||
}
|
||||
|
||||
// Weight 用于页面元素排序,weight 越大 排序越靠后
|
||||
var MsgTplMap = []MessageTemplate{
|
||||
{Name: "Jira", Ident: Jira, Weight: 18, Content: map[string]string{"content": NewTplMap[Jira]}},
|
||||
{Name: "JSMAlert", Ident: JSMAlert, Weight: 17, Content: map[string]string{"content": NewTplMap[Jira]}},
|
||||
{Name: "Callback", Ident: "callback", Weight: 16, Content: map[string]string{"content": ""}},
|
||||
{Name: "MattermostWebhook", Ident: MattermostWebhook, Weight: 15, Content: map[string]string{"content": NewTplMap[MattermostWebhook]}},
|
||||
{Name: "MattermostBot", Ident: MattermostBot, Weight: 14, Content: map[string]string{"content": NewTplMap[MattermostWebhook]}},
|
||||
|
||||
@@ -68,7 +68,8 @@ func MigrateTables(db *gorm.DB) error {
|
||||
&Board{}, &BoardBusigroup{}, &Users{}, &SsoConfig{}, &models.BuiltinMetric{},
|
||||
&models.MetricFilter{}, &models.NotificationRecord{}, &models.TargetBusiGroup{},
|
||||
&models.UserToken{}, &models.DashAnnotation{}, MessageTemplate{}, NotifyRule{}, NotifyChannelConfig{}, &EsIndexPatternMigrate{},
|
||||
&models.EventPipeline{}, &models.EmbeddedProduct{}, &models.SourceToken{}}
|
||||
&models.EventPipeline{}, &models.EventPipelineExecution{}, &models.EmbeddedProduct{}, &models.SourceToken{},
|
||||
&models.SavedView{}, &models.UserViewFavorite{}}
|
||||
|
||||
if isPostgres(db) {
|
||||
dts = append(dts, &models.PostgresBuiltinComponent{})
|
||||
@@ -98,7 +99,11 @@ func MigrateTables(db *gorm.DB) error {
|
||||
}()
|
||||
|
||||
if !db.Migrator().HasTable(&models.BuiltinPayload{}) {
|
||||
dts = append(dts, &models.BuiltinPayload{})
|
||||
if isPostgres(db) {
|
||||
dts = append(dts, &models.PostgresBuiltinPayload{})
|
||||
} else {
|
||||
dts = append(dts, &models.BuiltinPayload{})
|
||||
}
|
||||
} else {
|
||||
dts = append(dts, &BuiltinPayloads{})
|
||||
}
|
||||
@@ -229,6 +234,7 @@ type Target struct {
|
||||
type Datasource struct {
|
||||
IsDefault bool `gorm:"column:is_default;type:boolean;comment:is default datasource"`
|
||||
Identifier string `gorm:"column:identifier;type:varchar(255);default:'';comment:identifier"`
|
||||
Weight int `gorm:"column:weight;type:int;default:0;comment:weight for sorting"`
|
||||
}
|
||||
|
||||
type Configs struct {
|
||||
|
||||
@@ -354,8 +354,17 @@ func GetHTTPClient(nc *NotifyChannelConfig) (*http.Client, error) {
|
||||
|
||||
// 设置代理
|
||||
var proxyFunc func(*http.Request) (*url.URL, error)
|
||||
if httpConfig.Proxy != "" {
|
||||
proxyURL, err := url.Parse(httpConfig.Proxy)
|
||||
proxy := httpConfig.Proxy
|
||||
// 对于 FlashDuty 类型,优先使用 FlashDuty 配置中的代理
|
||||
if nc.RequestType == "flashduty" && nc.RequestConfig.FlashDutyRequestConfig != nil && nc.RequestConfig.FlashDutyRequestConfig.Proxy != "" {
|
||||
proxy = nc.RequestConfig.FlashDutyRequestConfig.Proxy
|
||||
}
|
||||
// 对于 PagerDuty 类型,优先使用 PagerDuty 配置中的代理
|
||||
if nc.RequestType == "pagerduty" && nc.RequestConfig.PagerDutyRequestConfig != nil && nc.RequestConfig.PagerDutyRequestConfig.Proxy != "" {
|
||||
proxy = nc.RequestConfig.PagerDutyRequestConfig.Proxy
|
||||
}
|
||||
if proxy != "" {
|
||||
proxyURL, err := url.Parse(proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid proxy URL: %v", err)
|
||||
}
|
||||
@@ -725,10 +734,10 @@ func (ncc *NotifyChannelConfig) SendHTTP(events []*AlertCurEvent, tpl map[string
|
||||
logger.Errorf("send_http: failed to read response. url=%s request_body=%s error=%v", url, string(body), err)
|
||||
}
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return string(body), nil
|
||||
return fmt.Sprintf("status_code:%d, response:%s", resp.StatusCode, string(body)), nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to send request, status code: %d, body: %s", resp.StatusCode, string(body))
|
||||
return fmt.Sprintf("status_code:%d, response:%s", resp.StatusCode, string(body)), fmt.Errorf("failed to send request, status code: %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return lastErrorMessage, errors.New("all retries failed, last error: " + lastErrorMessage)
|
||||
@@ -1204,24 +1213,56 @@ func (c NotiChList) IfUsed(nr *NotifyRule) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Weight 用于页面元素排序,weight 越大 排序越靠后
|
||||
var NotiChMap = []*NotifyChannelConfig{
|
||||
{
|
||||
Name: "Callback", Ident: "callback", RequestType: "http", Weight: 2, Enable: true,
|
||||
Name: "PagerDuty", Ident: "pagerduty", RequestType: "pagerduty", Weight: 19, Enable: true,
|
||||
RequestConfig: &RequestConfig{
|
||||
PagerDutyRequestConfig: &PagerDutyRequestConfig{
|
||||
ApiKey: "pagerduty api key",
|
||||
Timeout: 5000,
|
||||
RetryTimes: 3,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "JIRA", Ident: Jira, RequestType: "http", Weight: 18, Enable: true,
|
||||
RequestConfig: &RequestConfig{
|
||||
HTTPRequestConfig: &HTTPRequestConfig{
|
||||
URL: "{{$params.callback_url}}",
|
||||
Method: "POST", Headers: map[string]string{"Content-Type": "application/json"},
|
||||
URL: "https://{JIRA Service Account Email}:{API Token}@api.atlassian.com/ex/jira/{CloudID}/rest/api/3/issue",
|
||||
Method: "POST",
|
||||
Headers: map[string]string{"Content-Type": "application/json"},
|
||||
Timeout: 10000, Concurrency: 5, RetryTimes: 3, RetryInterval: 100,
|
||||
Request: RequestDetail{
|
||||
Body: `{{ jsonMarshal $events }}`,
|
||||
Body: `{"fields":{"project":{"key":"{{$params.project_key}}"},"issuetype":{"name":"{{if $event.IsRecovered}}Recovery{{else}}Alert{{end}}"},"summary":"{{$event.RuleName}}","description":{"type":"doc","version":1,"content":[{"type":"paragraph","content":[{"type":"text","text":"{{$tpl.content}}"}]}]},"labels":["{{join $event.TagsJSON "\",\""}}", "eventHash={{$event.Hash}}"]}}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
ParamConfig: &NotifyParamConfig{
|
||||
Custom: Params{
|
||||
Params: []ParamItem{
|
||||
{Key: "callback_url", CName: "Callback Url", Type: "string"},
|
||||
{Key: "note", CName: "Note", Type: "string"},
|
||||
{Key: "project_key", CName: "Project Key", Type: "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "JSM Alert", Ident: JSMAlert, RequestType: "http", Weight: 17, Enable: true,
|
||||
RequestConfig: &RequestConfig{
|
||||
HTTPRequestConfig: &HTTPRequestConfig{
|
||||
URL: `https://api.atlassian.com/jsm/ops/integration/v2/alerts{{if $event.IsRecovered}}/{{$event.Hash}}/close?identifierType=alias{{else}}{{end}}`,
|
||||
Method: "POST",
|
||||
Headers: map[string]string{"Content-Type": "application/json", "Authorization": "GenieKey {{$params.api_key}}"},
|
||||
Timeout: 10000, Concurrency: 5, RetryTimes: 3, RetryInterval: 100,
|
||||
Request: RequestDetail{
|
||||
Body: `{{if $event.IsRecovered}}{"note":"{{$tpl.content}}","source":"{{$event.Cluster}}"}{{else}}{"message":"{{$event.RuleName}}","description":"{{$tpl.content}}","alias":"{{$event.Hash}}","priority":"P{{$event.Severity}}","tags":[{{range $i, $v := $event.TagsJSON}}{{if $i}},{{end}}"{{$v}}"{{end}}],"details":{{jsonMarshal $event.AnnotationsJSON}},"entity":"{{$event.TargetIdent}}","source":"{{$event.Cluster}}"}{{end}}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
ParamConfig: &NotifyParamConfig{
|
||||
Custom: Params{
|
||||
Params: []ParamItem{
|
||||
{Key: "api_key", CName: "API Key", Type: "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1614,6 +1655,27 @@ var NotiChMap = []*NotifyChannelConfig{
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "Callback", Ident: "callback", RequestType: "http", Weight: 2, Enable: true,
|
||||
RequestConfig: &RequestConfig{
|
||||
HTTPRequestConfig: &HTTPRequestConfig{
|
||||
URL: "{{$params.callback_url}}",
|
||||
Method: "POST", Headers: map[string]string{"Content-Type": "application/json"},
|
||||
Timeout: 10000, Concurrency: 5, RetryTimes: 3, RetryInterval: 100,
|
||||
Request: RequestDetail{
|
||||
Body: `{{ jsonMarshal $events }}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
ParamConfig: &NotifyParamConfig{
|
||||
Custom: Params{
|
||||
Params: []ParamItem{
|
||||
{Key: "callback_url", CName: "Callback Url", Type: "string"},
|
||||
{Key: "note", CName: "Note", Type: "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "FlashDuty", Ident: "flashduty", RequestType: "flashduty", Weight: 1, Enable: true,
|
||||
RequestConfig: &RequestConfig{
|
||||
@@ -1630,16 +1692,6 @@ var NotiChMap = []*NotifyChannelConfig{
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "PagerDuty", Ident: "pagerduty", RequestType: "pagerduty", Weight: 1, Enable: true,
|
||||
RequestConfig: &RequestConfig{
|
||||
PagerDutyRequestConfig: &PagerDutyRequestConfig{
|
||||
ApiKey: "pagerduty api key",
|
||||
Timeout: 5000,
|
||||
RetryTimes: 3,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func InitNotifyChannel(ctx *ctx.Context) {
|
||||
|
||||
@@ -44,6 +44,7 @@ type QueryConfig struct {
|
||||
Exp string `json:"exp"`
|
||||
WriteDatasourceId int64 `json:"write_datasource_id"`
|
||||
Delay int `json:"delay"`
|
||||
WritebackEnabled bool `json:"writeback_enabled"` // 是否写入与查询数据源相同的数据源
|
||||
}
|
||||
|
||||
type Query struct {
|
||||
@@ -211,7 +212,6 @@ func (re *RecordingRule) Update(ctx *ctx.Context, ref RecordingRule) error {
|
||||
|
||||
ref.FE2DB()
|
||||
ref.Id = re.Id
|
||||
ref.GroupId = re.GroupId
|
||||
ref.CreateAt = re.CreateAt
|
||||
ref.CreateBy = re.CreateBy
|
||||
ref.UpdateAt = time.Now().Unix()
|
||||
|
||||
174
models/saved_view.go
Normal file
174
models/saved_view.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrSavedViewNameEmpty = errors.New("saved view name is blank")
|
||||
ErrSavedViewPageEmpty = errors.New("saved view page is blank")
|
||||
ErrSavedViewNotFound = errors.New("saved view not found")
|
||||
ErrSavedViewNameDuplicate = errors.New("saved view name already exists in this page")
|
||||
)
|
||||
|
||||
type SavedView struct {
|
||||
Id int64 `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null"`
|
||||
Page string `json:"page" gorm:"type:varchar(64);not null;index"`
|
||||
Filter string `json:"filter" gorm:"type:text"`
|
||||
PublicCate int `json:"public_cate" gorm:"default:0"` // 0: self, 1: team, 2: all
|
||||
Gids []int64 `json:"gids" gorm:"column:gids;type:text;serializer:json"`
|
||||
CreateAt int64 `json:"create_at" gorm:"type:bigint;not null;default:0"`
|
||||
CreateBy string `json:"create_by" gorm:"type:varchar(64);index"`
|
||||
UpdateAt int64 `json:"update_at" gorm:"type:bigint;not null;default:0"`
|
||||
UpdateBy string `json:"update_by" gorm:"type:varchar(64)"`
|
||||
|
||||
// 查询时填充的字段
|
||||
IsFavorite bool `json:"is_favorite" gorm:"-"`
|
||||
}
|
||||
|
||||
func (SavedView) TableName() string {
|
||||
return "saved_view"
|
||||
}
|
||||
|
||||
func (sv *SavedView) Verify() error {
|
||||
sv.Name = strings.TrimSpace(sv.Name)
|
||||
if sv.Name == "" {
|
||||
return ErrSavedViewNameEmpty
|
||||
}
|
||||
if sv.Page == "" {
|
||||
return ErrSavedViewPageEmpty
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SavedViewCheckDuplicateName(c *ctx.Context, page, name string, excludeId int64) error {
|
||||
var count int64
|
||||
session := DB(c).Model(&SavedView{}).Where("page = ? AND name = ? AND public_cate = 2", page, name)
|
||||
if excludeId > 0 {
|
||||
session = session.Where("id != ?", excludeId)
|
||||
}
|
||||
if err := session.Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if count > 0 {
|
||||
return ErrSavedViewNameDuplicate
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SavedViewAdd(c *ctx.Context, sv *SavedView) error {
|
||||
if err := sv.Verify(); err != nil {
|
||||
return err
|
||||
}
|
||||
// 当 PublicCate 为 all(2) 时,检查同一个 page 下 name 是否重复
|
||||
if sv.PublicCate == 2 {
|
||||
if err := SavedViewCheckDuplicateName(c, sv.Page, sv.Name, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
sv.CreateAt = now
|
||||
sv.UpdateAt = now
|
||||
return Insert(c, sv)
|
||||
}
|
||||
|
||||
func SavedViewUpdate(c *ctx.Context, sv *SavedView, username string) error {
|
||||
if err := sv.Verify(); err != nil {
|
||||
return err
|
||||
}
|
||||
// 当 PublicCate 为 all(2) 时,检查同一个 page 下 name 是否重复(排除自身)
|
||||
if sv.PublicCate == 2 {
|
||||
if err := SavedViewCheckDuplicateName(c, sv.Page, sv.Name, sv.Id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
sv.UpdateAt = time.Now().Unix()
|
||||
sv.UpdateBy = username
|
||||
return DB(c).Model(sv).Select("name", "filter", "public_cate", "gids", "update_at", "update_by").Updates(sv).Error
|
||||
}
|
||||
|
||||
func SavedViewDel(c *ctx.Context, id int64) error {
|
||||
// 先删除收藏关联
|
||||
if err := DB(c).Where("view_id = ?", id).Delete(&UserViewFavorite{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return DB(c).Where("id = ?", id).Delete(&SavedView{}).Error
|
||||
}
|
||||
|
||||
func SavedViewGetById(c *ctx.Context, id int64) (*SavedView, error) {
|
||||
var sv SavedView
|
||||
err := DB(c).Where("id = ?", id).First(&sv).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &sv, nil
|
||||
}
|
||||
|
||||
func SavedViewGets(c *ctx.Context, page string) ([]SavedView, error) {
|
||||
var views []SavedView
|
||||
|
||||
session := DB(c).Where("page = ?", page)
|
||||
|
||||
if err := session.Order("update_at DESC").Find(&views).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return views, nil
|
||||
}
|
||||
|
||||
func SavedViewFavoriteGetByUserId(c *ctx.Context, userId int64) (map[int64]bool, error) {
|
||||
var favorites []UserViewFavorite
|
||||
if err := DB(c).Where("user_id = ?", userId).Find(&favorites).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make(map[int64]bool)
|
||||
for _, f := range favorites {
|
||||
result[f.ViewId] = true
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type UserViewFavorite struct {
|
||||
Id int64 `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
ViewId int64 `json:"view_id" gorm:"index"`
|
||||
UserId int64 `json:"user_id" gorm:"index"`
|
||||
CreateAt int64 `json:"create_at"`
|
||||
}
|
||||
|
||||
func (UserViewFavorite) TableName() string {
|
||||
return "user_view_favorite"
|
||||
}
|
||||
|
||||
func UserViewFavoriteAdd(c *ctx.Context, viewId, userId int64) error {
|
||||
var count int64
|
||||
if err := DB(c).Model(&SavedView{}).Where("id = ?", viewId).Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
return ErrSavedViewNotFound
|
||||
}
|
||||
|
||||
if err := DB(c).Model(&UserViewFavorite{}).Where("view_id = ? AND user_id = ?", viewId, userId).Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if count > 0 {
|
||||
return nil // 已收藏,直接返回成功
|
||||
}
|
||||
|
||||
fav := &UserViewFavorite{
|
||||
ViewId: viewId,
|
||||
UserId: userId,
|
||||
CreateAt: time.Now().Unix(),
|
||||
}
|
||||
return DB(c).Create(fav).Error
|
||||
}
|
||||
|
||||
func UserViewFavoriteDel(c *ctx.Context, viewId, userId int64) error {
|
||||
return DB(c).Where("view_id = ? AND user_id = ?", viewId, userId).Delete(&UserViewFavorite{}).Error
|
||||
}
|
||||
114
models/target.go
114
models/target.go
@@ -1,6 +1,8 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -8,6 +10,7 @@ import (
|
||||
|
||||
"github.com/ccfos/nightingale/v6/pkg/ctx"
|
||||
"github.com/ccfos/nightingale/v6/pkg/poster"
|
||||
"github.com/ccfos/nightingale/v6/storage"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -36,6 +39,7 @@ type Target struct {
|
||||
OS string `json:"os" gorm:"column:os"`
|
||||
HostTags []string `json:"host_tags" gorm:"serializer:json"`
|
||||
|
||||
BeatTime int64 `json:"beat_time" gorm:"-"` // 实时心跳时间,从 Redis 获取
|
||||
UnixTime int64 `json:"unixtime" gorm:"-"`
|
||||
Offset int64 `json:"offset" gorm:"-"`
|
||||
TargetUp float64 `json:"target_up" gorm:"-"`
|
||||
@@ -97,12 +101,6 @@ func (t *Target) MatchGroupId(gid ...int64) bool {
|
||||
}
|
||||
|
||||
func (t *Target) AfterFind(tx *gorm.DB) (err error) {
|
||||
delta := time.Now().Unix() - t.UpdateAt
|
||||
if delta < 60 {
|
||||
t.TargetUp = 2
|
||||
} else if delta < 180 {
|
||||
t.TargetUp = 1
|
||||
}
|
||||
t.FillTagsMap()
|
||||
return
|
||||
}
|
||||
@@ -182,6 +180,24 @@ func BuildTargetWhereWithHosts(hosts []string) BuildTargetWhereOption {
|
||||
}
|
||||
}
|
||||
|
||||
func BuildTargetWhereWithIdents(idents []string) BuildTargetWhereOption {
|
||||
return func(session *gorm.DB) *gorm.DB {
|
||||
if len(idents) > 0 {
|
||||
session = session.Where("ident in (?)", idents)
|
||||
}
|
||||
return session
|
||||
}
|
||||
}
|
||||
|
||||
func BuildTargetWhereExcludeIdents(idents []string) BuildTargetWhereOption {
|
||||
return func(session *gorm.DB) *gorm.DB {
|
||||
if len(idents) > 0 {
|
||||
session = session.Where("ident not in (?)", idents)
|
||||
}
|
||||
return session
|
||||
}
|
||||
}
|
||||
|
||||
func BuildTargetWhereWithQuery(query string) BuildTargetWhereOption {
|
||||
return func(session *gorm.DB) *gorm.DB {
|
||||
if query != "" {
|
||||
@@ -203,17 +219,6 @@ func BuildTargetWhereWithQuery(query string) BuildTargetWhereOption {
|
||||
}
|
||||
}
|
||||
|
||||
func BuildTargetWhereWithDowntime(downtime int64) BuildTargetWhereOption {
|
||||
return func(session *gorm.DB) *gorm.DB {
|
||||
if downtime > 0 {
|
||||
session = session.Where("target.update_at < ?", time.Now().Unix()-downtime)
|
||||
} else if downtime < 0 {
|
||||
session = session.Where("target.update_at > ?", time.Now().Unix()+downtime)
|
||||
}
|
||||
return session
|
||||
}
|
||||
}
|
||||
|
||||
func buildTargetWhere(ctx *ctx.Context, options ...BuildTargetWhereOption) *gorm.DB {
|
||||
sub := DB(ctx).Model(&Target{}).Distinct("target.ident")
|
||||
for _, opt := range options {
|
||||
@@ -264,21 +269,6 @@ func TargetCountByFilter(ctx *ctx.Context, query []map[string]interface{}) (int6
|
||||
return Count(session)
|
||||
}
|
||||
|
||||
func MissTargetGetsByFilter(ctx *ctx.Context, query []map[string]interface{}, ts int64) ([]*Target, error) {
|
||||
var lst []*Target
|
||||
session := TargetFilterQueryBuild(ctx, query, 0, 0)
|
||||
session = session.Where("update_at < ?", ts)
|
||||
|
||||
err := session.Order("ident").Find(&lst).Error
|
||||
return lst, err
|
||||
}
|
||||
|
||||
func MissTargetCountByFilter(ctx *ctx.Context, query []map[string]interface{}, ts int64) (int64, error) {
|
||||
session := TargetFilterQueryBuild(ctx, query, 0, 0)
|
||||
session = session.Where("update_at < ?", ts)
|
||||
return Count(session)
|
||||
}
|
||||
|
||||
func TargetFilterQueryBuild(ctx *ctx.Context, query []map[string]interface{}, limit, offset int) *gorm.DB {
|
||||
sub := DB(ctx).Model(&Target{}).Distinct("target.ident").Joins("left join " +
|
||||
"target_busi_group on target.ident = target_busi_group.target_ident")
|
||||
@@ -619,6 +609,66 @@ func (t *Target) FillMeta(meta *HostMeta) {
|
||||
t.RemoteAddr = meta.RemoteAddr
|
||||
}
|
||||
|
||||
// FetchBeatTimesFromRedis 从 Redis 批量获取心跳时间,返回 ident -> updateTime 的映射
|
||||
func FetchBeatTimesFromRedis(redis storage.Redis, idents []string) map[string]int64 {
|
||||
result := make(map[string]int64, len(idents))
|
||||
if redis == nil || len(idents) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
num := 0
|
||||
var keys []string
|
||||
for i := 0; i < len(idents); i++ {
|
||||
keys = append(keys, WrapIdentUpdateTime(idents[i]))
|
||||
num++
|
||||
if num == 100 {
|
||||
fetchBeatTimeBatch(redis, keys, result)
|
||||
keys = keys[:0]
|
||||
num = 0
|
||||
}
|
||||
}
|
||||
|
||||
if len(keys) > 0 {
|
||||
fetchBeatTimeBatch(redis, keys, result)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func fetchBeatTimeBatch(redis storage.Redis, keys []string, result map[string]int64) {
|
||||
vals := storage.MGet(context.Background(), redis, keys)
|
||||
for _, value := range vals {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
var hut HostUpdateTime
|
||||
if err := json.Unmarshal(value, &hut); err != nil {
|
||||
logger.Warningf("failed to unmarshal host update time: %v", err)
|
||||
continue
|
||||
}
|
||||
result[hut.Ident] = hut.UpdateTime
|
||||
}
|
||||
}
|
||||
|
||||
// FillTargetsBeatTime 从 Redis 批量获取心跳时间填充 target.BeatTime
|
||||
func FillTargetsBeatTime(redis storage.Redis, targets []*Target) {
|
||||
if len(targets) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
idents := make([]string, len(targets))
|
||||
for i, t := range targets {
|
||||
idents[i] = t.Ident
|
||||
}
|
||||
|
||||
beatTimes := FetchBeatTimesFromRedis(redis, idents)
|
||||
for _, t := range targets {
|
||||
if ts, ok := beatTimes[t.Ident]; ok {
|
||||
t.BeatTime = ts
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TargetIdents(ctx *ctx.Context, ids []int64) ([]string, error) {
|
||||
var ret []string
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ type RelationKey struct {
|
||||
type QueryParam struct {
|
||||
Cate string `json:"cate"`
|
||||
DatasourceId int64 `json:"datasource_id"`
|
||||
Queries []interface{} `json:"query"`
|
||||
Queries []interface{} `json:"query"`
|
||||
}
|
||||
|
||||
type Series struct {
|
||||
|
||||
@@ -42,6 +42,8 @@ const (
|
||||
Lark = "lark"
|
||||
LarkCard = "larkcard"
|
||||
Phone = "phone"
|
||||
Jira = "jira"
|
||||
JSMAlert = "jsm_alert"
|
||||
|
||||
DingtalkKey = "dingtalk_robot_token"
|
||||
WecomKey = "wecom_robot_token"
|
||||
@@ -313,6 +315,18 @@ func (u *User) UpdatePassword(ctx *ctx.Context, password, updateBy string) error
|
||||
}).Error
|
||||
}
|
||||
|
||||
func (u *User) UpdateUserGroup(ctx *ctx.Context, userGroupIds []int64) error {
|
||||
|
||||
count := len(userGroupIds)
|
||||
for i := 0; i < count; i++ {
|
||||
err := UserGroupMemberAdd(ctx, userGroupIds[i], u.Id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateUserLastActiveTime(ctx *ctx.Context, userId int64, lastActiveTime int64) error {
|
||||
return DB(ctx).Model(&User{}).Where("id = ?", userId).Updates(map[string]interface{}{
|
||||
"last_active_time": lastActiveTime,
|
||||
@@ -339,6 +353,11 @@ func (u *User) Del(ctx *ctx.Context) error {
|
||||
}
|
||||
|
||||
func (u *User) ChangePassword(ctx *ctx.Context, oldpass, newpass string) error {
|
||||
// SSO 用户(ldap/oidc/cas/oauth2/dingtalk等)且未设置本地密码,不支持本地修改密码
|
||||
if u.Belong != "" && u.Password == "******" {
|
||||
return fmt.Errorf("SSO user(%s) cannot change password locally, please change password in %s", u.Username, u.Belong)
|
||||
}
|
||||
|
||||
_oldpass, err := CryptoPass(ctx, oldpass)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
150
models/workflow.go
Normal file
150
models/workflow.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package models
|
||||
|
||||
// WorkflowNode 工作流节点
|
||||
type WorkflowNode struct {
|
||||
ID string `json:"id"` // 节点唯一ID
|
||||
Name string `json:"name"` // 显示名称
|
||||
Type string `json:"type"` // 节点类型(对应 Processor typ)
|
||||
Position []float64 `json:"position,omitempty"` // [x, y] UI位置
|
||||
Config interface{} `json:"config"` // 节点配置
|
||||
|
||||
// 执行控制
|
||||
Disabled bool `json:"disabled,omitempty"`
|
||||
ContinueOnFail bool `json:"continue_on_fail,omitempty"`
|
||||
RetryOnFail bool `json:"retry_on_fail,omitempty"`
|
||||
MaxRetries int `json:"max_retries,omitempty"`
|
||||
RetryInterval int `json:"retry_interval,omitempty"` // 秒
|
||||
}
|
||||
|
||||
// Connections 节点连接关系 map[源节点ID]NodeConnections
|
||||
type Connections map[string]NodeConnections
|
||||
|
||||
// NodeConnections 单个节点的输出连接
|
||||
type NodeConnections struct {
|
||||
// Main 输出端口的连接
|
||||
// Main[outputIndex] = []ConnectionTarget
|
||||
Main [][]ConnectionTarget `json:"main"`
|
||||
}
|
||||
|
||||
// ConnectionTarget 连接目标
|
||||
type ConnectionTarget struct {
|
||||
Node string `json:"node"` // 目标节点ID
|
||||
Type string `json:"type"` // 输入类型,通常是 "main"
|
||||
Index int `json:"index"` // 目标节点的输入端口索引
|
||||
}
|
||||
|
||||
// InputVariable 输入参数
|
||||
type InputVariable struct {
|
||||
Key string `json:"key"` // 变量名
|
||||
Value string `json:"value"` // 默认值
|
||||
Description string `json:"description,omitempty"` // 描述
|
||||
}
|
||||
|
||||
// NodeOutput 节点执行输出
|
||||
type NodeOutput struct {
|
||||
WfCtx *WorkflowContext `json:"wf_ctx"` // 处理后的工作流上下文
|
||||
Message string `json:"message"` // 处理消息
|
||||
Terminate bool `json:"terminate"` // 是否终止流程
|
||||
BranchIndex *int `json:"branch_index,omitempty"` // 分支索引(条件节点使用)
|
||||
|
||||
// 流式输出支持
|
||||
Stream bool `json:"stream,omitempty"` // 是否流式输出
|
||||
StreamChan chan *StreamChunk `json:"-"` // 流式数据通道(不序列化)
|
||||
}
|
||||
|
||||
// WorkflowResult 工作流执行结果
|
||||
type WorkflowResult struct {
|
||||
Event *AlertCurEvent `json:"event"` // 最终事件
|
||||
Status string `json:"status"` // success, failed, streaming
|
||||
Message string `json:"message"` // 汇总消息
|
||||
NodeResults []*NodeExecutionResult `json:"node_results"` // 各节点执行结果
|
||||
ErrorNode string `json:"error_node,omitempty"`
|
||||
|
||||
// 流式输出支持
|
||||
Stream bool `json:"stream,omitempty"` // 是否流式输出
|
||||
StreamChan chan *StreamChunk `json:"-"` // 流式数据通道(不序列化)
|
||||
}
|
||||
|
||||
// NodeExecutionResult 节点执行结果
|
||||
type NodeExecutionResult struct {
|
||||
NodeID string `json:"node_id"`
|
||||
NodeName string `json:"node_name"`
|
||||
NodeType string `json:"node_type"`
|
||||
Status string `json:"status"` // success, failed, skipped
|
||||
Message string `json:"message"`
|
||||
StartedAt int64 `json:"started_at"`
|
||||
FinishedAt int64 `json:"finished_at"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Error string `json:"error,omitempty"`
|
||||
BranchIndex *int `json:"branch_index,omitempty"` // 条件节点的分支选择
|
||||
}
|
||||
|
||||
// 触发模式常量
|
||||
const (
|
||||
TriggerModeEvent = "event" // 告警事件触发
|
||||
TriggerModeAPI = "api" // API 触发
|
||||
TriggerModeCron = "cron" // 定时触发(后续支持)
|
||||
)
|
||||
|
||||
const (
|
||||
UseCaseEventPipeline = "event_pipeline"
|
||||
UseCaseEventSummary = "firemap"
|
||||
)
|
||||
|
||||
// WorkflowTriggerContext 工作流触发上下文
|
||||
type WorkflowTriggerContext struct {
|
||||
// 触发模式
|
||||
Mode string `json:"mode"`
|
||||
|
||||
// 触发者
|
||||
TriggerBy string `json:"trigger_by"`
|
||||
|
||||
// 请求ID(API/Cron 触发使用)
|
||||
RequestID string `json:"request_id"`
|
||||
|
||||
// 输入参数覆盖
|
||||
InputsOverrides map[string]string `json:"inputs_overrides"`
|
||||
|
||||
// 流式输出(API 调用时动态指定)
|
||||
Stream bool `json:"stream"`
|
||||
|
||||
// Cron 相关(后续使用)
|
||||
CronJobID string `json:"cron_job_id,omitempty"`
|
||||
CronExpr string `json:"cron_expr,omitempty"`
|
||||
ScheduledAt int64 `json:"scheduled_at,omitempty"`
|
||||
}
|
||||
|
||||
type WorkflowContext struct {
|
||||
Event *AlertCurEvent `json:"event"` // 当前事件
|
||||
Inputs map[string]string `json:"inputs"` // 前置输入参数(静态,用户配置)
|
||||
Vars map[string]interface{} `json:"vars"` // 节点间传递的数据(动态,运行时产生)
|
||||
Metadata map[string]string `json:"metadata"` // 执行元数据(request_id、start_time 等)
|
||||
Output map[string]interface{} `json:"output,omitempty"` // 输出结果(非告警场景使用)
|
||||
|
||||
// 流式输出支持
|
||||
Stream bool `json:"-"` // 是否启用流式输出(不序列化)
|
||||
StreamChan chan *StreamChunk `json:"-"` // 流式数据通道(不序列化)
|
||||
}
|
||||
|
||||
// StreamChunk 类型常量
|
||||
const (
|
||||
StreamTypeThinking = "thinking" // AI 思考过程(ReAct Thought)
|
||||
StreamTypeToolCall = "tool_call" // 工具调用
|
||||
StreamTypeToolResult = "tool_result" // 工具执行结果
|
||||
StreamTypeText = "text" // LLM 文本输出
|
||||
StreamTypeDone = "done" // 完成
|
||||
StreamTypeError = "error" // 错误
|
||||
)
|
||||
|
||||
// StreamChunk 流式数据块
|
||||
type StreamChunk struct {
|
||||
Type string `json:"type"` // thinking / tool_call / tool_result / text / done / error
|
||||
Content string `json:"content"` // 完整内容(累积)
|
||||
Delta string `json:"delta,omitempty"` // 增量内容
|
||||
NodeID string `json:"node_id,omitempty"` // 当前节点 ID
|
||||
RequestID string `json:"request_id,omitempty"` // 请求追踪 ID
|
||||
Metadata interface{} `json:"metadata,omitempty"` // 额外元数据(如工具调用参数)
|
||||
Done bool `json:"done"` // 是否结束
|
||||
Error string `json:"error,omitempty"` // 错误信息
|
||||
Timestamp int64 `json:"timestamp"` // 时间戳(毫秒)
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package cfg
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
type scanner struct {
|
||||
@@ -23,6 +23,6 @@ func (s *scanner) Data() []byte {
|
||||
|
||||
func (s *scanner) Read(file string) {
|
||||
if s.err == nil {
|
||||
s.data, s.err = ioutil.ReadFile(file)
|
||||
s.data, s.err = os.ReadFile(file)
|
||||
}
|
||||
}
|
||||
|
||||
348
pkg/feishu/feishu.go
Normal file
348
pkg/feishu/feishu.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package feishu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/storage"
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
|
||||
lark "github.com/larksuite/oapi-sdk-go/v3"
|
||||
larkcore "github.com/larksuite/oapi-sdk-go/v3/core"
|
||||
larkauthen "github.com/larksuite/oapi-sdk-go/v3/service/authen/v1"
|
||||
larkcontact "github.com/larksuite/oapi-sdk-go/v3/service/contact/v3"
|
||||
)
|
||||
|
||||
const defaultAuthURL = "https://accounts.feishu.cn/open-apis/authen/v1/authorize"
|
||||
const SsoTypeName = "feishu"
|
||||
|
||||
type SsoClient struct {
|
||||
Enable bool
|
||||
FeiShuConfig *Config `json:"-"`
|
||||
Ctx context.Context
|
||||
client *lark.Client
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Enable bool `json:"enable"`
|
||||
AuthURL string `json:"auth_url"`
|
||||
DisplayName string `json:"display_name"`
|
||||
AppID string `json:"app_id"`
|
||||
AppSecret string `json:"app_secret"`
|
||||
RedirectURL string `json:"redirect_url"`
|
||||
UsernameField string `json:"username_field"` // name, email, phone
|
||||
FeiShuEndpoint string `json:"feishu_endpoint"` // 飞书API端点,默认为 open.feishu.cn
|
||||
Proxy string `json:"proxy"`
|
||||
CoverAttributes bool `json:"cover_attributes"`
|
||||
DefaultRoles []string `json:"default_roles"`
|
||||
DefaultUserGroups []int64 `json:"default_user_groups"`
|
||||
}
|
||||
|
||||
type CallbackOutput struct {
|
||||
Redirect string `json:"redirect"`
|
||||
Msg string `json:"msg"`
|
||||
AccessToken string `json:"accessToken"`
|
||||
Username string `json:"Username"`
|
||||
Nickname string `json:"Nickname"`
|
||||
Phone string `yaml:"Phone"`
|
||||
Email string `yaml:"Email"`
|
||||
}
|
||||
|
||||
func wrapStateKey(key string) string {
|
||||
return "n9e_feishu_oauth_" + key
|
||||
}
|
||||
|
||||
// createClient 创建飞书SDK客户端(v3版本)
|
||||
func (c *Config) createClient() (*lark.Client, error) {
|
||||
opts := []lark.ClientOptionFunc{
|
||||
lark.WithLogLevel(larkcore.LogLevelInfo),
|
||||
lark.WithEnableTokenCache(true), // 启用token缓存
|
||||
}
|
||||
|
||||
if c.FeiShuEndpoint != "" {
|
||||
lark.FeishuBaseUrl = c.FeiShuEndpoint
|
||||
}
|
||||
|
||||
// 创建客户端(v3版本)
|
||||
client := lark.NewClient(
|
||||
c.AppID,
|
||||
c.AppSecret,
|
||||
opts...,
|
||||
)
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func New(cf Config) *SsoClient {
|
||||
var s = &SsoClient{}
|
||||
if !cf.Enable {
|
||||
return s
|
||||
}
|
||||
s.Reload(cf)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *SsoClient) AuthCodeURL(state string) (string, error) {
|
||||
var buf bytes.Buffer
|
||||
feishuAuthURL := defaultAuthURL
|
||||
if s.FeiShuConfig.AuthURL != "" {
|
||||
feishuAuthURL = s.FeiShuConfig.AuthURL
|
||||
}
|
||||
buf.WriteString(feishuAuthURL)
|
||||
v := url.Values{
|
||||
"app_id": {s.FeiShuConfig.AppID},
|
||||
"state": {state},
|
||||
}
|
||||
v.Set("redirect_uri", s.FeiShuConfig.RedirectURL)
|
||||
|
||||
if s.FeiShuConfig.RedirectURL == "" {
|
||||
return "", errors.New("FeiShu OAuth RedirectURL is empty")
|
||||
}
|
||||
|
||||
if strings.Contains(feishuAuthURL, "?") {
|
||||
buf.WriteByte('&')
|
||||
} else {
|
||||
buf.WriteByte('?')
|
||||
}
|
||||
buf.WriteString(v.Encode())
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
// GetUserToken 通过授权码获取用户access token和user_id(使用SDK v3)
|
||||
func (s *SsoClient) GetUserToken(code string) (string, string, error) {
|
||||
if s.client == nil {
|
||||
return "", "", errors.New("feishu client is not initialized")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 使用SDK v3的authen服务获取access token
|
||||
req := larkauthen.NewCreateAccessTokenReqBuilder().
|
||||
Body(larkauthen.NewCreateAccessTokenReqBodyBuilder().
|
||||
GrantType("authorization_code").
|
||||
Code(code).
|
||||
Build()).
|
||||
Build()
|
||||
|
||||
resp, err := s.client.Authen.AccessToken.Create(ctx, req)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("feishu get access token error: %w", err)
|
||||
}
|
||||
|
||||
// 检查响应
|
||||
if !resp.Success() {
|
||||
return "", "", fmt.Errorf("feishu api error: code=%d, msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
if resp.Data == nil {
|
||||
return "", "", errors.New("feishu api returned empty data")
|
||||
}
|
||||
|
||||
userID := ""
|
||||
if resp.Data.UserId != nil {
|
||||
userID = *resp.Data.UserId
|
||||
}
|
||||
if userID == "" {
|
||||
return "", "", errors.New("feishu api returned empty user_id")
|
||||
}
|
||||
|
||||
accessToken := ""
|
||||
if resp.Data.AccessToken != nil {
|
||||
accessToken = *resp.Data.AccessToken
|
||||
}
|
||||
if accessToken == "" {
|
||||
return "", "", errors.New("feishu api returned empty access_token")
|
||||
}
|
||||
|
||||
return accessToken, userID, nil
|
||||
}
|
||||
|
||||
// GetUserInfo 通过user_id获取用户详细信息(使用SDK v3)
|
||||
// 注意:SDK内部会自动管理token,所以不需要传入accessToken
|
||||
func (s *SsoClient) GetUserInfo(userID string) (*larkcontact.GetUserRespData, error) {
|
||||
if s.client == nil {
|
||||
return nil, errors.New("feishu client is not initialized")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 使用SDK v3的contact服务获取用户详情
|
||||
req := larkcontact.NewGetUserReqBuilder().
|
||||
UserId(userID).
|
||||
UserIdType(larkcontact.UserIdTypeUserId).
|
||||
Build()
|
||||
|
||||
resp, err := s.client.Contact.User.Get(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("feishu get user detail error: %w", err)
|
||||
}
|
||||
|
||||
// 检查响应
|
||||
if !resp.Success() {
|
||||
return nil, fmt.Errorf("feishu api error: code=%d, msg=%s", resp.Code, resp.Msg)
|
||||
}
|
||||
|
||||
if resp.Data == nil || resp.Data.User == nil {
|
||||
return nil, errors.New("feishu api returned empty user data")
|
||||
}
|
||||
|
||||
return resp.Data, nil
|
||||
}
|
||||
|
||||
func (s *SsoClient) Reload(feishuConfig Config) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
s.Enable = feishuConfig.Enable
|
||||
s.FeiShuConfig = &feishuConfig
|
||||
|
||||
// 重新创建客户端
|
||||
if feishuConfig.Enable && feishuConfig.AppID != "" && feishuConfig.AppSecret != "" {
|
||||
client, err := feishuConfig.createClient()
|
||||
if err != nil {
|
||||
logger.Errorf("create feishu client error: %v", err)
|
||||
} else {
|
||||
s.client = client
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SsoClient) GetDisplayName() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
if !s.Enable {
|
||||
return ""
|
||||
}
|
||||
|
||||
return s.FeiShuConfig.DisplayName
|
||||
}
|
||||
|
||||
func (s *SsoClient) Authorize(redis storage.Redis, redirect string) (string, error) {
|
||||
state := uuid.New().String()
|
||||
ctx := context.Background()
|
||||
|
||||
err := redis.Set(ctx, wrapStateKey(state), redirect, time.Duration(300*time.Second)).Err()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return s.AuthCodeURL(state)
|
||||
}
|
||||
|
||||
func (s *SsoClient) Callback(redis storage.Redis, ctx context.Context, code, state string) (*CallbackOutput, error) {
|
||||
// 通过code获取access token和user_id
|
||||
accessToken, userID, err := s.GetUserToken(code)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("feishu GetUserToken error: %s", err)
|
||||
}
|
||||
|
||||
// 获取用户详细信息
|
||||
userData, err := s.GetUserInfo(userID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("feishu GetUserInfo error: %s", err)
|
||||
}
|
||||
|
||||
// 获取redirect URL
|
||||
redirect := ""
|
||||
if redis != nil {
|
||||
redirect, err = fetchRedirect(redis, ctx, state)
|
||||
if err != nil {
|
||||
logger.Errorf("get redirect err:%v code:%s state:%s", err, code, state)
|
||||
}
|
||||
}
|
||||
if redirect == "" {
|
||||
redirect = "/"
|
||||
}
|
||||
|
||||
err = deleteRedirect(redis, ctx, state)
|
||||
if err != nil {
|
||||
logger.Errorf("delete redirect err:%v code:%s state:%s", err, code, state)
|
||||
}
|
||||
|
||||
var callbackOutput CallbackOutput
|
||||
if userData == nil || userData.User == nil {
|
||||
return nil, fmt.Errorf("feishu GetUserInfo failed, user data is nil")
|
||||
}
|
||||
|
||||
user := userData.User
|
||||
logger.Debugf("feishu get user info userID %s result %+v", userID, user)
|
||||
|
||||
// 提取用户信息
|
||||
username := ""
|
||||
if user.UserId != nil {
|
||||
username = *user.UserId
|
||||
}
|
||||
if username == "" {
|
||||
return nil, errors.New("feishu user_id is empty")
|
||||
}
|
||||
|
||||
nickname := ""
|
||||
if user.Name != nil {
|
||||
nickname = *user.Name
|
||||
}
|
||||
|
||||
phone := ""
|
||||
if user.Mobile != nil {
|
||||
phone = *user.Mobile
|
||||
}
|
||||
|
||||
email := ""
|
||||
if user.Email != nil {
|
||||
email = *user.Email
|
||||
}
|
||||
|
||||
if email == "" {
|
||||
if user.EnterpriseEmail != nil {
|
||||
email = *user.EnterpriseEmail
|
||||
}
|
||||
}
|
||||
|
||||
callbackOutput.Redirect = redirect
|
||||
callbackOutput.AccessToken = accessToken
|
||||
|
||||
// 根据UsernameField配置确定username
|
||||
switch s.FeiShuConfig.UsernameField {
|
||||
case "userid":
|
||||
callbackOutput.Username = username
|
||||
case "name":
|
||||
if nickname == "" {
|
||||
return nil, errors.New("feishu user name is empty")
|
||||
}
|
||||
callbackOutput.Username = nickname
|
||||
case "phone":
|
||||
if phone == "" {
|
||||
return nil, errors.New("feishu user phone is empty")
|
||||
}
|
||||
callbackOutput.Username = phone
|
||||
default:
|
||||
if email == "" {
|
||||
return nil, errors.New("feishu user email is empty")
|
||||
}
|
||||
callbackOutput.Username = email
|
||||
}
|
||||
|
||||
callbackOutput.Nickname = nickname
|
||||
callbackOutput.Email = email
|
||||
callbackOutput.Phone = phone
|
||||
|
||||
return &callbackOutput, nil
|
||||
}
|
||||
|
||||
func fetchRedirect(redis storage.Redis, ctx context.Context, state string) (string, error) {
|
||||
return redis.Get(ctx, wrapStateKey(state)).Result()
|
||||
}
|
||||
|
||||
func deleteRedirect(redis storage.Redis, ctx context.Context, state string) error {
|
||||
return redis.Del(ctx, wrapStateKey(state)).Err()
|
||||
}
|
||||
@@ -201,6 +201,11 @@ var I18N = `{
|
||||
"Some recovery scripts still in the BusiGroup": "业务组中仍有自愈脚本",
|
||||
"Some target busigroups still in the BusiGroup": "业务组中仍有监控对象",
|
||||
|
||||
"saved view not found": "保存的视图不存在",
|
||||
"saved view name is blank": "视图名称不能为空",
|
||||
"saved view page is blank": "视图页面不能为空",
|
||||
"saved view name already exists in this page": "该页面下已存在同名的公开视图",
|
||||
|
||||
"---------zh_CN--------": "---------zh_CN--------"
|
||||
},
|
||||
"zh_HK": {
|
||||
@@ -405,6 +410,11 @@ var I18N = `{
|
||||
"Some recovery scripts still in the BusiGroup": "業務組中仍有自愈腳本",
|
||||
"Some target busigroups still in the BusiGroup": "業務組中仍有監控對象",
|
||||
|
||||
"saved view not found": "保存的視圖不存在",
|
||||
"saved view name is blank": "視圖名稱不能為空",
|
||||
"saved view page is blank": "視圖頁面不能為空",
|
||||
"saved view name already exists in this page": "該頁面下已存在同名的公開視圖",
|
||||
|
||||
"---------zh_HK--------": "---------zh_HK--------"
|
||||
},
|
||||
"ja_JP": {
|
||||
@@ -606,6 +616,11 @@ var I18N = `{
|
||||
"Some recovery scripts still in the BusiGroup": "ビジネスグループにまだ自己回復スクリプトがあります",
|
||||
"Some target busigroups still in the BusiGroup": "ビジネスグループにまだ監視対象があります",
|
||||
|
||||
"saved view not found": "保存されたビューが見つかりません",
|
||||
"saved view name is blank": "ビュー名を空にすることはできません",
|
||||
"saved view page is blank": "ビューページを空にすることはできません",
|
||||
"saved view name already exists in this page": "このページには同名の公開ビューが既に存在します",
|
||||
|
||||
"---------ja_JP--------": "---------ja_JP--------"
|
||||
},
|
||||
"ru_RU": {
|
||||
@@ -807,6 +822,11 @@ var I18N = `{
|
||||
"Some recovery scripts still in the BusiGroup": "В бизнес-группе еще есть скрипты самоисцеления",
|
||||
"Some target busigroups still in the BusiGroup": "В бизнес-группе еще есть объекты мониторинга",
|
||||
|
||||
"saved view not found": "Сохраненный вид не найден",
|
||||
"saved view name is blank": "Название вида не может быть пустым",
|
||||
"saved view page is blank": "Страница вида не может быть пустой",
|
||||
"saved view name already exists in this page": "На этой странице уже существует публичный вид с таким названием",
|
||||
|
||||
"---------ru_RU--------": "---------ru_RU--------"
|
||||
}
|
||||
}`
|
||||
|
||||
@@ -1084,7 +1084,7 @@ type InitPostgresDatasource struct {
|
||||
Status string `gorm:"size:255;not null;default:''"`
|
||||
HTTP string `gorm:"size:4096;not null;default:''"`
|
||||
Auth string `gorm:"size:8192;not null;default:''"`
|
||||
IsDefault bool `gorm:"typr:boolean;not null;default:0"`
|
||||
IsDefault bool `gorm:"type:boolean;not null;default:0"`
|
||||
CreatedAt int64 `gorm:"not null;default:0"`
|
||||
CreatedBy string `gorm:"size:64;not null;default:''"`
|
||||
UpdatedAt int64 `gorm:"not null;default:0"`
|
||||
@@ -1494,10 +1494,6 @@ func sqliteDataBaseInit(db *gorm.DB) error {
|
||||
{RoleName: "Standard", Operation: "/alert-rules-built-in"},
|
||||
{RoleName: "Standard", Operation: "/dashboards-built-in"},
|
||||
{RoleName: "Standard", Operation: "/trace/dependencies"},
|
||||
{RoleName: "Admin", Operation: "/help/source"},
|
||||
{RoleName: "Admin", Operation: "/help/sso"},
|
||||
{RoleName: "Admin", Operation: "/help/notification-tpls"},
|
||||
{RoleName: "Admin", Operation: "/help/notification-settings"},
|
||||
{RoleName: "Standard", Operation: "/users"},
|
||||
{RoleName: "Standard", Operation: "/user-groups"},
|
||||
{RoleName: "Standard", Operation: "/user-groups/add"},
|
||||
@@ -1659,8 +1655,7 @@ func mysqlDataBaseInit(db *gorm.DB) error {
|
||||
for _, dt := range dts {
|
||||
err := db.AutoMigrate(dt)
|
||||
if err != nil {
|
||||
fmt.Printf("mysqlDataBaseInit AutoMigrate error: %v\n", err)
|
||||
return err
|
||||
logger.Errorf("mysqlDataBaseInit AutoMigrate error: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1668,7 +1663,7 @@ func mysqlDataBaseInit(db *gorm.DB) error {
|
||||
tableName := "task_host_" + strconv.Itoa(i)
|
||||
err := db.Table(tableName).AutoMigrate(&InitTaskHost{})
|
||||
if err != nil {
|
||||
return err
|
||||
logger.Errorf("mysqlDataBaseInit AutoMigrate task_host_%d error: %v\n", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1690,10 +1685,6 @@ func mysqlDataBaseInit(db *gorm.DB) error {
|
||||
{RoleName: "Standard", Operation: "/alert-rules-built-in"},
|
||||
{RoleName: "Standard", Operation: "/dashboards-built-in"},
|
||||
{RoleName: "Standard", Operation: "/trace/dependencies"},
|
||||
{RoleName: "Admin", Operation: "/help/source"},
|
||||
{RoleName: "Admin", Operation: "/help/sso"},
|
||||
{RoleName: "Admin", Operation: "/help/notification-tpls"},
|
||||
{RoleName: "Admin", Operation: "/help/notification-settings"},
|
||||
{RoleName: "Standard", Operation: "/users"},
|
||||
{RoleName: "Standard", Operation: "/user-groups"},
|
||||
{RoleName: "Standard", Operation: "/user-groups/add"},
|
||||
@@ -1886,10 +1877,6 @@ func postgresDataBaseInit(db *gorm.DB) error {
|
||||
{RoleName: "Standard", Operation: "/alert-rules-built-in"},
|
||||
{RoleName: "Standard", Operation: "/dashboards-built-in"},
|
||||
{RoleName: "Standard", Operation: "/trace/dependencies"},
|
||||
{RoleName: "Admin", Operation: "/help/source"},
|
||||
{RoleName: "Admin", Operation: "/help/sso"},
|
||||
{RoleName: "Admin", Operation: "/help/notification-tpls"},
|
||||
{RoleName: "Admin", Operation: "/help/notification-settings"},
|
||||
{RoleName: "Standard", Operation: "/users"},
|
||||
{RoleName: "Standard", Operation: "/user-groups"},
|
||||
{RoleName: "Standard", Operation: "/user-groups/add"},
|
||||
|
||||
@@ -348,6 +348,15 @@ func New(c DBConfig) (*gorm.DB, error) {
|
||||
return nil, fmt.Errorf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
// 检查 user 表是否存在,可能用户自己创建了空的数据库,如果不存在也执行 DataBaseInit
|
||||
if dbExist && !db.Migrator().HasTable("users") {
|
||||
fmt.Printf("Database exists but user table not found, initializing tables for %s\n", c.DBType)
|
||||
err = DataBaseInit(c, db)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to init database: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if c.Debug {
|
||||
db = db.Debug()
|
||||
}
|
||||
|
||||
@@ -531,8 +531,13 @@ func Printf(format string, value interface{}) string {
|
||||
|
||||
switch valType {
|
||||
case reflect.String:
|
||||
strValue := value.(string)
|
||||
// Check if it's a value with unit (contains both digits and non-numeric chars like letters or %)
|
||||
if isValueWithUnit(strValue) {
|
||||
return strValue
|
||||
}
|
||||
// Try converting string to float
|
||||
if floatValue, err := strconv.ParseFloat(value.(string), 64); err == nil {
|
||||
if floatValue, err := strconv.ParseFloat(strValue, 64); err == nil {
|
||||
return fmt.Sprintf(format, floatValue)
|
||||
}
|
||||
return fmt.Sprintf(format, value)
|
||||
@@ -544,6 +549,32 @@ func Printf(format string, value interface{}) string {
|
||||
}
|
||||
}
|
||||
|
||||
// isValueWithUnit checks if a string is a numeric value with unit
|
||||
// e.g., "11.5%", "100MB", "10a" returns true
|
||||
// e.g., "11", "11.11", "-3.14" returns false
|
||||
func isValueWithUnit(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
hasDigit := false
|
||||
hasUnit := false
|
||||
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
hasDigit = true
|
||||
} else if r == '.' || r == '-' || r == '+' {
|
||||
// These are valid numeric characters, not units
|
||||
continue
|
||||
} else {
|
||||
// Any other character (letters, %, etc.) is considered a unit
|
||||
hasUnit = true
|
||||
}
|
||||
}
|
||||
|
||||
return hasDigit && hasUnit
|
||||
}
|
||||
|
||||
func floatToTime(v float64) (*time.Time, error) {
|
||||
if math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return nil, errNaNOrInf
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
templateT "text/template"
|
||||
|
||||
"encoding/base64"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
)
|
||||
|
||||
@@ -64,6 +64,16 @@ var TemplateFuncMap = template.FuncMap{
|
||||
"jsonMarshal": JsonMarshal,
|
||||
"mapDifference": MapDifference,
|
||||
"tagsMapToStr": TagsMapToStr,
|
||||
"b64enc": func(s string) string {
|
||||
return base64.StdEncoding.EncodeToString([]byte(s))
|
||||
},
|
||||
"b64dec": func(s string) string {
|
||||
data, err := base64.StdEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
return s
|
||||
}
|
||||
return string(data)
|
||||
},
|
||||
}
|
||||
|
||||
// NewTemplateFuncMap copy on write for TemplateFuncMap
|
||||
|
||||
@@ -3,7 +3,7 @@ package prom
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/pkg/tlsx"
|
||||
"github.com/ccfos/nightingale/v6/models"
|
||||
)
|
||||
|
||||
type PromOption struct {
|
||||
@@ -20,7 +20,8 @@ type PromOption struct {
|
||||
|
||||
Headers []string
|
||||
|
||||
tlsx.ClientConfig
|
||||
// TLS 配置(支持 mTLS)
|
||||
TLS models.TLS
|
||||
}
|
||||
|
||||
func (po *PromOption) Equal(target PromOption) bool {
|
||||
@@ -52,10 +53,6 @@ func (po *PromOption) Equal(target PromOption) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
if po.InsecureSkipVerify != target.InsecureSkipVerify {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(po.Headers) != len(target.Headers) {
|
||||
return false
|
||||
}
|
||||
@@ -66,6 +63,29 @@ func (po *PromOption) Equal(target PromOption) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// 比较 TLS 配置
|
||||
if po.TLS.SkipTlsVerify != target.TLS.SkipTlsVerify {
|
||||
return false
|
||||
}
|
||||
if po.TLS.CACert != target.TLS.CACert {
|
||||
return false
|
||||
}
|
||||
if po.TLS.ClientCert != target.TLS.ClientCert {
|
||||
return false
|
||||
}
|
||||
if po.TLS.ClientKey != target.TLS.ClientKey {
|
||||
return false
|
||||
}
|
||||
if po.TLS.ServerName != target.TLS.ServerName {
|
||||
return false
|
||||
}
|
||||
if po.TLS.MinVersion != target.TLS.MinVersion {
|
||||
return false
|
||||
}
|
||||
if po.TLS.MaxVersion != target.TLS.MaxVersion {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -101,11 +101,7 @@ func (pc *PromClientMap) loadFromDatabase() {
|
||||
DialTimeout: ds.HTTPJson.DialTimeout,
|
||||
MaxIdleConnsPerHost: ds.HTTPJson.MaxIdleConnsPerHost,
|
||||
Headers: header,
|
||||
}
|
||||
|
||||
if strings.HasPrefix(ds.HTTPJson.Url, "https") {
|
||||
po.UseTLS = true
|
||||
po.InsecureSkipVerify = ds.HTTPJson.TLS.SkipTlsVerify
|
||||
TLS: ds.HTTPJson.TLS,
|
||||
}
|
||||
|
||||
if internalAddr != "" && !pc.ctx.IsCenter {
|
||||
@@ -149,7 +145,10 @@ func (pc *PromClientMap) loadFromDatabase() {
|
||||
}
|
||||
|
||||
func (pc *PromClientMap) newReaderClientFromPromOption(po PromOption) (api.Client, error) {
|
||||
tlsConfig, _ := po.TLSConfig()
|
||||
tlsConfig, err := po.TLS.TLSConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create TLS config: %v", err)
|
||||
}
|
||||
|
||||
return api.NewClient(api.Config{
|
||||
Address: po.Url,
|
||||
@@ -166,7 +165,10 @@ func (pc *PromClientMap) newReaderClientFromPromOption(po PromOption) (api.Clien
|
||||
}
|
||||
|
||||
func (pc *PromClientMap) newWriterClientFromPromOption(po PromOption) (api.Client, error) {
|
||||
tlsConfig, _ := po.TLSConfig()
|
||||
tlsConfig, err := po.TLS.TLSConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create TLS config: %v", err)
|
||||
}
|
||||
|
||||
return api.NewClient(api.Config{
|
||||
Address: po.WriteAddr,
|
||||
|
||||
@@ -106,6 +106,7 @@ func (s *Set) UpdateTargets(lst []string, now int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 心跳时间只写入 Redis,不再写入 MySQL update_at
|
||||
err := s.updateTargetsUpdateTs(lst, now, s.redis)
|
||||
if err != nil {
|
||||
logger.Errorf("update_ts: failed to update targets: %v error: %v", lst, err)
|
||||
@@ -133,12 +134,7 @@ func (s *Set) UpdateTargets(lst []string, now int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.configs.UpdateDBTargetTimestampDisable {
|
||||
// 如果 mysql 压力太大,关闭更新 db 的操作
|
||||
return nil
|
||||
}
|
||||
|
||||
// there are some idents not found in db, so insert them
|
||||
// 新 target 仍需 INSERT 注册到 MySQL
|
||||
var exists []string
|
||||
err = s.ctx.DB.Table("target").Where("ident in ?", lst).Pluck("ident", &exists).Error
|
||||
if err != nil {
|
||||
@@ -153,38 +149,13 @@ func (s *Set) UpdateTargets(lst []string, now int64) error {
|
||||
}
|
||||
}
|
||||
|
||||
// 从批量更新一批机器的时间戳,改成逐台更新,是为了避免批量更新时,mysql的锁竞争问题
|
||||
start := time.Now()
|
||||
duration := time.Since(start).Seconds()
|
||||
if len(exists) > 0 {
|
||||
sema := semaphore.NewSemaphore(s.configs.UpdateDBTargetConcurrency)
|
||||
wg := sync.WaitGroup{}
|
||||
for i := 0; i < len(exists); i++ {
|
||||
sema.Acquire()
|
||||
wg.Add(1)
|
||||
go func(ident string) {
|
||||
defer sema.Release()
|
||||
defer wg.Done()
|
||||
s.updateDBTargetTs(ident, now)
|
||||
}(exists[i])
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
pstat.DBOperationLatency.WithLabelValues("update_targets_ts").Observe(duration)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Set) updateDBTargetTs(ident string, now int64) {
|
||||
err := s.ctx.DB.Exec("UPDATE target SET update_at = ? WHERE ident = ?", now, ident).Error
|
||||
if err != nil {
|
||||
logger.Error("update_target: failed to update target:", ident, "error:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Set) updateTargetsUpdateTs(lst []string, now int64, redis storage.Redis) error {
|
||||
if redis == nil {
|
||||
return fmt.Errorf("redis is nil")
|
||||
logger.Debugf("update_ts: redis is nil")
|
||||
return nil
|
||||
}
|
||||
|
||||
newMap := make(map[string]interface{}, len(lst))
|
||||
@@ -247,7 +218,7 @@ func (s *Set) writeTargetTsInRedis(ctx context.Context, redis storage.Redis, con
|
||||
|
||||
for i := 0; i < retryCount; i++ {
|
||||
start := time.Now()
|
||||
err := storage.MSet(ctx, redis, content)
|
||||
err := storage.MSet(ctx, redis, content, 24*time.Hour)
|
||||
duration := time.Since(start).Seconds()
|
||||
|
||||
logger.Debugf("update_ts: write target ts in redis, keys: %v, retryCount: %d, retryInterval: %v, error: %v", keys, retryCount, retryInterval, err)
|
||||
|
||||
@@ -18,8 +18,6 @@ type Pushgw struct {
|
||||
UpdateTargetRetryIntervalMills int64
|
||||
UpdateTargetTimeoutMills int64
|
||||
UpdateTargetBatchSize int
|
||||
UpdateDBTargetConcurrency int
|
||||
UpdateDBTargetTimestampDisable bool
|
||||
PushConcurrency int
|
||||
UpdateTargetByUrlConcurrency int
|
||||
|
||||
@@ -129,10 +127,6 @@ func (p *Pushgw) PreCheck() {
|
||||
p.UpdateTargetBatchSize = 20
|
||||
}
|
||||
|
||||
if p.UpdateDBTargetConcurrency <= 0 {
|
||||
p.UpdateDBTargetConcurrency = 16
|
||||
}
|
||||
|
||||
if p.PushConcurrency <= 0 {
|
||||
p.PushConcurrency = 16
|
||||
}
|
||||
|
||||
@@ -109,21 +109,30 @@ func (rt *Router) debugSample(remoteAddr string, v *prompb.TimeSeries) {
|
||||
}
|
||||
|
||||
func (rt *Router) DropSample(v *prompb.TimeSeries) bool {
|
||||
filters := rt.Pushgw.DropSample
|
||||
if len(filters) == 0 {
|
||||
// 快速路径:检查仅 __name__ 的过滤器 O(1)
|
||||
if len(rt.dropByNameOnly) > 0 {
|
||||
for i := 0; i < len(v.Labels); i++ {
|
||||
if v.Labels[i].Name == "__name__" {
|
||||
if _, ok := rt.dropByNameOnly[v.Labels[i].Value]; ok {
|
||||
return true
|
||||
}
|
||||
break // __name__ 只会出现一次,找到后直接跳出
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 慢速路径:处理复杂的多条件过滤器
|
||||
if len(rt.dropComplex) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
labelMap := make(map[string]string)
|
||||
// 只有复杂过滤器存在时才创建 labelMap
|
||||
labelMap := make(map[string]string, len(v.Labels))
|
||||
for i := 0; i < len(v.Labels); i++ {
|
||||
labelMap[v.Labels[i].Name] = v.Labels[i].Value
|
||||
}
|
||||
|
||||
for _, filter := range filters {
|
||||
if len(filter) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, filter := range rt.dropComplex {
|
||||
if matchSample(filter, labelMap) {
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/toolkits/pkg/logger"
|
||||
|
||||
"github.com/ccfos/nightingale/v6/alert/aconf"
|
||||
"github.com/ccfos/nightingale/v6/center/metas"
|
||||
@@ -32,7 +33,11 @@ type Router struct {
|
||||
Writers *writer.WritersType
|
||||
Ctx *ctx.Context
|
||||
HandleTS HandleTSFunc
|
||||
HeartbeatApi string
|
||||
HeartbeatApi string
|
||||
|
||||
// 预编译的 DropSample 过滤器
|
||||
dropByNameOnly map[string]struct{} // 仅 __name__ 条件的快速匹配
|
||||
dropComplex []map[string]string // 多条件的复杂匹配
|
||||
}
|
||||
|
||||
func stat() gin.HandlerFunc {
|
||||
@@ -51,7 +56,7 @@ func stat() gin.HandlerFunc {
|
||||
func New(httpConfig httpx.Config, pushgw pconf.Pushgw, aconf aconf.Alert, tc *memsto.TargetCacheType, bg *memsto.BusiGroupCacheType,
|
||||
idents *idents.Set, metas *metas.Set,
|
||||
writers *writer.WritersType, ctx *ctx.Context) *Router {
|
||||
return &Router{
|
||||
rt := &Router{
|
||||
HTTP: httpConfig,
|
||||
Pushgw: pushgw,
|
||||
Aconf: aconf,
|
||||
@@ -63,6 +68,38 @@ func New(httpConfig httpx.Config, pushgw pconf.Pushgw, aconf aconf.Alert, tc *me
|
||||
MetaSet: metas,
|
||||
HandleTS: func(pt *prompb.TimeSeries) *prompb.TimeSeries { return pt },
|
||||
}
|
||||
|
||||
// 预编译 DropSample 过滤器
|
||||
rt.initDropSampleFilters()
|
||||
|
||||
return rt
|
||||
}
|
||||
|
||||
// initDropSampleFilters 预编译 DropSample 过滤器,将单条件 __name__ 过滤器
|
||||
// 放入 map 实现 O(1) 查找,多条件过滤器保留原有逻辑
|
||||
func (rt *Router) initDropSampleFilters() {
|
||||
rt.dropByNameOnly = make(map[string]struct{})
|
||||
rt.dropComplex = make([]map[string]string, 0)
|
||||
|
||||
for _, filter := range rt.Pushgw.DropSample {
|
||||
if len(filter) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 如果只有一个条件且是 __name__,放入快速匹配 map
|
||||
if len(filter) == 1 {
|
||||
if name, ok := filter["__name__"]; ok {
|
||||
rt.dropByNameOnly[name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 其他情况放入复杂匹配列表
|
||||
rt.dropComplex = append(rt.dropComplex, filter)
|
||||
}
|
||||
|
||||
logger.Infof("DropSample filters initialized: %d name-only, %d complex",
|
||||
len(rt.dropByNameOnly), len(rt.dropComplex))
|
||||
}
|
||||
|
||||
func (rt *Router) Config(r *gin.Engine) {
|
||||
|
||||
@@ -163,10 +163,10 @@ func MGet(ctx context.Context, r Redis, keys []string) [][]byte {
|
||||
return vals
|
||||
}
|
||||
|
||||
func MSet(ctx context.Context, r Redis, m map[string]interface{}) error {
|
||||
func MSet(ctx context.Context, r Redis, m map[string]interface{}, expiration time.Duration) error {
|
||||
pipe := r.Pipeline()
|
||||
for k, v := range m {
|
||||
pipe.Set(ctx, k, v, 0)
|
||||
pipe.Set(ctx, k, v, expiration)
|
||||
}
|
||||
_, err := pipe.Exec(ctx)
|
||||
return err
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestMiniRedisMGet(t *testing.T) {
|
||||
mp["key2"] = "value2"
|
||||
mp["key3"] = "value3"
|
||||
|
||||
err = MSet(context.Background(), rdb, mp)
|
||||
err = MSet(context.Background(), rdb, mp, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set miniredis value: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user