add global webhook

Made-with: Cursor

alert/aconf/conf.go

@@ -33,6 +33,17 @@ type Alerting struct {
 	TemplatesDir      string
 	NotifyConcurrency int
 	WebhookBatchSend  bool
+	GlobalWebhook     GlobalWebhook
+}
+
+type GlobalWebhook struct {
+	Enable        bool
+	Url           string
+	BasicAuthUser string
+	BasicAuthPass string
+	Timeout       int
+	Headers       []string
+	SkipVerify    bool
 }
 
 type CallPlugin struct {

alert/alert.go

@@ -117,6 +117,8 @@ func Start(alertc aconf.Alert, pushgwc pconf.Pushgw, syncStats *memsto.Stats, al
 
 	eventProcessorCache := memsto.NewEventProcessorCache(ctx, syncStats)
 
+	sender.InitGlobalWebhook(alertc.Alerting.GlobalWebhook)
+
 	dp := dispatch.NewDispatch(alertRuleCache, userCache, userGroupCache, alertSubscribeCache, targetCache, notifyConfigCache, taskTplsCache, notifyRuleCache, notifyChannelCache, messageTemplateCache, eventProcessorCache, configCvalCache, alertc.Alerting, ctx, alertStats)
 	consumer := dispatch.NewConsumer(alertc.Alerting, ctx, dp, promClients, alertMuteCache)
 

alert/dispatch/dispatch.go

@@ -604,6 +604,7 @@ func NeedBatchContacts(requestConfig *models.HTTPRequestConfig) bool {
 // isSubscribe: 告警事件是否由subscribe的配置产生
 func (e *Dispatch) HandleEventNotify(event *models.AlertCurEvent, isSubscribe bool) {
 	go e.HandleEventWithNotifyRule(event)
+	go sender.SendGlobalWebhook(event.DeepCopy(), e.Astats)
 	if event.IsRecovered && event.NotifyRecovered == 0 {
 		return
 	}

alert/sender/global_webhook.go

@@ -0,0 +1,106 @@
+package sender
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/ccfos/nightingale/v6/alert/aconf"
+	"github.com/ccfos/nightingale/v6/alert/astats"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/poster"
+
+	"github.com/toolkits/pkg/logger"
+)
+
+var globalWebhookClient *http.Client
+var globalWebhookConf aconf.GlobalWebhook
+
+func InitGlobalWebhook(conf aconf.GlobalWebhook) {
+	globalWebhookConf = conf
+	if !conf.Enable || conf.Url == "" {
+		return
+	}
+
+	if len(conf.Headers) > 0 && len(conf.Headers)%2 != 0 {
+		logger.Warningf("global_webhook headers count is odd(%d), headers will be ignored", len(conf.Headers))
+	}
+
+	timeout := conf.Timeout
+	if timeout <= 0 {
+		timeout = 10
+	}
+
+	transport := &http.Transport{
+		TLSClientConfig:     &tls.Config{InsecureSkipVerify: conf.SkipVerify},
+		MaxIdleConns:        100,
+		MaxIdleConnsPerHost: 10,
+		IdleConnTimeout:     90 * time.Second,
+	}
+
+	if poster.UseProxy(conf.Url) {
+		transport.Proxy = http.ProxyFromEnvironment
+	}
+
+	globalWebhookClient = &http.Client{
+		Timeout:   time.Duration(timeout) * time.Second,
+		Transport: transport,
+	}
+
+	logger.Infof("global_webhook initialized, url:%s", conf.Url)
+}
+
+func SendGlobalWebhook(event *models.AlertCurEvent, stats *astats.Stats) {
+	if globalWebhookClient == nil {
+		return
+	}
+
+	bs, err := json.Marshal(event)
+	if err != nil {
+		logger.Errorf("global_webhook failed to marshal event err:%v", err)
+		return
+	}
+
+	req, err := http.NewRequest("POST", globalWebhookConf.Url, bytes.NewBuffer(bs))
+	if err != nil {
+		logger.Warningf("global_webhook failed to new request event:%s err:%v", string(bs), err)
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	if globalWebhookConf.BasicAuthUser != "" && globalWebhookConf.BasicAuthPass != "" {
+		req.SetBasicAuth(globalWebhookConf.BasicAuthUser, globalWebhookConf.BasicAuthPass)
+	}
+
+	if len(globalWebhookConf.Headers) > 0 && len(globalWebhookConf.Headers)%2 == 0 {
+		for i := 0; i < len(globalWebhookConf.Headers); i += 2 {
+			if globalWebhookConf.Headers[i] == "Host" || globalWebhookConf.Headers[i] == "host" {
+				req.Host = globalWebhookConf.Headers[i+1]
+				continue
+			}
+			req.Header.Set(globalWebhookConf.Headers[i], globalWebhookConf.Headers[i+1])
+		}
+	}
+
+	stats.AlertNotifyTotal.WithLabelValues("global_webhook").Inc()
+	resp, err := globalWebhookClient.Do(req)
+	if err != nil {
+		stats.AlertNotifyErrorTotal.WithLabelValues("global_webhook").Inc()
+		logger.Errorf("global_webhook_fail url:%s event:%s error:%v", globalWebhookConf.Url, event.Hash, err)
+		return
+	}
+
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+
+	if resp.StatusCode >= 400 {
+		stats.AlertNotifyErrorTotal.WithLabelValues("global_webhook").Inc()
+		logger.Errorf("global_webhook_fail url:%s status:%d body:%s event:%s", globalWebhookConf.Url, resp.StatusCode, string(body), event.Hash)
+		return
+	}
+
+	logger.Debugf("global_webhook_succ url:%s status:%d body:%s event:%s", globalWebhookConf.Url, resp.StatusCode, string(body), event.Hash)
+}

center/router/router_alert_rule.go

@@ -13,10 +13,10 @@ import (
 
 	"github.com/ccfos/nightingale/v6/alert/mute"
 	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ginx"
 	"github.com/ccfos/nightingale/v6/pkg/strx"
 	"github.com/ccfos/nightingale/v6/pushgw/pconf"
 	"github.com/ccfos/nightingale/v6/pushgw/writer"
-	"github.com/ccfos/nightingale/v6/pkg/ginx"
 
 	"github.com/gin-gonic/gin"
 	"github.com/jinzhu/copier"
@@ -886,6 +886,7 @@ func (rt *Router) batchAlertRuleClone(c *gin.Context) {
 func (rt *Router) timezonesGet(c *gin.Context) {
 	// 返回常用时区列表（按时差去重，每个时差只保留一个代表性时区）
 	timezones := []string{
+		"Local",
 		"UTC",
 		"Asia/Shanghai",       // UTC+8 (代表 Asia/Hong_Kong, Asia/Singapore 等)
 		"Asia/Tokyo",          // UTC+9 (代表 Asia/Seoul 等)

dskit/doris/doris.go

@@ -201,7 +201,7 @@ func (d *Doris) NewWriteConn(ctx context.Context, database string) (*sql.DB, err
 func (d *Doris) createTimeoutContext(ctx context.Context) (context.Context, context.CancelFunc) {
 	timeout := d.Timeout
 	if timeout == 0 {
-		timeout = 60
+		timeout = 60000
 	}
 	return context.WithTimeout(ctx, time.Duration(timeout)*time.Millisecond)
 }