feat: introduce SLA alarm filtering based on minimum success rate and refactor…

feat: introduce SLA alarm filtering based on minimum success rate and refactor alarm message handling

alarm-server/src/main/java/com/ebaiyihui/alarm/server/config/SkyWalkingProperties.java

@@ -40,4 +40,9 @@ public class SkyWalkingProperties {
      * 为 true 时，纯响应时间告警且查不到错误 Trace 则不发送飞书通知
      */
     private boolean skipRespTimeWithoutError = false;
+
+    /**
+     * SLA 告警时，若最近窗口内最低成功率仍高于该值(%)，则不推送。0 表示关闭过滤。
+     */
+    private double skipSlaWhenMinRateAbove = 95;
 }

alarm-server/src/main/java/com/ebaiyihui/alarm/server/service/Impl/AlarmServiceImpl.java

@@ -3,6 +3,7 @@ package com.ebaiyihui.alarm.server.service.Impl;
 import com.ebaiyihui.alarm.server.config.ProjProperties;
 import com.ebaiyihui.alarm.server.config.ServiceInfo;
 import com.ebaiyihui.alarm.server.config.SkyWalkingProperties;
+import com.ebaiyihui.alarm.server.entity.ServiceMetricsStats;
 import com.ebaiyihui.alarm.server.entity.AlarmMessage;
 import com.ebaiyihui.alarm.server.entity.Tag;
 import com.ebaiyihui.alarm.server.entity.TraceErrorDetail;
@@ -69,7 +70,7 @@ public class AlarmServiceImpl implements AlarmService {
             Pattern.CASE_INSENSITIVE
     );
     private static final Pattern ENDPOINT_SLA_PATTERN = Pattern.compile(
-            "Successful rate of endpoint (.+) in service (.+) is lower than (.+) in (.+) of last (.+)",
+            "Successful rate of endpoint (.+) in (.+) is lower than (.+) in (.+) of last (.+)",
             Pattern.CASE_INSENSITIVE
     );
     private static final Pattern ENDPOINT_RELATION_RESPONSE_TIME_PATTERN = Pattern.compile(
@@ -118,8 +119,7 @@ public class AlarmServiceImpl implements AlarmService {
                     continue;
                 }
                 alarmMessage.setAlarmMessage(notifyMessage);
-                String sendMessage = "{\"msg_type\":\"text\",\"content\":{\"text\":\"%s\"}}";
-                String requestBody = getRequestBody(sendMessage, alarmMessage);
+                String requestBody = getRequestBody(alarmMessage);
                 HttpKit.jsonPost(webHookUrl, requestBody);
             } catch (Exception e) {
                 hasError = true;
@@ -156,7 +156,16 @@ public class AlarmServiceImpl implements AlarmService {
         String serviceName = resolveServiceName(alarmMessage, rawMessage);
         String endpointPath = resolveEndpointPath(alarmMessage, rawMessage);
         String message = formatReadableMessage(alarmMessage, serviceName, rawMessage);
-        message = appendMetricsStats(message, serviceName, endpointPath);
+        ServiceMetricsStats metricsStats = skyWalkingProperties.isEnabled()
+                ? skyWalkingQueryService.queryServiceStats(serviceName, endpointPath)
+                : null;
+        if (shouldSkipSlaAlarm(alarmMessage, metricsStats)) {
+            log.info("Skip SLA alarm because min success rate is above threshold, service={}, endpoint={}, minRate={}%",
+                    serviceName, endpointPath,
+                    metricsStats != null ? metricsStats.getMinSuccessRatePercent() : null);
+            return "";
+        }
+        message = appendMetricsStats(message, metricsStats);
         message = appendErrorDetails(message, alarmMessage, serviceName, endpointPath);
         if (StringUtil.isBlank(message)) {
             return "";
@@ -320,18 +329,36 @@ public class AlarmServiceImpl implements AlarmService {
         return false;
     }
 
-    private String appendMetricsStats(String message, String serviceName, String endpointPath) {
-        if (!skyWalkingProperties.isEnabled()) {
+    private String appendMetricsStats(String message, ServiceMetricsStats metricsStats) {
+        if (metricsStats == null) {
             return message;
         }
-        String metricsSection = skyWalkingQueryService.formatMetricsSection(
-                skyWalkingQueryService.queryServiceStats(serviceName, endpointPath));
+        String metricsSection = skyWalkingQueryService.formatMetricsSection(metricsStats);
         if (StringUtil.isBlank(metricsSection)) {
             return message;
         }
         return message + "\n\n" + metricsSection;
     }
 
+    private boolean shouldSkipSlaAlarm(AlarmMessage alarmMessage, ServiceMetricsStats metricsStats) {
+        if (metricsStats == null) {
+            return false;
+        }
+        double threshold = skyWalkingProperties.getSkipSlaWhenMinRateAbove();
+        if (threshold <= 0 || !isSlaRule(alarmMessage.getRuleName())) {
+            return false;
+        }
+        return metricsStats.getMinSuccessRatePercent() >= threshold;
+    }
+
+    private boolean isSlaRule(String ruleName) {
+        if (StringUtil.isBlank(ruleName)) {
+            return false;
+        }
+        String key = ruleName.trim().toLowerCase(Locale.ROOT);
+        return "service_sla_rule".equals(key) || "endpoint_sla_rule".equals(key);
+    }
+
     private String appendErrorDetails(String message, AlarmMessage alarmMessage, String serviceName, String endpointPath) {
         if (!skyWalkingProperties.isEnabled()) {
             return message;
@@ -519,12 +546,13 @@ public class AlarmServiceImpl implements AlarmService {
     /**
      * deal requestBody,if it has sign set the sign
      */
-    private String getRequestBody(String sendMessage, AlarmMessage alarmMessage) {
-        String escapedMessage = alarmMessage.getAlarmMessage().replace("%", "%%");
-        String requestBody = String.format(sendMessage, escapedMessage);
-        Gson gson = new Gson();
-        JsonObject jsonObject = gson.fromJson(requestBody, JsonObject.class);
-        Map<String, Object> content = buildContent(jsonObject);
+    private String getRequestBody(AlarmMessage alarmMessage) {
+        JsonObject textContent = new JsonObject();
+        textContent.addProperty("text", alarmMessage.getAlarmMessage());
+        JsonObject requestJson = new JsonObject();
+        requestJson.addProperty("msg_type", "text");
+        requestJson.add("content", textContent);
+        Map<String, Object> content = buildContent(requestJson);
         if (!StringUtil.isBlank(secret)) {
             Long timestamp = System.currentTimeMillis() / 1000;
             content.put("timestamp", timestamp);
@@ -534,7 +562,7 @@ public class AlarmServiceImpl implements AlarmService {
                 throw new RuntimeException(e);
             }
         }
-        return gson.toJson(content);
+        return new Gson().toJson(content);
     }
 
     /**

alarm-server/src/main/resources/application.yml

@@ -81,6 +81,8 @@ skywalking:
   max-stack-chars: 3000
   # 设为 true 时，纯响应时间告警且查不到错误 Trace 则不推送飞书
   skip-resp-time-without-error: true
+  # SLA 告警时，若最近窗口最低成功率仍 >= 该值(%)，则不推飞书；0 表示不过滤
+  skip-sla-when-min-rate-above: 90
 
 #   ou_c6455365320227160724033bc454c666 王猛
 #   ou_6cb2f35f9bf8e7d7e977483801ab13de 杨凯