Skip to content

B
byh-alarm-service
提交 df71b912 authored 作者: Edwin's avatar Edwin
浏览文件
操作

feat: 增强告警服务，添加SkyWalking集成与响应时间规则处理

上级 ed581b73
隐藏空白字符变更
内嵌 并排
正在显示 包含 531 行增加 和 1 行删除
alarm-server/src/main/java/com/ebaiyihui/alarm/server/config/SkyWalkingProperties.java 0 → 100644
浏览文件 @ df71b912
package com.ebaiyihui.alarm.server.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "skywalking")
public class SkyWalkingProperties {
private boolean enabled = true;
/**
* SkyWalking GraphQL 地址，Docker 内网可用 http://skywalking-ui:8080/graphql
*/
private String graphqlUrl = "http://skywalking-ui:8080/graphql";
/**
* 回溯查询最近 N 分钟内的错误 Trace
*/
private int lookbackMinutes = 10;
/**
* 每条告警最多附带几条错误 Trace
*/
private int maxErrorTraces = 3;
/**
* 堆栈最多展示行数
*/
private int maxStackLines = 5;
/**
* 为 true 时，纯响应时间告警且查不到错误 Trace 则不发送飞书通知
*/
private boolean skipRespTimeWithoutError = false;
}
alarm-server/src/main/java/com/ebaiyihui/alarm/server/entity/TraceErrorDetail.java 0 → 100644
浏览文件 @ df71b912
package com.ebaiyihui.alarm.server.entity;
import lombok.Getter;
@Getter
public class TraceErrorDetail {
private final String traceId;
private final String endpoint;
private final String service;
private final String errorKind;
private final String message;
private final String stack;
public TraceErrorDetail(String traceId, String endpoint, String service,
String errorKind, String message, String stack) {
this.traceId = traceId;
this.endpoint = endpoint;
this.service = service;
this.errorKind = errorKind;
this.message = message;
this.stack = stack;
}
}
alarm-server/src/main/java/com/ebaiyihui/alarm/server/service/Impl/AlarmServiceImpl.java
浏览文件 @ df71b912
......@@ -2,9 +2,12 @@ package com.ebaiyihui.alarm.server.service.Impl;
import com.ebaiyihui.alarm.server.config.ProjProperties;
import com.ebaiyihui.alarm.server.config.ServiceInfo;
import com.ebaiyihui.alarm.server.config.SkyWalkingProperties;
import com.ebaiyihui.alarm.server.entity.AlarmMessage;
import com.ebaiyihui.alarm.server.entity.Tag;
import com.ebaiyihui.alarm.server.entity.TraceErrorDetail;
import com.ebaiyihui.alarm.server.service.AlarmService;
import com.ebaiyihui.alarm.server.service.SkyWalkingQueryService;
import com.ebaiyihui.alarm.server.util.HttpKit;
import com.ebaiyihui.framework.utils.StringUtil;
import com.google.gson.Gson;
......@@ -61,6 +64,27 @@ public class AlarmServiceImpl implements AlarmService {
"Service instance (.+) of (.+) is down in (.+) of last (.+)",
Pattern.CASE_INSENSITIVE
);
private static final Pattern SERVICE_SLA_PATTERN = Pattern.compile(
"Successful rate of service (.+) is lower than (.+) in (.+) of last (.+)",
Pattern.CASE_INSENSITIVE
);
private static final Pattern ENDPOINT_SLA_PATTERN = Pattern.compile(
"Successful rate of endpoint (.+) in service (.+) is lower than (.+) in (.+) of last (.+)",
Pattern.CASE_INSENSITIVE
);
private static final Pattern ENDPOINT_RELATION_RESPONSE_TIME_PATTERN = Pattern.compile(
"Response time of endpoint relation .+ to (.+) in (.+) is more than (.+) in (.+) of last (.+)",
Pattern.CASE_INSENSITIVE
);
private static final Set<String> RESPONSE_TIME_RULES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
"service_resp_time_rule",
"service_instance_resp_time_rule",
"endpoint_resp_time_rule",
"endpoint_relation_resp_time_rule",
"service_resp_time_percentile_rule",
"service_percentile_rule",
"database_access_resp_time_rule"
)));
private static final Map<String, String> RULE_NAME_TRANSLATIONS = buildRuleNameTranslations();
@Value("${webHookUrl}")
......@@ -72,6 +96,12 @@ public class AlarmServiceImpl implements AlarmService {
@Autowired
private ProjProperties projProperties;
@Autowired
private SkyWalkingProperties skyWalkingProperties;
@Autowired
private SkyWalkingQueryService skyWalkingQueryService;
@Override
public String receive(List<AlarmMessage> alarmMessages) {
if (alarmMessages == null || alarmMessages.isEmpty()) {
......@@ -84,6 +114,9 @@ public class AlarmServiceImpl implements AlarmService {
continue;
}
String notifyMessage = buildNotifyMessage(alarmMessage);
if (StringUtil.isBlank(notifyMessage)) {
continue;
}
alarmMessage.setAlarmMessage(notifyMessage);
String sendMessage = "{\"msg_type\":\"text\",\"content\":{\"text\":\"%s\"}}";
String requestBody = getRequestBody(sendMessage, alarmMessage);
......@@ -120,8 +153,13 @@ public class AlarmServiceImpl implements AlarmService {
if (StringUtil.isBlank(rawMessage)) {
return "";
}
String serviceName = extractServiceName(alarmMessage.getName());
String serviceName = resolveServiceName(alarmMessage, rawMessage);
String endpointPath = resolveEndpointPath(alarmMessage, rawMessage);
String message = formatReadableMessage(alarmMessage, serviceName, rawMessage);
message = appendErrorDetails(message, alarmMessage, serviceName, endpointPath);
if (StringUtil.isBlank(message)) {
return "";
}
List<ServiceInfo> serviceInfos = projProperties.getServiceInfo();
if (serviceInfos == null || serviceInfos.isEmpty()) {
return message;
......@@ -240,9 +278,150 @@ public class AlarmServiceImpl implements AlarmService {
.append("内有").append(normalizeDuration(matcher.group(3))).append("不可用");
return true;
}
matcher = SERVICE_SLA_PATTERN.matcher(rawMessage);
if (matcher.find()) {
builder.append("\n类型: 服务可用率过低");
if (StringUtil.isBlank(serviceName)) {
builder.append("\n服务: ").append(matcher.group(1));
}
builder.append("\n阈值: < ").append(normalizePercent(matcher.group(2)));
builder.append("\n窗口: 最近").append(normalizeDuration(matcher.group(4)))
.append("内有").append(normalizeDuration(matcher.group(3))).append("低于阈值");
return true;
}
matcher = ENDPOINT_SLA_PATTERN.matcher(rawMessage);
if (matcher.find()) {
builder.append("\n类型: 接口可用率过低");
builder.append("\n接口: ").append(matcher.group(1));
if (StringUtil.isBlank(serviceName)) {
builder.append("\n服务: ").append(matcher.group(2));
}
builder.append("\n阈值: < ").append(normalizePercent(matcher.group(3)));
builder.append("\n窗口: 最近").append(normalizeDuration(matcher.group(5)))
.append("内有").append(normalizeDuration(matcher.group(4))).append("低于阈值");
return true;
}
matcher = ENDPOINT_RELATION_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
builder.append("\n类型: 接口关系响应时间过高");
builder.append("\n接口: ").append(matcher.group(1));
if (StringUtil.isBlank(serviceName)) {
builder.append("\n服务: ").append(matcher.group(2));
}
builder.append("\n阈值: > ").append(matcher.group(3));
builder.append("\n窗口: 最近").append(normalizeDuration(matcher.group(5)))
.append("内有").append(normalizeDuration(matcher.group(4))).append("超过阈值");
return true;
}
return false;
}
private String appendErrorDetails(String message, AlarmMessage alarmMessage, String serviceName, String endpointPath) {
if (!skyWalkingProperties.isEnabled()) {
return message;
}
List<TraceErrorDetail> errors = skyWalkingQueryService.findRecentErrors(serviceName, endpointPath);
if (errors.isEmpty()) {
if (skyWalkingProperties.isSkipRespTimeWithoutError() && isResponseTimeRule(alarmMessage.getRuleName())) {
log.info("Skip response time alarm without error trace, service={}, rule={}", serviceName, alarmMessage.getRuleName());
return "";
}
if (isResponseTimeRule(alarmMessage.getRuleName())) {
return message + "\n\n说明: 最近" + skyWalkingProperties.getLookbackMinutes()
+ "分钟内未发现错误 Trace，当前告警原因为接口响应慢。";
}
return message;
}
return message + "\n\n" + skyWalkingQueryService.formatErrorSection(errors);
}
private boolean isResponseTimeRule(String ruleName) {
if (StringUtil.isBlank(ruleName)) {
return false;
}
return RESPONSE_TIME_RULES.contains(ruleName.trim().toLowerCase(Locale.ROOT));
}
private String resolveServiceName(AlarmMessage alarmMessage, String rawMessage) {
String serviceName = extractServiceName(alarmMessage.getName());
if (!StringUtil.isBlank(serviceName)) {
return serviceName;
}
Matcher matcher = SERVICE_SLA_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(1).trim();
}
matcher = ENDPOINT_SLA_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(2).trim();
}
matcher = ENDPOINT_RELATION_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(2).trim();
}
matcher = SERVICE_INSTANCE_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(2).trim();
}
matcher = SERVICE_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(1).trim();
}
matcher = ENDPOINT_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(2).trim();
}
if (SERVICE.equals(alarmMessage.getScope()) && !StringUtil.isBlank(alarmMessage.getName())) {
return alarmMessage.getName().trim();
}
return serviceName;
}
private String resolveEndpointPath(AlarmMessage alarmMessage, String rawMessage) {
Matcher matcher = ENDPOINT_RELATION_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(1).trim();
}
matcher = ENDPOINT_RESPONSE_TIME_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(1).trim();
}
matcher = ENDPOINT_ERROR_RATE_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(1).trim();
}
matcher = ENDPOINT_SLA_PATTERN.matcher(rawMessage);
if (matcher.find()) {
return matcher.group(1).trim();
}
if (ENDPOINT_RELATION.equalsIgnoreCase(alarmMessage.getScope())) {
return extractPathUrl(alarmMessage.getName());
}
return "";
}
private String normalizePercent(String value) {
if (StringUtil.isBlank(value)) {
return "";
}
String trimmed = value.trim();
if (trimmed.endsWith("%")) {
return trimmed;
}
try {
long numeric = Long.parseLong(trimmed);
if (numeric > 100) {
return (numeric / 100) + "%";
}
} catch (NumberFormatException ignored) {
// keep original value
}
return trimmed;
}
private String normalizeDuration(String value) {
if (StringUtil.isBlank(value)) {
return "";
......
alarm-server/src/main/java/com/ebaiyihui/alarm/server/service/SkyWalkingQueryService.java 0 → 100644
浏览文件 @ df71b912
package com.ebaiyihui.alarm.server.service;
import com.ebaiyihui.alarm.server.config.SkyWalkingProperties;
import com.ebaiyihui.alarm.server.entity.TraceErrorDetail;
import com.ebaiyihui.alarm.server.util.HttpKit;
import com.ebaiyihui.framework.utils.StringUtil;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
@Slf4j
@Service
public class SkyWalkingQueryService {
private static final Gson GSON = new Gson();
private static final TimeZone TZ = TimeZone.getTimeZone("GMT+8");
@Autowired
private SkyWalkingProperties skyWalkingProperties;
public List<TraceErrorDetail> findRecentErrors(String serviceName, String endpointPath) {
if (!skyWalkingProperties.isEnabled() || StringUtil.isBlank(serviceName)) {
return Collections.emptyList();
}
try {
String serviceId = findServiceId(serviceName);
if (StringUtil.isBlank(serviceId)) {
log.warn("SkyWalking service not found: {}", serviceName);
return Collections.emptyList();
}
String endpointId = findEndpointId(serviceId, endpointPath);
return queryErrorTraces(serviceId, endpointId);
} catch (Exception e) {
log.error("Query SkyWalking error traces failed, service={}, endpoint={}", serviceName, endpointPath, e);
return Collections.emptyList();
}
}
public String formatErrorSection(List<TraceErrorDetail> errors) {
if (errors == null || errors.isEmpty()) {
return "";
}
StringBuilder builder = new StringBuilder("【最近报错】");
int index = 1;
for (TraceErrorDetail error : errors) {
builder.append("\n").append(index++).append(". ");
if (!StringUtil.isBlank(error.getEndpoint())) {
builder.append("接口: ").append(error.getEndpoint());
}
if (!StringUtil.isBlank(error.getService())) {
builder.append("\n 服务: ").append(error.getService());
}
if (!StringUtil.isBlank(error.getErrorKind())) {
builder.append("\n 异常: ").append(error.getErrorKind());
}
if (!StringUtil.isBlank(error.getMessage())) {
builder.append("\n 信息: ").append(truncate(error.getMessage(), 300));
}
if (!StringUtil.isBlank(error.getStack())) {
builder.append("\n 堆栈:\n").append(formatStack(error.getStack()));
}
if (!StringUtil.isBlank(error.getTraceId())) {
builder.append("\n TraceId: ").append(error.getTraceId());
}
}
return builder.toString();
}
private List<TraceErrorDetail> queryErrorTraces(String serviceId, String endpointId) throws Exception {
String[] range = buildDurationRange(skyWalkingProperties.getLookbackMinutes());
String endpointField = StringUtil.isBlank(endpointId)
? ""
: String.format(", endpointId: \\\"%s\\\"", escapeGraphql(endpointId));
String query = String.format(
"{ queryBasicTraces(condition: {serviceId: \\\"%s\\\"%s, traceState: ERROR, "
+ "queryDuration: {start: \\\"%s\\\", end: \\\"%s\\\", step: MINUTE}, "
+ "queryOrder: BY_START_TIME, paging: {pageNum: 1, pageSize: %d}}) "
+ "{ traces { traceIds endpointNames } } }",
escapeGraphql(serviceId),
endpointField,
range[0],
range[1],
skyWalkingProperties.getMaxErrorTraces()
);
JsonObject data = executeGraphQL(query);
JsonArray traces = data.getAsJsonObject("queryBasicTraces").getAsJsonArray("traces");
if (traces == null || traces.size() == 0) {
return Collections.emptyList();
}
List<TraceErrorDetail> details = new ArrayList<>();
for (JsonElement traceElement : traces) {
JsonObject trace = traceElement.getAsJsonObject();
String traceId = extractFirstTraceId(trace.getAsJsonArray("traceIds"));
if (StringUtil.isBlank(traceId)) {
continue;
}
TraceErrorDetail detail = queryTraceError(traceId);
if (detail != null) {
details.add(detail);
}
}
return details;
}
private TraceErrorDetail queryTraceError(String traceId) throws Exception {
String query = String.format(
"{ queryTrace(traceId: \\\"%s\\\") { spans { endpointName serviceCode isError logs { data { key value } } } } }",
escapeGraphql(traceId)
);
JsonObject data = executeGraphQL(query);
JsonArray spans = data.getAsJsonObject("queryTrace").getAsJsonArray("spans");
if (spans == null) {
return null;
}
for (JsonElement spanElement : spans) {
JsonObject span = spanElement.getAsJsonObject();
if (!span.has("isError") || !span.get("isError").getAsBoolean()) {
continue;
}
String endpoint = getAsString(span, "endpointName");
String service = getAsString(span, "serviceCode");
String errorKind = "";
String message = "";
String stack = "";
JsonArray logs = span.getAsJsonArray("logs");
if (logs != null) {
for (JsonElement logElement : logs) {
JsonArray logData = logElement.getAsJsonObject().getAsJsonArray("data");
if (logData == null) {
continue;
}
for (JsonElement dataElement : logData) {
JsonObject item = dataElement.getAsJsonObject();
String key = getAsString(item, "key");
String value = getAsString(item, "value");
if ("error.kind".equals(key)) {
errorKind = value;
} else if ("message".equals(key)) {
message = value;
} else if ("stack".equals(key)) {
stack = value;
}
}
}
}
return new TraceErrorDetail(traceId, endpoint, service, errorKind, message, stack);
}
return null;
}
private String findServiceId(String serviceName) throws Exception {
String[] range = buildDurationRange(skyWalkingProperties.getLookbackMinutes());
String query = String.format(
"{ getAllServices(duration: {start: \\\"%s\\\", end: \\\"%s\\\", step: MINUTE}) { key: id label: name } }",
range[0],
range[1]
);
JsonObject data = executeGraphQL(query);
JsonArray services = data.getAsJsonArray("getAllServices");
if (services == null) {
return "";
}
for (JsonElement serviceElement : services) {
JsonObject service = serviceElement.getAsJsonObject();
if (serviceName.equals(getAsString(service, "label"))) {
return getAsString(service, "key");
}
}
return "";
}
private String findEndpointId(String serviceId, String endpointPath) throws Exception {
if (StringUtil.isBlank(endpointPath)) {
return "";
}
String keyword = endpointPath;
int colonIndex = endpointPath.indexOf(':');
if (colonIndex >= 0 && colonIndex < endpointPath.length() - 1) {
keyword = endpointPath.substring(colonIndex + 1);
}
int slashIndex = keyword.lastIndexOf('/');
if (slashIndex >= 0 && slashIndex < keyword.length() - 1) {
keyword = keyword.substring(slashIndex + 1);
}
String query = String.format(
"{ searchEndpoint(serviceId: \\\"%s\\\", keyword: \\\"%s\\\", limit: 10) { key: id label: name } }",
escapeGraphql(serviceId),
escapeGraphql(keyword)
);
JsonObject data = executeGraphQL(query);
JsonArray endpoints = data.getAsJsonArray("searchEndpoint");
if (endpoints == null || endpoints.size() == 0) {
return "";
}
for (JsonElement endpointElement : endpoints) {
JsonObject endpoint = endpointElement.getAsJsonObject();
if (endpointPath.equals(getAsString(endpoint, "label"))) {
return getAsString(endpoint, "key");
}
}
return getAsString(endpoints.get(0).getAsJsonObject(), "key");
}
private JsonObject executeGraphQL(String query) throws Exception {
JsonObject request = new JsonObject();
request.addProperty("query", query);
String response = HttpKit.jsonPost(skyWalkingProperties.getGraphqlUrl(), GSON.toJson(request));
JsonObject root = JsonParser.parseString(response).getAsJsonObject();
if (root.has("errors")) {
throw new IllegalStateException(root.getAsJsonArray("errors").toString());
}
return root.getAsJsonObject("data");
}
private String[] buildDurationRange(int lookbackMinutes) {
Calendar calendar = Calendar.getInstance(TZ);
Date end = calendar.getTime();
calendar.add(Calendar.MINUTE, -lookbackMinutes);
Date start = calendar.getTime();
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HHmm", Locale.ROOT);
formatter.setTimeZone(TZ);
return new String[]{formatter.format(start), formatter.format(end)};
}
private String extractFirstTraceId(JsonArray traceIds) {
if (traceIds == null || traceIds.size() == 0) {
return "";
}
return traceIds.get(0).getAsString();
}
private String formatStack(String stack) {
String[] lines = stack.split("\\r?\\n");
int maxLines = Math.max(1, skyWalkingProperties.getMaxStackLines());
StringBuilder builder = new StringBuilder();
int count = 0;
for (String line : lines) {
if (count >= maxLines) {
builder.append(" ...");
break;
}
builder.append(" ").append(line.trim()).append("\n");
count++;
}
return builder.toString().trim();
}
private String truncate(String value, int maxLength) {
if (value == null || value.length() <= maxLength) {
return value;
}
return value.substring(0, maxLength) + "...";
}
private String escapeGraphql(String value) {
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private String getAsString(JsonObject jsonObject, String field) {
if (jsonObject == null || !jsonObject.has(field) || jsonObject.get(field).isJsonNull()) {
return "";
}
return jsonObject.get(field).getAsString();
}
}
alarm-server/src/main/resources/application.yml
浏览文件 @ df71b912
......@@ -71,6 +71,16 @@ spring:
webHookUrl: https://open.feishu.cn/open-apis/bot/v2/hook/480d39dc-e470-475b-95a9-a5210b23c1f9
secret: ddjydsLwgajAXWEpWoK2ff
skywalking:
enabled: true
# Docker 内网访问 SkyWalking UI/OAP GraphQL；本地调试可改为 http://192.168.0.107:8080/graphql
graphql-url: http://192.168.0.107:8080/graphql
lookback-minutes: 10
max-error-traces: 3
max-stack-lines: 5
# 设为 true 时，纯响应时间告警且查不到错误 Trace 则不推送飞书
skip-resp-time-without-error: false
# ou_c6455365320227160724033bc454c666 王猛
# ou_6cb2f35f9bf8e7d7e977483801ab13de 杨凯
# ou_76b08d1b3343bffdd5b4304dd920d19a 秦静平
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论