提交 a5c3686e authored 作者: Edwin's avatar Edwin

feat: enhance SkyWalking properties and error trace handling in alarm service

上级 4106125d
......@@ -27,9 +27,14 @@ public class SkyWalkingProperties {
private int maxErrorTraces = 3;
/**
* 堆栈最多展示行数
* 堆栈最多展示行数,0 表示不限制行数(仍受 maxStackChars 约束)
*/
private int maxStackLines = 5;
private int maxStackLines = 25;
/**
* 单条错误堆栈最多字符数,避免飞书消息过长
*/
private int maxStackChars = 3000;
/**
* 为 true 时,纯响应时间告警且查不到错误 Trace 则不发送飞书通知
......
......@@ -507,7 +507,8 @@ public class AlarmServiceImpl implements AlarmService {
* deal requestBody,if it has sign set the sign
*/
private String getRequestBody(String sendMessage, AlarmMessage alarmMessage) {
String requestBody = String.format(sendMessage, alarmMessage.getAlarmMessage());
String escapedMessage = alarmMessage.getAlarmMessage().replace("%", "%%");
String requestBody = String.format(sendMessage, escapedMessage);
Gson gson = new Gson();
JsonObject jsonObject = gson.fromJson(requestBody, JsonObject.class);
Map<String, Object> content = buildContent(jsonObject);
......
......@@ -43,7 +43,11 @@ public class SkyWalkingQueryService {
return Collections.emptyList();
}
String endpointId = findEndpointId(serviceId, endpointPath);
return queryErrorTraces(serviceId, endpointId);
List<TraceErrorDetail> details = queryErrorTraces(serviceId, endpointId);
if (details.isEmpty() && !StringUtil.isBlank(endpointId)) {
details = queryErrorTraces(serviceId, "");
}
return details;
} catch (Exception e) {
log.error("Query SkyWalking error traces failed, service={}, endpoint={}", serviceName, endpointPath, e);
return Collections.emptyList();
......@@ -72,6 +76,8 @@ public class SkyWalkingQueryService {
}
if (!StringUtil.isBlank(error.getStack())) {
builder.append("\n 堆栈:\n").append(formatStack(error.getStack()));
} else if (!StringUtil.isBlank(error.getErrorKind()) || !StringUtil.isBlank(error.getMessage())) {
builder.append("\n 堆栈: (SkyWalking 未采集到 stack,请在业务 Agent 中开启异常栈上报)");
}
if (!StringUtil.isBlank(error.getTraceId())) {
builder.append("\n TraceId: ").append(error.getTraceId());
......@@ -126,11 +132,24 @@ public class SkyWalkingQueryService {
if (spans == null) {
return null;
}
TraceErrorDetail best = null;
for (JsonElement spanElement : spans) {
JsonObject span = spanElement.getAsJsonObject();
if (!span.has("isError") || !span.get("isError").getAsBoolean()) {
continue;
}
TraceErrorDetail candidate = extractErrorFromSpan(traceId, span);
if (candidate == null) {
continue;
}
if (best == null || errorDetailScore(candidate) > errorDetailScore(best)) {
best = candidate;
}
}
return best;
}
private TraceErrorDetail extractErrorFromSpan(String traceId, JsonObject span) {
String endpoint = getAsString(span, "endpointName");
String service = getAsString(span, "serviceCode");
String errorKind = "";
......@@ -147,19 +166,34 @@ public class SkyWalkingQueryService {
JsonObject item = dataElement.getAsJsonObject();
String key = getAsString(item, "key");
String value = getAsString(item, "value");
if ("error.kind".equals(key)) {
if ("error.kind".equals(key) && StringUtil.isBlank(errorKind)) {
errorKind = value;
} else if ("message".equals(key)) {
} else if ("message".equals(key) && StringUtil.isBlank(message)) {
message = value;
} else if ("stack".equals(key)) {
} else if ("stack".equals(key) && value.length() > stack.length()) {
stack = value;
}
}
}
}
if (StringUtil.isBlank(errorKind) && StringUtil.isBlank(message) && StringUtil.isBlank(stack)) {
return null;
}
return new TraceErrorDetail(traceId, endpoint, service, errorKind, message, stack);
}
return null;
private int errorDetailScore(TraceErrorDetail detail) {
int score = 0;
if (!StringUtil.isBlank(detail.getStack())) {
score += 10000 + detail.getStack().length();
}
if (!StringUtil.isBlank(detail.getMessage())) {
score += 100 + detail.getMessage().length();
}
if (!StringUtil.isBlank(detail.getErrorKind())) {
score += 10;
}
return score;
}
private String findServiceId(String serviceName) throws Exception {
......@@ -245,16 +279,22 @@ public class SkyWalkingQueryService {
private String formatStack(String stack) {
String[] lines = stack.split("\\r?\\n");
int maxLines = Math.max(1, skyWalkingProperties.getMaxStackLines());
int maxLines = skyWalkingProperties.getMaxStackLines();
int maxChars = Math.max(500, skyWalkingProperties.getMaxStackChars());
StringBuilder builder = new StringBuilder();
int count = 0;
int lineCount = 0;
for (String line : lines) {
if (count >= maxLines) {
builder.append(" ...");
if (maxLines > 0 && lineCount >= maxLines) {
builder.append(" ...(堆栈已截断,可在 SkyWalking UI 用 TraceId 查看完整堆栈)");
break;
}
String formattedLine = " " + line.trim() + "\n";
if (builder.length() + formattedLine.length() > maxChars) {
builder.append(" ...(堆栈已截断,可在 SkyWalking UI 用 TraceId 查看完整堆栈)");
break;
}
builder.append(" ").append(line.trim()).append("\n");
count++;
builder.append(formattedLine);
lineCount++;
}
return builder.toString().trim();
}
......
......@@ -77,7 +77,8 @@ skywalking:
graphql-url: http://192.168.0.107:8080/graphql
lookback-minutes: 10
max-error-traces: 3
max-stack-lines: 5
max-stack-lines: 25
max-stack-chars: 3000
# 设为 true 时,纯响应时间告警且查不到错误 Trace 则不推送飞书
skip-resp-time-without-error: true
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论