提交 a5c3686e authored 作者: Edwin's avatar Edwin

feat: enhance SkyWalking properties and error trace handling in alarm service

上级 4106125d
...@@ -27,9 +27,14 @@ public class SkyWalkingProperties { ...@@ -27,9 +27,14 @@ public class SkyWalkingProperties {
private int maxErrorTraces = 3; private int maxErrorTraces = 3;
/** /**
* 堆栈最多展示行数 * 堆栈最多展示行数,0 表示不限制行数(仍受 maxStackChars 约束)
*/ */
private int maxStackLines = 5; private int maxStackLines = 25;
/**
* 单条错误堆栈最多字符数,避免飞书消息过长
*/
private int maxStackChars = 3000;
/** /**
* 为 true 时,纯响应时间告警且查不到错误 Trace 则不发送飞书通知 * 为 true 时,纯响应时间告警且查不到错误 Trace 则不发送飞书通知
......
...@@ -507,7 +507,8 @@ public class AlarmServiceImpl implements AlarmService { ...@@ -507,7 +507,8 @@ public class AlarmServiceImpl implements AlarmService {
* deal requestBody,if it has sign set the sign * deal requestBody,if it has sign set the sign
*/ */
private String getRequestBody(String sendMessage, AlarmMessage alarmMessage) { private String getRequestBody(String sendMessage, AlarmMessage alarmMessage) {
String requestBody = String.format(sendMessage, alarmMessage.getAlarmMessage()); String escapedMessage = alarmMessage.getAlarmMessage().replace("%", "%%");
String requestBody = String.format(sendMessage, escapedMessage);
Gson gson = new Gson(); Gson gson = new Gson();
JsonObject jsonObject = gson.fromJson(requestBody, JsonObject.class); JsonObject jsonObject = gson.fromJson(requestBody, JsonObject.class);
Map<String, Object> content = buildContent(jsonObject); Map<String, Object> content = buildContent(jsonObject);
......
...@@ -43,7 +43,11 @@ public class SkyWalkingQueryService { ...@@ -43,7 +43,11 @@ public class SkyWalkingQueryService {
return Collections.emptyList(); return Collections.emptyList();
} }
String endpointId = findEndpointId(serviceId, endpointPath); String endpointId = findEndpointId(serviceId, endpointPath);
return queryErrorTraces(serviceId, endpointId); List<TraceErrorDetail> details = queryErrorTraces(serviceId, endpointId);
if (details.isEmpty() && !StringUtil.isBlank(endpointId)) {
details = queryErrorTraces(serviceId, "");
}
return details;
} catch (Exception e) { } catch (Exception e) {
log.error("Query SkyWalking error traces failed, service={}, endpoint={}", serviceName, endpointPath, e); log.error("Query SkyWalking error traces failed, service={}, endpoint={}", serviceName, endpointPath, e);
return Collections.emptyList(); return Collections.emptyList();
...@@ -72,6 +76,8 @@ public class SkyWalkingQueryService { ...@@ -72,6 +76,8 @@ public class SkyWalkingQueryService {
} }
if (!StringUtil.isBlank(error.getStack())) { if (!StringUtil.isBlank(error.getStack())) {
builder.append("\n 堆栈:\n").append(formatStack(error.getStack())); builder.append("\n 堆栈:\n").append(formatStack(error.getStack()));
} else if (!StringUtil.isBlank(error.getErrorKind()) || !StringUtil.isBlank(error.getMessage())) {
builder.append("\n 堆栈: (SkyWalking 未采集到 stack,请在业务 Agent 中开启异常栈上报)");
} }
if (!StringUtil.isBlank(error.getTraceId())) { if (!StringUtil.isBlank(error.getTraceId())) {
builder.append("\n TraceId: ").append(error.getTraceId()); builder.append("\n TraceId: ").append(error.getTraceId());
...@@ -126,40 +132,68 @@ public class SkyWalkingQueryService { ...@@ -126,40 +132,68 @@ public class SkyWalkingQueryService {
if (spans == null) { if (spans == null) {
return null; return null;
} }
TraceErrorDetail best = null;
for (JsonElement spanElement : spans) { for (JsonElement spanElement : spans) {
JsonObject span = spanElement.getAsJsonObject(); JsonObject span = spanElement.getAsJsonObject();
if (!span.has("isError") || !span.get("isError").getAsBoolean()) { if (!span.has("isError") || !span.get("isError").getAsBoolean()) {
continue; continue;
} }
String endpoint = getAsString(span, "endpointName"); TraceErrorDetail candidate = extractErrorFromSpan(traceId, span);
String service = getAsString(span, "serviceCode"); if (candidate == null) {
String errorKind = ""; continue;
String message = ""; }
String stack = ""; if (best == null || errorDetailScore(candidate) > errorDetailScore(best)) {
JsonArray logs = span.getAsJsonArray("logs"); best = candidate;
if (logs != null) { }
for (JsonElement logElement : logs) { }
JsonArray logData = logElement.getAsJsonObject().getAsJsonArray("data"); return best;
if (logData == null) { }
continue;
} private TraceErrorDetail extractErrorFromSpan(String traceId, JsonObject span) {
for (JsonElement dataElement : logData) { String endpoint = getAsString(span, "endpointName");
JsonObject item = dataElement.getAsJsonObject(); String service = getAsString(span, "serviceCode");
String key = getAsString(item, "key"); String errorKind = "";
String value = getAsString(item, "value"); String message = "";
if ("error.kind".equals(key)) { String stack = "";
errorKind = value; JsonArray logs = span.getAsJsonArray("logs");
} else if ("message".equals(key)) { if (logs != null) {
message = value; for (JsonElement logElement : logs) {
} else if ("stack".equals(key)) { JsonArray logData = logElement.getAsJsonObject().getAsJsonArray("data");
stack = value; if (logData == null) {
} continue;
}
for (JsonElement dataElement : logData) {
JsonObject item = dataElement.getAsJsonObject();
String key = getAsString(item, "key");
String value = getAsString(item, "value");
if ("error.kind".equals(key) && StringUtil.isBlank(errorKind)) {
errorKind = value;
} else if ("message".equals(key) && StringUtil.isBlank(message)) {
message = value;
} else if ("stack".equals(key) && value.length() > stack.length()) {
stack = value;
} }
} }
} }
return new TraceErrorDetail(traceId, endpoint, service, errorKind, message, stack);
} }
return null; if (StringUtil.isBlank(errorKind) && StringUtil.isBlank(message) && StringUtil.isBlank(stack)) {
return null;
}
return new TraceErrorDetail(traceId, endpoint, service, errorKind, message, stack);
}
private int errorDetailScore(TraceErrorDetail detail) {
int score = 0;
if (!StringUtil.isBlank(detail.getStack())) {
score += 10000 + detail.getStack().length();
}
if (!StringUtil.isBlank(detail.getMessage())) {
score += 100 + detail.getMessage().length();
}
if (!StringUtil.isBlank(detail.getErrorKind())) {
score += 10;
}
return score;
} }
private String findServiceId(String serviceName) throws Exception { private String findServiceId(String serviceName) throws Exception {
...@@ -245,16 +279,22 @@ public class SkyWalkingQueryService { ...@@ -245,16 +279,22 @@ public class SkyWalkingQueryService {
private String formatStack(String stack) { private String formatStack(String stack) {
String[] lines = stack.split("\\r?\\n"); String[] lines = stack.split("\\r?\\n");
int maxLines = Math.max(1, skyWalkingProperties.getMaxStackLines()); int maxLines = skyWalkingProperties.getMaxStackLines();
int maxChars = Math.max(500, skyWalkingProperties.getMaxStackChars());
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
int count = 0; int lineCount = 0;
for (String line : lines) { for (String line : lines) {
if (count >= maxLines) { if (maxLines > 0 && lineCount >= maxLines) {
builder.append(" ..."); builder.append(" ...(堆栈已截断,可在 SkyWalking UI 用 TraceId 查看完整堆栈)");
break;
}
String formattedLine = " " + line.trim() + "\n";
if (builder.length() + formattedLine.length() > maxChars) {
builder.append(" ...(堆栈已截断,可在 SkyWalking UI 用 TraceId 查看完整堆栈)");
break; break;
} }
builder.append(" ").append(line.trim()).append("\n"); builder.append(formattedLine);
count++; lineCount++;
} }
return builder.toString().trim(); return builder.toString().trim();
} }
......
...@@ -77,7 +77,8 @@ skywalking: ...@@ -77,7 +77,8 @@ skywalking:
graphql-url: http://192.168.0.107:8080/graphql graphql-url: http://192.168.0.107:8080/graphql
lookback-minutes: 10 lookback-minutes: 10
max-error-traces: 3 max-error-traces: 3
max-stack-lines: 5 max-stack-lines: 25
max-stack-chars: 3000
# 设为 true 时,纯响应时间告警且查不到错误 Trace 则不推送飞书 # 设为 true 时,纯响应时间告警且查不到错误 Trace 则不推送飞书
skip-resp-time-without-error: true skip-resp-time-without-error: true
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论