Skip to content

B
byh-alarm-service
提交 ba6c1b1f authored 作者: Edwin's avatar Edwin
浏览文件
操作

feat: add service metrics statistics and formatting to alarm service

上级 a5c3686e
显示空白字符变更
内嵌 并排
正在显示 包含 179 行增加 和 0 行删除
alarm-server/src/main/java/com/ebaiyihui/alarm/server/entity/ServiceMetricsStats.java 0 → 100644
浏览文件 @ ba6c1b1f
package com.ebaiyihui.alarm.server.entity;
import lombok.Getter;
@Getter
public class ServiceMetricsStats {
private final int lookbackMinutes;
private final long totalRequests;
private final long failedRequests;
private final long errorTraceCount;
private final long slowTraceCount;
private final double avgSuccessRatePercent;
private final double minSuccessRatePercent;
private final long avgResponseTimeMs;
private final long maxResponseTimeMs;
public ServiceMetricsStats(int lookbackMinutes, long totalRequests, long failedRequests,
long errorTraceCount, long slowTraceCount,
double avgSuccessRatePercent, double minSuccessRatePercent,
long avgResponseTimeMs, long maxResponseTimeMs) {
this.lookbackMinutes = lookbackMinutes;
this.totalRequests = totalRequests;
this.failedRequests = failedRequests;
this.errorTraceCount = errorTraceCount;
this.slowTraceCount = slowTraceCount;
this.avgSuccessRatePercent = avgSuccessRatePercent;
this.minSuccessRatePercent = minSuccessRatePercent;
this.avgResponseTimeMs = avgResponseTimeMs;
this.maxResponseTimeMs = maxResponseTimeMs;
}
}
alarm-server/src/main/java/com/ebaiyihui/alarm/server/service/Impl/AlarmServiceImpl.java
浏览文件 @ ba6c1b1f
......@@ -156,6 +156,7 @@ public class AlarmServiceImpl implements AlarmService {
String serviceName = resolveServiceName(alarmMessage, rawMessage);
String endpointPath = resolveEndpointPath(alarmMessage, rawMessage);
String message = formatReadableMessage(alarmMessage, serviceName, rawMessage);
message = appendMetricsStats(message, serviceName, endpointPath);
message = appendErrorDetails(message, alarmMessage, serviceName, endpointPath);
if (StringUtil.isBlank(message)) {
return "";
......@@ -319,6 +320,18 @@ public class AlarmServiceImpl implements AlarmService {
return false;
}
private String appendMetricsStats(String message, String serviceName, String endpointPath) {
if (!skyWalkingProperties.isEnabled()) {
return message;
}
String metricsSection = skyWalkingQueryService.formatMetricsSection(
skyWalkingQueryService.queryServiceStats(serviceName, endpointPath));
if (StringUtil.isBlank(metricsSection)) {
return message;
}
return message + "\n\n" + metricsSection;
}
private String appendErrorDetails(String message, AlarmMessage alarmMessage, String serviceName, String endpointPath) {
if (!skyWalkingProperties.isEnabled()) {
return message;
......
alarm-server/src/main/java/com/ebaiyihui/alarm/server/service/SkyWalkingQueryService.java
浏览文件 @ ba6c1b1f
package com.ebaiyihui.alarm.server.service;
import com.ebaiyihui.alarm.server.config.SkyWalkingProperties;
import com.ebaiyihui.alarm.server.entity.ServiceMetricsStats;
import com.ebaiyihui.alarm.server.entity.TraceErrorDetail;
import com.ebaiyihui.alarm.server.util.HttpKit;
import com.ebaiyihui.framework.utils.StringUtil;
......@@ -54,6 +55,116 @@ public class SkyWalkingQueryService {
}
}
public ServiceMetricsStats queryServiceStats(String serviceName, String endpointPath) {
if (!skyWalkingProperties.isEnabled() || StringUtil.isBlank(serviceName)) {
return null;
}
try {
int lookbackMinutes = skyWalkingProperties.getLookbackMinutes();
String[] range = buildDurationRange(lookbackMinutes);
boolean endpointScope = !StringUtil.isBlank(endpointPath);
String entity = buildEntityCondition(serviceName, endpointPath);
String cpmMetric = endpointScope ? "endpoint_cpm" : "service_cpm";
String slaMetric = endpointScope ? "endpoint_sla" : "service_sla";
String respMetric = endpointScope ? "endpoint_avg" : "service_resp_time";
String serviceId = findServiceId(serviceName);
if (StringUtil.isBlank(serviceId)) {
return null;
}
String endpointId = endpointScope ? findEndpointId(serviceId, endpointPath) : "";
String endpointField = StringUtil.isBlank(endpointId)
? ""
: String.format(", endpointId: \"%s\"", quoteGraphqlValue(endpointId));
String query = String.format(
"{ cpm: readMetricsValues(condition: {name: \"%s\", entity: %s}, duration: {start: \"%s\", end: \"%s\", step: MINUTE}) { values { values { value } } } "
+ "sla: readMetricsValues(condition: {name: \"%s\", entity: %s}, duration: {start: \"%s\", end: \"%s\", step: MINUTE}) { values { values { value } } } "
+ "resp: readMetricsValues(condition: {name: \"%s\", entity: %s}, duration: {start: \"%s\", end: \"%s\", step: MINUTE}) { values { values { value } } } "
+ "errors: queryBasicTraces(condition: {serviceId: \"%s\"%s, traceState: ERROR, queryDuration: {start: \"%s\", end: \"%s\", step: MINUTE}, queryOrder: BY_START_TIME, paging: {pageNum: 1, pageSize: 1}}) { total } "
+ "slow: queryBasicTraces(condition: {serviceId: \"%s\"%s, traceState: ALL, minTraceDuration: 3000, queryDuration: {start: \"%s\", end: \"%s\", step: MINUTE}, queryOrder: BY_START_TIME, paging: {pageNum: 1, pageSize: 1}}) { total } }",
cpmMetric, entity, range[0], range[1],
slaMetric, entity, range[0], range[1],
respMetric, entity, range[0], range[1],
quoteGraphqlValue(serviceId), endpointField, range[0], range[1],
quoteGraphqlValue(serviceId), endpointField, range[0], range[1]
);
JsonObject data = executeGraphQL(query);
long totalRequests = 0;
long failedRequests = 0;
double slaSum = 0;
int slaCount = 0;
double minSla = 10000;
long respSum = 0;
int respCount = 0;
long maxResp = 0;
JsonArray cpmValues = extractMetricValues(data.getAsJsonObject("cpm"));
JsonArray slaValues = extractMetricValues(data.getAsJsonObject("sla"));
JsonArray respValues = extractMetricValues(data.getAsJsonObject("resp"));
int points = Math.min(cpmValues.size(), Math.min(slaValues.size(), respValues.size()));
for (int i = 0; i < points; i++) {
long cpm = cpmValues.get(i).getAsJsonObject().get("value").getAsLong();
long sla = slaValues.get(i).getAsJsonObject().get("value").getAsLong();
long resp = respValues.get(i).getAsJsonObject().get("value").getAsLong();
if (cpm <= 0) {
continue;
}
totalRequests += cpm;
failedRequests += Math.round(cpm * (10000D - sla) / 10000D);
slaSum += sla;
slaCount++;
if (sla < minSla) {
minSla = sla;
}
if (resp > 0) {
respSum += resp;
respCount++;
if (resp > maxResp) {
maxResp = resp;
}
}
}
long errorTraceCount = data.getAsJsonObject("errors").get("total").getAsLong();
long slowTraceCount = data.getAsJsonObject("slow").get("total").getAsLong();
double avgSla = slaCount > 0 ? slaSum / slaCount : 10000;
long avgResp = respCount > 0 ? Math.round(respSum * 1D / respCount) : 0;
return new ServiceMetricsStats(
lookbackMinutes,
totalRequests,
failedRequests,
errorTraceCount,
slowTraceCount,
avgSla / 100D,
minSla / 100D,
avgResp,
maxResp
);
} catch (Exception e) {
log.error("Query SkyWalking service stats failed, service={}, endpoint={}", serviceName, endpointPath, e);
return null;
}
}
public String formatMetricsSection(ServiceMetricsStats stats) {
if (stats == null) {
return "";
}
StringBuilder builder = new StringBuilder("【最近").append(stats.getLookbackMinutes()).append("分钟统计】");
builder.append("\n总请求: ").append(stats.getTotalRequests()).append(" 次");
builder.append("\n失败约: ").append(stats.getFailedRequests()).append(" 次");
builder.append(" (成功率平均 ").append(formatPercent(stats.getAvgSuccessRatePercent()));
builder.append(", 最低 ").append(formatPercent(stats.getMinSuccessRatePercent())).append(")");
builder.append("\n错误Trace: ").append(stats.getErrorTraceCount()).append(" 条");
builder.append("\n平均响应: ").append(stats.getAvgResponseTimeMs()).append("ms");
builder.append(", 最慢: ").append(stats.getMaxResponseTimeMs()).append("ms");
builder.append("\n慢请求(>3s): ").append(stats.getSlowTraceCount()).append(" 条");
return builder.toString();
}
public String formatErrorSection(List<TraceErrorDetail> errors) {
if (errors == null || errors.isEmpty()) {
return "";
......@@ -122,6 +233,29 @@ public class SkyWalkingQueryService {
return details;
}
private JsonArray extractMetricValues(JsonObject metricsObject) {
if (metricsObject == null || !metricsObject.has("values")) {
return new JsonArray();
}
JsonObject valuesWrapper = metricsObject.getAsJsonObject("values");
if (valuesWrapper == null || !valuesWrapper.has("values")) {
return new JsonArray();
}
return valuesWrapper.getAsJsonArray("values");
}
private String buildEntityCondition(String serviceName, String endpointPath) {
if (!StringUtil.isBlank(endpointPath)) {
return String.format("{scope: Endpoint, serviceName: \"%s\", endpointName: \"%s\", normal: true}",
quoteGraphqlValue(serviceName), quoteGraphqlValue(endpointPath));
}
return String.format("{scope: Service, serviceName: \"%s\", normal: true}", quoteGraphqlValue(serviceName));
}
private String formatPercent(double percent) {
return String.format(Locale.ROOT, "%.2f%%", percent);
}
private TraceErrorDetail queryTraceError(String traceId) throws Exception {
String query = String.format(
"{ queryTrace(traceId: \"%s\") { spans { endpointName serviceCode isError logs { data { key value } } } } }",
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论