// send metrics to backend central logging system
void sendMetrics(object sender, ElapsedEventArgs arg)
{
try
{
if (EndpointHost.MetadataMap == null)
{
return;
}
foreach (ServiceMetadata metadata in EndpointHost.MetadataMap.Values)
{
try
{
if (metadata.ServiceName == ServiceMetadata.AnonymousServiceName)
{
continue;
}
var tagMap = new Dictionary <string, string>();
tagMap["webservice"] = (ServiceUtils.ConvertNamespaceToMetricPrefix(metadata.ServiceNamespace) + "." + metadata.ServiceName)
.ToLower().Replace("soa.ant.com.", string.Empty);
tagMap["frameworkversion"] = string.Format("SS-{0} CG-{1}", metadata.AntServiceStackVersion, metadata.AntCodeGenVersion);
var now = DateTime.Now;
Action <HystrixCommandMetrics, long, string, string> logEventDistribution = (m, c, tn, tv) =>
{
if (c <= 0)
{
return;
}
tagMap[tn] = tv;
//metricLog.log(m.MetricNameEventDistribution, c, tagMap, now);
};
Action <HystrixCommandMetrics, long, long?, string, string> logLatencyDistribution = (m, s, e, tn, tv) =>
{
int count = m.GetServiceExecutionCountInTotalTimeRange(s, e);
if (count <= 0)
{
return;
}
tagMap[tn] = tv;
// metricLog.log(m.MetricNameLatencyDistribution, count, tagMap, now);
};
Action <HystrixCommandMetrics, double, string, string> logLatencyPencentile = (m, p, tn, tv) =>
{
long pencentile = m.GetTotalTimePercentile(p);
if (pencentile <= 0)
{
return;
}
tagMap[tn] = tv;
// metricLog.log(m.MetricNameLatencyPercentile, pencentile, tagMap, now);
};
foreach (Operation operation in metadata.Operations)
{
HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics;
tagMap["operation"] = tagMap["webservice"] + "." + commandMetrics.OperationName.ToLower();
long successCount = commandMetrics.GetSuccessCount();
long frameworkErrorCount = commandMetrics.GetFrameworkErrorCount();
long serviceErrorCount = commandMetrics.GetServiceErrorCount();
long validationErrorCount = commandMetrics.GetValidationErrorCount();
long shortCircuitedCount = commandMetrics.GetShortCircuitCount();
long timeoutCount = commandMetrics.GetTimeoutCount();
long threadPoolRejectedCount = commandMetrics.GetThreadPoolRejectedCount();
commandMetrics.ResetMetricsCounters();
long totalCount = successCount + frameworkErrorCount + serviceErrorCount + validationErrorCount + shortCircuitedCount + timeoutCount + threadPoolRejectedCount;
//metricLog.log(commandMetrics.MetricNameRequestCount, totalCount, tagMap, now);
// if(operation.IsAsync)
//metricLog.log(commandMetrics.MetricNameConcurrentExecutionCount, commandMetrics.CurrentConcurrentExecutionCount, tagMap, now);
var tagName = "distribution";
logEventDistribution(commandMetrics, successCount, tagName, "Success");
logEventDistribution(commandMetrics, shortCircuitedCount, tagName, "Short Circuited");
logEventDistribution(commandMetrics, timeoutCount, tagName, "Timeout");
logEventDistribution(commandMetrics, threadPoolRejectedCount, tagName, "Threadpool Rejected");
logEventDistribution(commandMetrics, frameworkErrorCount, tagName, "Framework Exception");
logEventDistribution(commandMetrics, serviceErrorCount, tagName, "Service Exception");
logEventDistribution(commandMetrics, validationErrorCount, tagName, "Validation Exception");
if (tagMap.ContainsKey(tagName))
{
tagMap.Remove(tagName);
}
int count;
long sum, min, max;
commandMetrics.GetTotalTimeMetricsData(out count, out sum, out min, out max);
tagName = "SetFeatureType";
tagMap[tagName] = "count";
// metricLog.log(commandMetrics.MetricNameLatency, count, tagMap, now);
tagMap[tagName] = "sum";
//metricLog.log(commandMetrics.MetricNameLatency, sum, tagMap, now);
tagMap[tagName] = "min";
// metricLog.log(commandMetrics.MetricNameLatency, min, tagMap, now);
// tagMap[tagName] = "max";
//metricLog.log(commandMetrics.MetricNameLatency, max, tagMap, now);
tagMap.Remove(tagName);
tagName = "distribution";
logLatencyDistribution(commandMetrics, 0, 10, tagName, "0 ~ 10ms");
logLatencyDistribution(commandMetrics, 10, 50, tagName, "10 ~ 50ms");
logLatencyDistribution(commandMetrics, 50, 200, tagName, "50 ~ 200ms");
logLatencyDistribution(commandMetrics, 200, 500, tagName, "200 ~ 500ms");
logLatencyDistribution(commandMetrics, 500, 1000, tagName, "500ms ~ 1s");
logLatencyDistribution(commandMetrics, 1000, 5 * 1000, tagName, "1 ~ 5s");
logLatencyDistribution(commandMetrics, 5 * 1000, 10 * 1000, tagName, "5 ~ 10s");
logLatencyDistribution(commandMetrics, 10 * 1000, 30 * 1000, tagName, "10 ~ 30s");
logLatencyDistribution(commandMetrics, 30 * 1000, 100 * 1000, tagName, "30 ~ 100s");
logLatencyDistribution(commandMetrics, 100 * 1000, null, tagName, ">= 100s");
if (tagMap.ContainsKey(tagName))
{
tagMap.Remove(tagName);
}
tagName = "percentile";
logLatencyPencentile(commandMetrics, 0, tagName, "0");
logLatencyPencentile(commandMetrics, 25, tagName, "25");
logLatencyPencentile(commandMetrics, 50, tagName, "50");
logLatencyPencentile(commandMetrics, 75, tagName, "75");
logLatencyPencentile(commandMetrics, 90, tagName, "90");
logLatencyPencentile(commandMetrics, 95, tagName, "95");
logLatencyPencentile(commandMetrics, 99, tagName, "99");
logLatencyPencentile(commandMetrics, 99.5, tagName, "99.5");
logLatencyPencentile(commandMetrics, 100, tagName, "100");
if (tagMap.ContainsKey(tagName))
{
tagMap.Remove(tagName);
}
}
}
catch (Exception ex)
{
log.Error("Fail to send metrics to backend!", ex, new Dictionary <string, string>()
{
{ "ErrorCode", "FXD300009" }
});
}
}
}
catch (Exception ex)
{
log.Error("Fail to send metrics to backend!", ex, new Dictionary <string, string>()
{
{ "ErrorCode", "FXD300009" }
});
}
}
