Exemple #1
0
        // send metrics to backend central logging system
        void sendMetrics(object sender, ElapsedEventArgs arg)
        {
            try
            {
                if (EndpointHost.MetadataMap == null)
                {
                    return;
                }
                foreach (ServiceMetadata metadata in EndpointHost.MetadataMap.Values)
                {
                    try
                    {
                        if (metadata.ServiceName == ServiceMetadata.AnonymousServiceName)
                        {
                            continue;
                        }

                        var tagMap = new Dictionary <string, string>();
                        tagMap["webservice"] = (ServiceUtils.ConvertNamespaceToMetricPrefix(metadata.ServiceNamespace) + "." + metadata.ServiceName)
                                               .ToLower().Replace("soa.ant.com.", string.Empty);
                        tagMap["frameworkversion"] = string.Format("SS-{0} CG-{1}", metadata.AntServiceStackVersion, metadata.AntCodeGenVersion);
                        var now = DateTime.Now;
                        Action <HystrixCommandMetrics, long, string, string> logEventDistribution = (m, c, tn, tv) =>
                        {
                            if (c <= 0)
                            {
                                return;
                            }

                            tagMap[tn] = tv;
                            //metricLog.log(m.MetricNameEventDistribution, c, tagMap, now);
                        };
                        Action <HystrixCommandMetrics, long, long?, string, string> logLatencyDistribution = (m, s, e, tn, tv) =>
                        {
                            int count = m.GetServiceExecutionCountInTotalTimeRange(s, e);
                            if (count <= 0)
                            {
                                return;
                            }

                            tagMap[tn] = tv;
                            // metricLog.log(m.MetricNameLatencyDistribution, count, tagMap, now);
                        };
                        Action <HystrixCommandMetrics, double, string, string> logLatencyPencentile = (m, p, tn, tv) =>
                        {
                            long pencentile = m.GetTotalTimePercentile(p);
                            if (pencentile <= 0)
                            {
                                return;
                            }

                            tagMap[tn] = tv;
                            // metricLog.log(m.MetricNameLatencyPercentile, pencentile, tagMap, now);
                        };

                        foreach (Operation operation in metadata.Operations)
                        {
                            HystrixCommandMetrics commandMetrics = operation.HystrixCommand.Metrics;
                            tagMap["operation"] = tagMap["webservice"] + "." + commandMetrics.OperationName.ToLower();

                            long successCount            = commandMetrics.GetSuccessCount();
                            long frameworkErrorCount     = commandMetrics.GetFrameworkErrorCount();
                            long serviceErrorCount       = commandMetrics.GetServiceErrorCount();
                            long validationErrorCount    = commandMetrics.GetValidationErrorCount();
                            long shortCircuitedCount     = commandMetrics.GetShortCircuitCount();
                            long timeoutCount            = commandMetrics.GetTimeoutCount();
                            long threadPoolRejectedCount = commandMetrics.GetThreadPoolRejectedCount();
                            commandMetrics.ResetMetricsCounters();

                            long totalCount = successCount + frameworkErrorCount + serviceErrorCount + validationErrorCount + shortCircuitedCount + timeoutCount + threadPoolRejectedCount;
                            //metricLog.log(commandMetrics.MetricNameRequestCount, totalCount, tagMap, now);

                            // if(operation.IsAsync)
                            //metricLog.log(commandMetrics.MetricNameConcurrentExecutionCount, commandMetrics.CurrentConcurrentExecutionCount, tagMap, now);

                            var tagName = "distribution";
                            logEventDistribution(commandMetrics, successCount, tagName, "Success");
                            logEventDistribution(commandMetrics, shortCircuitedCount, tagName, "Short Circuited");
                            logEventDistribution(commandMetrics, timeoutCount, tagName, "Timeout");
                            logEventDistribution(commandMetrics, threadPoolRejectedCount, tagName, "Threadpool Rejected");
                            logEventDistribution(commandMetrics, frameworkErrorCount, tagName, "Framework Exception");
                            logEventDistribution(commandMetrics, serviceErrorCount, tagName, "Service Exception");
                            logEventDistribution(commandMetrics, validationErrorCount, tagName, "Validation Exception");
                            if (tagMap.ContainsKey(tagName))
                            {
                                tagMap.Remove(tagName);
                            }

                            int  count;
                            long sum, min, max;
                            commandMetrics.GetTotalTimeMetricsData(out count, out sum, out min, out max);
                            tagName         = "SetFeatureType";
                            tagMap[tagName] = "count";
                            // metricLog.log(commandMetrics.MetricNameLatency, count, tagMap, now);
                            tagMap[tagName] = "sum";
                            //metricLog.log(commandMetrics.MetricNameLatency, sum, tagMap, now);
                            tagMap[tagName] = "min";
                            // metricLog.log(commandMetrics.MetricNameLatency, min, tagMap, now);
                            // tagMap[tagName] = "max";
                            //metricLog.log(commandMetrics.MetricNameLatency, max, tagMap, now);
                            tagMap.Remove(tagName);

                            tagName = "distribution";
                            logLatencyDistribution(commandMetrics, 0, 10, tagName, "0 ~ 10ms");
                            logLatencyDistribution(commandMetrics, 10, 50, tagName, "10 ~ 50ms");
                            logLatencyDistribution(commandMetrics, 50, 200, tagName, "50 ~ 200ms");
                            logLatencyDistribution(commandMetrics, 200, 500, tagName, "200 ~ 500ms");
                            logLatencyDistribution(commandMetrics, 500, 1000, tagName, "500ms ~ 1s");
                            logLatencyDistribution(commandMetrics, 1000, 5 * 1000, tagName, "1 ~ 5s");
                            logLatencyDistribution(commandMetrics, 5 * 1000, 10 * 1000, tagName, "5 ~ 10s");
                            logLatencyDistribution(commandMetrics, 10 * 1000, 30 * 1000, tagName, "10 ~ 30s");
                            logLatencyDistribution(commandMetrics, 30 * 1000, 100 * 1000, tagName, "30 ~ 100s");
                            logLatencyDistribution(commandMetrics, 100 * 1000, null, tagName, ">= 100s");
                            if (tagMap.ContainsKey(tagName))
                            {
                                tagMap.Remove(tagName);
                            }

                            tagName = "percentile";
                            logLatencyPencentile(commandMetrics, 0, tagName, "0");
                            logLatencyPencentile(commandMetrics, 25, tagName, "25");
                            logLatencyPencentile(commandMetrics, 50, tagName, "50");
                            logLatencyPencentile(commandMetrics, 75, tagName, "75");
                            logLatencyPencentile(commandMetrics, 90, tagName, "90");
                            logLatencyPencentile(commandMetrics, 95, tagName, "95");
                            logLatencyPencentile(commandMetrics, 99, tagName, "99");
                            logLatencyPencentile(commandMetrics, 99.5, tagName, "99.5");
                            logLatencyPencentile(commandMetrics, 100, tagName, "100");
                            if (tagMap.ContainsKey(tagName))
                            {
                                tagMap.Remove(tagName);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        log.Error("Fail to send metrics to backend!", ex, new Dictionary <string, string>()
                        {
                            { "ErrorCode", "FXD300009" }
                        });
                    }
                }
            }
            catch (Exception ex)
            {
                log.Error("Fail to send metrics to backend!", ex, new Dictionary <string, string>()
                {
                    { "ErrorCode", "FXD300009" }
                });
            }
        }