/// <summary>
/// Optionally registers an aggregate metric to provide additional aggregation metadata.
/// </summary>
public static void RegisterAggregateMetric(this TelemetryClient telemetryClient, string name, string p1Name = null, string p2Name = null, string p3Name = null, PercentileCalculation percentileCalculation = PercentileCalculation.DoNotCalculate)
{
if (string.IsNullOrWhiteSpace(name))
{
AggregateMetricsEventSource.Log.RegisterAggregateMetricInvalidMetricName();
return;
}
name = GetAggregateMetricName(name);
int registrationKey = AggregationSet.GetKey(telemetryClient, name);
if (!metricRegistrations.TryAdd(registrationKey, new AggregateMetricProperties()
{
P1Name = p1Name,
P2Name = p2Name,
P3Name = p3Name,
PercentileCalculation = percentileCalculation
}))
{
AggregateMetricsEventSource.Log.RegisterAggregateMetricDuplicateMetricName(name);
}
}
internal AggregationResult CalculateAggregation(PercentileCalculation percentileCalculation = PercentileCalculation.DoNotCalculate)
{
if (this.Count == 0)
{
return(new AggregationResult());
}
// Use sorted enumerables if we need percentiles.
IEnumerable <double> values;
if (percentileCalculation == PercentileCalculation.OrderByLargest)
{
values = this.OrderBy(i => i);
}
else if (percentileCalculation == PercentileCalculation.OrderBySmallest)
{
values = this.OrderByDescending(i => i);
}
else
{
values = this;
}
double first = values.First();
double sum = 0;
double min = first;
double max = first;
int p50Index = 0;
int p75Index = 0;
int p90Index = 0;
int p95Index = 0;
int p99Index = 0;
bool shouldCalculatePercentile = percentileCalculation != PercentileCalculation.DoNotCalculate;
if (shouldCalculatePercentile)
{
p50Index = this.GetPercentileNearestIndex(50);
p75Index = this.GetPercentileNearestIndex(75);
p90Index = this.GetPercentileNearestIndex(90);
p95Index = this.GetPercentileNearestIndex(95);
p99Index = this.GetPercentileNearestIndex(99);
}
var aggregation = new AggregationResult()
{
Count = this.Count
};
int valueIndex = 0;
foreach (double value in values)
{
if (value < min)
{
min = value;
}
else if (value > max)
{
max = value;
}
sum += value;
if (shouldCalculatePercentile)
{
// Note: This only works if sample size >= 100 (which we enforce with Constants.PercentileMinimumCount).
if (valueIndex == p50Index)
{
aggregation.P50 = value;
}
else if (valueIndex == p75Index)
{
aggregation.P75 = value;
}
else if (valueIndex == p90Index)
{
aggregation.P90 = value;
}
else if (valueIndex == p95Index)
{
aggregation.P95 = value;
}
else if (valueIndex == p99Index)
{
aggregation.P99 = value;
}
}
valueIndex++;
}
aggregation.Sum = sum;
aggregation.Min = min;
aggregation.Max = max;
aggregation.StdDev = Math.Sqrt(this.Average(v => Math.Pow(v - aggregation.Average, 2)));
return(aggregation);
}
/// <summary>
/// Optionally registers an aggregate metric to provide additional aggregation metadata.
/// </summary>
public static void RegisterAggregateMetric(this TelemetryClient telemetryClient, string name, string p1Name = null, string p2Name = null, string p3Name = null, PercentileCalculation percentileCalculation = PercentileCalculation.DoNotCalculate)
{
if (string.IsNullOrWhiteSpace(name))
{
AggregateMetricsEventSource.Log.RegisterAggregateMetricInvalidMetricName();
return;
}
name = GetAggregateMetricName(name);
int registrationKey = AggregationSet.GetKey(telemetryClient, name);
if (!metricRegistrations.TryAdd(registrationKey, new AggregateMetricProperties()
{
P1Name = p1Name,
P2Name = p2Name,
P3Name = p3Name,
PercentileCalculation = percentileCalculation
}))
{
AggregateMetricsEventSource.Log.RegisterAggregateMetricDuplicateMetricName(name);
}
}
public static ParserResults ParseData(string dataSetPath, List <DataSetAttribute> previousAttributes = null, bool convertContinuousValues = true)
{
var lines = File.ReadAllLines(dataSetPath);
List <DataSetValue> rows = new List <DataSetValue>();
foreach (var line in lines)
{
var split = line.Split(new [] { ',' });
List <int> values = new List <int>();
bool isFirst = true;
foreach (var s in split)
{
if (isFirst)
{
isFirst = false;
continue;
}
values.Add(int.Parse(s));
}
// Remove the last value as it is the class
int classValue = values[values.Count - 1];
values.RemoveAt(values.Count - 1);
rows.Add(new DataSetValue(values, classValue == 1));
}
List <DataSetAttribute> attributeList = previousAttributes;
if (previousAttributes == null)
{
int i = 1;
attributeList = new List <DataSetAttribute>()
{
new DataSetAttribute("LIMIT_BAL", null, 1),
new DataSetAttribute("SEX", new HashSet <int>()
{
1, 2
}, 2),
new DataSetAttribute("EDUCATION", new HashSet <int>()
{
0, 1, 2, 3, 4, 5, 6
}, 3),
new DataSetAttribute("MARRIAGE", new HashSet <int>()
{
0, 1, 2, 3
}, 4),
new DataSetAttribute("AGE", null, 5),
new DataSetAttribute("PAY_0", new HashSet <int>()
{
-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8
}, 6),
new DataSetAttribute("PAY_2", new HashSet <int>()
{
-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8
}, 7),
new DataSetAttribute("PAY_3", new HashSet <int>()
{
-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8
}, 8),
new DataSetAttribute("PAY_4", new HashSet <int>()
{
-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8
}, 9),
new DataSetAttribute("PAY_5", new HashSet <int>()
{
-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8
}, 10),
new DataSetAttribute("PAY_6", new HashSet <int>()
{
-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8
}, 11),
new DataSetAttribute("BILL_AMT1", null, 12),
new DataSetAttribute("BILL_AMT2", null, 13),
new DataSetAttribute("BILL_AMT3", null, 14),
new DataSetAttribute("BILL_AMT4", null, 15),
new DataSetAttribute("BILL_AMT5", null, 16),
new DataSetAttribute("BILL_AMT6", null, 17),
new DataSetAttribute("PAY_AMT1", null, 18),
new DataSetAttribute("PAY_AMT2", null, 19),
new DataSetAttribute("PAY_AMT3", null, 20),
new DataSetAttribute("PAY_AMT4", null, 21),
new DataSetAttribute("PAY_AMT5", null, 22),
new DataSetAttribute("PAY_AMT6", null, 23),
};
// Calculate percentiles for all numeric fields and classify them
foreach (var dataSetAttribute in attributeList)
{
if (dataSetAttribute.PossibleValues == null)
{
dataSetAttribute.IsContinuous = true;
dataSetAttribute.Percentiles = PercentileCalculation.CalculatePercentiles(rows, dataSetAttribute.ValueIndex);
}
}
}
// Turn percentile values into buckets
foreach (var dataSetValue in rows)
{
foreach (var dataSetAttribute in attributeList)
{
if (dataSetAttribute.IsContinuous && convertContinuousValues)
{
int currentValue = dataSetValue.Values[dataSetAttribute.ValueIndex];
var percentiles = dataSetAttribute.Percentiles;
int tranformedValue = PercentileCalculation.CalculatePercentileBucket(dataSetAttribute, currentValue);
dataSetValue.Values[dataSetAttribute.ValueIndex] = tranformedValue;
dataSetAttribute.PossibleValues = new HashSet <int>(Enumerable.Range(0, Program.PercentilesToBreakIn + 1));
}
}
}
return(new ParserResults(attributeList, rows));
}
