/// <summary>
/// This method reads the training data and calculates a variety of statistical data about each column.
/// </summary>
public static MetricStatistics UnderstandData(DataSetColumns dataSetColumns)
{
//Calculate and output the statistics for your pleasure
var stats = MetricStatistics.CalculateMetricStatisticsAsync(dataSetColumns.TimeSeries, DateTime.Now, TimeSpan.FromSeconds(1)).Result;
Console.WriteLine("Time Series Statistics:");
Console.WriteLine(stats.ToString());
return(stats);
}
/// <summary>
/// Computes statistics for a data serie (metric)
/// </summary>
/// <param name="values">Serie of data</param>
/// <param name="startTime">Begining of the time period the statistics are computed for</param>
/// <param name="duration">Length of the time period the statistics are computed for</param>
/// <returns>Computed statistics</returns>
public static Task <MetricStatistics> CalculateMetricStatisticsAsync(double[] values, DateTime startTime, TimeSpan duration)
{
if (values == null || values.Length < RequiredValuesCountForComputation)
{
throw new ArgumentException($"{nameof(values)} is expected to be not null and have a length >= {RequiredValuesCountForComputation}.");
}
return(Task.Run(() =>
{
var retval = new MetricStatistics
{
StartTime = startTime,
Duration = duration
};
var sortedArray = new double[values.Length];
Array.Copy(values, sortedArray, values.Length);
Array.Sort(sortedArray);
var slopeIntercept = CalculateSlope(values);
retval.Slope = slopeIntercept.Item1;
retval.SlopeYIntercept = slopeIntercept.Item2;
retval.MeanSquaredError = slopeIntercept.Item3;
retval.Maximum = sortedArray[sortedArray.Length - 1];
retval.Minimum = sortedArray[0];
retval.Count = sortedArray.Length;
retval.Sum = sortedArray.Sum();
retval.Mean = retval.Sum / retval.Count;
var quartiles = GetQuartiles(sortedArray);
retval.Median = quartiles.Item2;
retval.FirstQuartile = quartiles.Item1;
retval.ThirdQuartile = quartiles.Item3;
var stdDevSum = sortedArray.Sum(i => Math.Pow(i - retval.Mean, 2));
retval.VarianceSum = stdDevSum;
retval.Variance = stdDevSum / (retval.Count - 1);
retval.StandardDeviation = Math.Sqrt(retval.Variance);
retval.GesdValue = retval.StandardDeviation < double.Epsilon
? 0.0
: Math.Max(retval.Mean - retval.Minimum, retval.Maximum - retval.Mean) / retval.StandardDeviation;
var skewnessValue = values.Sum(d => Math.Pow(d - retval.Mean, 3));
var kurtosisValue = values.Sum(d => Math.Pow(d - retval.Mean, 4));
retval.Kurtosis = kurtosisValue / retval.Count / Math.Pow(retval.Variance, 2) - 3;
retval.Skewness = Math.Sqrt((long)retval.Count * ((long)retval.Count - 1)) / (retval.Count - 2) *
(skewnessValue / retval.Count) / Math.Pow(retval.StandardDeviation, 3);
//sorted array will now contain sorted values of absolute deviation
for (int i = 0; i < sortedArray.Length; i++)
{
sortedArray[i] = Math.Abs(sortedArray[i] - retval.Median);
}
Array.Sort(sortedArray);
retval.MedianAbsoluteDeviation = MedianOfSortedArrayRange(sortedArray, 0, sortedArray.Length - 1);
retval.Cardinality = values.Distinct().Count();
retval.CardinalityRatio = (double)retval.Cardinality / retval.Count;
return retval;
}));
}