private DimensionMetricPair SetData(Request input, string dimensionMnemonic, string dimensionValue)
{
var dimensionList = new DimensionValuesList();
var metricList = new MetricValuesList();
dimensionList.Add(new DimensionValues
{
Key = dimensionMnemonic,
Value = dimensionValue
});
foreach (var metric in input.Metrics)
{
if (_metricRepository.Exists(metric))
{
var comp = new MetricValues
{
Key = metric,
Value = GenerateData(_metricRepository.GetType(metric))
};
metricList.Add(comp);
}
}
return(new DimensionMetricPair
{
DimensionValues = dimensionList,
MetricValues = metricList
});
}
/// <summary>
/// Gets the metric values.
/// </summary>
/// <param name="dataLakeClient">The data lake client.</param>
/// <param name="testContent">Content of the test.</param>
/// <param name="sparkClient">The spark client.</param>
/// <param name="sparkClientSettings">The spark client settings.</param>
/// <returns>MetricValues.</returns>
private static MetricValues GetMetricValues(
DataLakeClient dataLakeClient,
SparkMetricsTestContent testContent,
SparkClient sparkClient,
SparkClientSettings sparkClientSettings,
CancellationToken cancellationToken)
{
string currentPath = testContent.GetCurrentStreamPath();
string previousPath = testContent.GetPreviousStreamPath();
var current = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
currentPath,
dataLakeClient,
cancellationToken);
var previous = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
previousPath,
dataLakeClient,
cancellationToken);
var metricValues = new MetricValues
{
Baseline = previous,
Current = current
};
return(metricValues);
}
internal static float GetValueFromMetricMap(IDictionary<string, object> metrics, string name, MetricValues type)
{
return (float)((Array)metrics[name]).GetValue((int)type);
}
/// <summary>
/// Processes the metrics correctness test.
/// </summary>
/// <param name="dataLakeClient">The data lake client.</param>
/// <param name="testContent">Content of the test.</param>
/// <param name="sparkClient">The spark client.</param>
/// <param name="sparkClientSettings">The spark client settings.</param>
/// <returns>SparkMetricsTestResult.</returns>
/// <exception cref="NotSupportedException">ComparisonType {testContent.ComparisonType}</exception>
private static List <MetricsTestResult> ProcessMetricsCorrectnessTest(
DataLakeClient dataLakeClient,
SparkMetricsTestContent testContent,
SparkClient sparkClient,
SparkClientSettings sparkClientSettings,
CancellationToken cancellationToken)
{
MetricValues metricValues = null;
switch (testContent.ComparisonType)
{
case ComparisonType.DayOverDay:
case ComparisonType.WeekOverWeek:
case ComparisonType.MonthOverMonth:
case ComparisonType.YearOverYear:
metricValues = GetMetricValues(
dataLakeClient,
testContent,
sparkClient,
sparkClientSettings,
cancellationToken);
break;
case ComparisonType.VarianceToTarget:
var metricValue = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
testContent.GetCurrentStreamPath(),
dataLakeClient,
cancellationToken);
testContent.NotebookParameters["cmdText"] = testContent.NotebookParameters["targetCmdText"];
var targetValue = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
testContent.TargetStreamPath != null ? testContent.GetCurrentTargetStreamPath() : testContent.GetCurrentStreamPath(),
dataLakeClient,
cancellationToken);
metricValues = new MetricValues
{
Baseline = targetValue,
Current = metricValue
};
break;
default:
throw new NotSupportedException($"ComparisonType {testContent.ComparisonType} not supported for SparkWorker");
}
var results = new List <MetricsTestResult>();
foreach (var threshold in testContent.Thresholds)
{
var result = new MetricsTestResult
{
ComparisonType = testContent.ComparisonType,
Date = testContent.Date,
BaselineMetricValue = metricValues.Baseline[threshold.Name],
MetricValue = metricValues.Current[threshold.Name],
LowerBoundThreshold = threshold.LowerBound,
UpperBoundThreshold = threshold.UpperBound,
PercentDiff = MetricValues.ComputePercentDiff(metricValues, threshold.Name),
PreviousDate = testContent.GetPreviousDate(),
TestName = testRunName,
TestRunId = testRunId,
MetricName = threshold.Name
};
results.Add(result);
}
return(results);
}
internal static float GetValueFromMetricMap(IDictionary <string, object> metrics, string name, MetricValues type)
{
return((float)((Array)metrics[name]).GetValue((int)type));
}
