public void should_execute_incremental_metrics_example()
{
DataFrame dataSetDE = LoadIncrementalMetricsData(
new[] { new object[] { 1, "ManufacturerA", "DE" }, new object[] { 2, "ManufacturerB", "DE" },
new object[] { 2, "ManufacturerC", "DE" } });
DataFrame dataSetUS = LoadIncrementalMetricsData(
new[]
{
new object[] { 3, "ManufacturerD", "US" }, new object[] { 4, "ManufacturerE", "US" },
new object[] { 5, "ManufacturerF", "US" }
});
DataFrame dataSetCN = LoadIncrementalMetricsData(
new[] { new object[] { 6, "ManufacturerG", "CN" }, new object[] { 7, "ManufacturerH", "CN" }, });
// We initialize a new check for the following data fields
var check = new Check(CheckLevel.Warning, "generic check")
.IsComplete("manufacturerName")
.ContainsURL("manufacturerName", val => val == 0.0)
.IsContainedIn("countryCode", new[] { "DE", "US", "CN" });
// We create a new Analysis instance with the corresponding RequiredAnalyzers defined in the check
Analysis analysis = new Analysis(check.RequiredAnalyzers());
// We create a new in-memory state provider for each countryCode defined in the dataset
InMemoryStateProvider deStates = new InMemoryStateProvider();
InMemoryStateProvider usStates = new InMemoryStateProvider();
InMemoryStateProvider cnStates = new InMemoryStateProvider();
// These call will store the resulting metrics in the separate states providers for each dataSet
AnalysisRunner.Run(dataSetDE, analysis, saveStatesWith: deStates);
AnalysisRunner.Run(dataSetUS, analysis, saveStatesWith: usStates);
AnalysisRunner.Run(dataSetCN, analysis, saveStatesWith: cnStates);
// Next, we are able to compute the metrics for the whole table from the partition states
// This just aggregates the previously calculated metrics, it doesn't performs computation on the data
AnalyzerContext tableMetrics = AnalysisRunner.RunOnAggregatedStates(dataSetDE.Schema(), analysis,
new[] { deStates, usStates, cnStates });
// Lets now assume that a single partition changes. We only need to recompute the state of this
// partition in order to update the metrics for the whole table.
DataFrame updatedUsManufacturers = LoadIncrementalMetricsData(new[]
{
new object[] { 3, "ManufacturerDNew", "US" }, new object[] { 4, null, "US" },
new object[] { 5, "ManufacturerFNew http://clickme.com", "US" },
});
// Recompute state of partition
InMemoryStateProvider updatedUsStates = new InMemoryStateProvider();
AnalysisRunner.Run(updatedUsManufacturers, analysis, updatedUsStates);
// Recompute metrics for whole tables from states. We do not need to touch old data!
AnalyzerContext updatedTableMetrics = AnalysisRunner.RunOnAggregatedStates(dataSetDE.Schema(), analysis,
new[] { deStates, usStates, cnStates });
}
/// <summary>
/// Runs all check groups and returns the verification result. Metrics are computed from aggregated states. Verification result includes all the metrics generated during the run
/// </summary>
/// <param name="schema">schema of the tabular data on which the checks should be verified</param>
/// <param name="checks">A sequence of check objects to be executed</param>
/// <param name="stateLoaders">loaders from which we retrieve the states to aggregate</param>
/// <param name="requiredAnalysis">can be used to enforce the some metrics regardless of if there are constraints on them (optional)</param>
/// <param name="saveStatesWith">persist resulting states for the configured analyzers (optional)</param>
/// <param name="metricsRepository"></param>
/// <param name="saveOrAppendResultsWithKey"></param>
/// <returns>Result for every check including the overall status, detailed status for each constraints and all metrics produced</returns>
public VerificationResult RunOnAggregatedStates(
StructType schema,
IEnumerable <Check> checks,
IEnumerable <IStateLoader> stateLoaders,
Analysis requiredAnalysis,
Option <IStatePersister> saveStatesWith,
Option <IMetricsRepository> metricsRepository,
Option <ResultKey> saveOrAppendResultsWithKey)
{
Analysis analysis = requiredAnalysis.AddAnalyzers(checks.SelectMany(check => check.RequiredAnalyzers()));
AnalyzerContext analysisResults = AnalysisRunner.RunOnAggregatedStates(
schema,
analysis,
stateLoaders,
saveStatesWith,
metricsRepository, saveOrAppendResultsWithKey, new StorageLevel());
return(Evaluate(checks, analysisResults));
}
