public void should_execute_incremental_metrics_example() { DataFrame dataSetDE = LoadIncrementalMetricsData( new[] { new object[] { 1, "ManufacturerA", "DE" }, new object[] { 2, "ManufacturerB", "DE" }, new object[] { 2, "ManufacturerC", "DE" } }); DataFrame dataSetUS = LoadIncrementalMetricsData( new[] { new object[] { 3, "ManufacturerD", "US" }, new object[] { 4, "ManufacturerE", "US" }, new object[] { 5, "ManufacturerF", "US" } }); DataFrame dataSetCN = LoadIncrementalMetricsData( new[] { new object[] { 6, "ManufacturerG", "CN" }, new object[] { 7, "ManufacturerH", "CN" }, }); // We initialize a new check for the following data fields var check = new Check(CheckLevel.Warning, "generic check") .IsComplete("manufacturerName") .ContainsURL("manufacturerName", val => val == 0.0) .IsContainedIn("countryCode", new[] { "DE", "US", "CN" }); // We create a new Analysis instance with the corresponding RequiredAnalyzers defined in the check Analysis analysis = new Analysis(check.RequiredAnalyzers()); // We create a new in-memory state provider for each countryCode defined in the dataset InMemoryStateProvider deStates = new InMemoryStateProvider(); InMemoryStateProvider usStates = new InMemoryStateProvider(); InMemoryStateProvider cnStates = new InMemoryStateProvider(); // These call will store the resulting metrics in the separate states providers for each dataSet AnalysisRunner.Run(dataSetDE, analysis, saveStatesWith: deStates); AnalysisRunner.Run(dataSetUS, analysis, saveStatesWith: usStates); AnalysisRunner.Run(dataSetCN, analysis, saveStatesWith: cnStates); // Next, we are able to compute the metrics for the whole table from the partition states // This just aggregates the previously calculated metrics, it doesn't performs computation on the data AnalyzerContext tableMetrics = AnalysisRunner.RunOnAggregatedStates(dataSetDE.Schema(), analysis, new[] { deStates, usStates, cnStates }); // Lets now assume that a single partition changes. We only need to recompute the state of this // partition in order to update the metrics for the whole table. DataFrame updatedUsManufacturers = LoadIncrementalMetricsData(new[] { new object[] { 3, "ManufacturerDNew", "US" }, new object[] { 4, null, "US" }, new object[] { 5, "ManufacturerFNew http://clickme.com", "US" }, }); // Recompute state of partition InMemoryStateProvider updatedUsStates = new InMemoryStateProvider(); AnalysisRunner.Run(updatedUsManufacturers, analysis, updatedUsStates); // Recompute metrics for whole tables from states. We do not need to touch old data! AnalyzerContext updatedTableMetrics = AnalysisRunner.RunOnAggregatedStates(dataSetDE.Schema(), analysis, new[] { deStates, usStates, cnStates }); }
/// <summary> /// Runs all check groups and returns the verification result. Metrics are computed from aggregated states. Verification result includes all the metrics generated during the run /// </summary> /// <param name="schema">schema of the tabular data on which the checks should be verified</param> /// <param name="checks">A sequence of check objects to be executed</param> /// <param name="stateLoaders">loaders from which we retrieve the states to aggregate</param> /// <param name="requiredAnalysis">can be used to enforce the some metrics regardless of if there are constraints on them (optional)</param> /// <param name="saveStatesWith">persist resulting states for the configured analyzers (optional)</param> /// <param name="metricsRepository"></param> /// <param name="saveOrAppendResultsWithKey"></param> /// <returns>Result for every check including the overall status, detailed status for each constraints and all metrics produced</returns> public VerificationResult RunOnAggregatedStates( StructType schema, IEnumerable <Check> checks, IEnumerable <IStateLoader> stateLoaders, Analysis requiredAnalysis, Option <IStatePersister> saveStatesWith, Option <IMetricsRepository> metricsRepository, Option <ResultKey> saveOrAppendResultsWithKey) { Analysis analysis = requiredAnalysis.AddAnalyzers(checks.SelectMany(check => check.RequiredAnalyzers())); AnalyzerContext analysisResults = AnalysisRunner.RunOnAggregatedStates( schema, analysis, stateLoaders, saveStatesWith, metricsRepository, saveOrAppendResultsWithKey, new StorageLevel()); return(Evaluate(checks, analysisResults)); }