support_saving_data_with_different_tags_and_returning_DataFrame_with_them() =>
Evaluate(_session, (context, repository) =>
{
repository.Save(new ResultKey(DATE_ONE, new Dictionary <string, string>(REGION_EU_AND_DATASET_NAME)),
context);
repository.Save(new ResultKey(DATE_TWO, new Dictionary <string, string>(REGION_NA_AND_DATASET_VERSION)),
context);
DataFrame analysisResultsAsDataFrame = repository.Load()
.GetSuccessMetricsAsDataFrame(_session, Enumerable.Empty <string>());
List <GenericRow> elements = new List <GenericRow>
{
new GenericRow(new object[] { "Dataset", "*", "Size", 4.0, DATE_ONE, "EU", "Some", null }),
new GenericRow(new object[] { "Column", "att1", "Completeness", 1.0, DATE_ONE, "EU", "Some", null }),
new GenericRow(new object[] { "Column", "item", "Distinctness", 1.0, DATE_ONE, "EU", "Some", null }),
new GenericRow(new object[]
{
"Multicolumn", "att1,att2", "Uniqueness", 0.25, DATE_ONE, "EU", "Some", null
}),
new GenericRow(new object[] { "Dataset", "*", "Size", 4.0, DATE_TWO, "NA", null, "2.0" }),
new GenericRow(
new object[] { "Column", "att1", "Completeness", 1.0, DATE_TWO, "NA", null, "2.0" }),
new GenericRow(
new object[] { "Column", "item", "Distinctness", 1.0, DATE_TWO, "NA", null, "2.0" }),
new GenericRow(new object[]
{
"Multicolumn", "att1,att2", "Uniqueness", 0.25, DATE_TWO, "NA", null, "2.0"
})
};
StructType schema = new StructType(
new List <StructField>
{
new StructField("entity", new StringType()),
new StructField("instance", new StringType()),
new StructField("name", new StringType()),
new StructField("value", new DoubleType()),
new StructField("dataset_date", new LongType()),
new StructField("region", new StringType()),
new StructField("dataset_name", new StringType()),
new StructField("dataset_version", new StringType())
});
DataFrame df = _session.CreateDataFrame(elements, schema);
FixtureSupport.AssertSameRows(analysisResultsAsDataFrame, df, Option <ITestOutputHelper> .None);
});
public void save_and_retrieve_AnalyzerResults() =>
Evaluate(_session, (context, repository) =>
{
repository.Save(new ResultKey(DATE_ONE, new Dictionary <string, string>(REGION_EU)), context);
repository.Save(new ResultKey(DATE_TWO, new Dictionary <string, string>(REGION_NA)), context);
DataFrame analysisResultsAsDataFrame = repository.Load()
.After(DATE_ONE)
.GetSuccessMetricsAsDataFrame(_session, Enumerable.Empty <string>());
List <GenericRow> elements = new List <GenericRow>
{
new GenericRow(new object[] { "Dataset", "*", "Size", 4.0, DATE_ONE, "EU" }),
new GenericRow(new object[] { "Column", "att1", "Completeness", 1.0, DATE_ONE, "EU" }),
new GenericRow(new object[] { "Column", "item", "Distinctness", 1.0, DATE_ONE, "EU" }),
new GenericRow(new object[] { "Multicolumn", "att1,att2", "Uniqueness", 0.25, DATE_ONE, "EU" }),
new GenericRow(new object[] { "Dataset", "*", "Size", 4.0, DATE_TWO, "NA" }),
new GenericRow(new object[] { "Column", "att1", "Completeness", 1.0, DATE_TWO, "NA" }),
new GenericRow(new object[] { "Column", "item", "Distinctness", 1.0, DATE_TWO, "NA" }),
new GenericRow(new object[] { "Multicolumn", "att1,att2", "Uniqueness", 0.25, DATE_TWO, "NA" })
};
StructType schema = new StructType(
new List <StructField>
{
new StructField("entity", new StringType()),
new StructField("instance", new StringType()),
new StructField("name", new StringType()),
new StructField("value", new DoubleType()),
new StructField("dataset_date", new LongType()),
new StructField("region", new StringType())
});
DataFrame df = _session.CreateDataFrame(elements, schema);
FixtureSupport.AssertSameRows(analysisResultsAsDataFrame, df, Option <ITestOutputHelper> .None);
});
public void only_include_specifics_metrics_in_loaded_AnalysisResults_if_requested() =>
Evaluate(_session, (context, repository) =>
{
repository.Save(new ResultKey(DATE_ONE, new Dictionary <string, string>(REGION_EU)), context);
repository.Save(new ResultKey(DATE_TWO, new Dictionary <string, string>(REGION_NA)), context);
DataFrame analysisResultsAsDataFrame = repository.Load()
.After(DATE_TWO)
.ForAnalyzers(new List <IAnalyzer <IMetric> >
{
Initializers.Completeness("att1"), Initializers.Uniqueness(new[] { "att1", "att2" })
})
.GetSuccessMetricsAsDataFrame(_session, Enumerable.Empty <string>());
List <GenericRow> elements = new List <GenericRow>
{
new GenericRow(new object[] { "Column", "att1", "Completeness", 1.0, DATE_ONE, "EU" }),
new GenericRow(new object[] { "Multicolumn", "att1,att2", "Uniqueness", 0.25, DATE_ONE, "EU" }),
new GenericRow(new object[] { "Column", "att1", "Completeness", 1.0, DATE_ONE, "NA" }),
new GenericRow(new object[] { "Multicolumn", "att1,att2", "Uniqueness", 0.25, DATE_ONE, "NA" })
};
StructType schema = new StructType(
new List <StructField>
{
new StructField("entity", new StringType()),
new StructField("instance", new StringType()),
new StructField("name", new StringType()),
new StructField("value", new DoubleType()),
new StructField("dataset_date", new LongType()),
new StructField("region", new StringType())
});
DataFrame df = _session.CreateDataFrame(elements, schema);
FixtureSupport.AssertSameRows(analysisResultsAsDataFrame, df, new Option <ITestOutputHelper>(_helper));
});