public override void Execute()
{
base.Execute();
var catas = _catalogues ?? BasicActivator
.SelectMany("Which catalogues do you want to extract metadata for", typeof(Catalogue),
BasicActivator.CoreChildProvider.AllCatalogues);
if (catas == null || !catas.Any())
{
return;
}
var outputDir = _outputDirectory ?? BasicActivator.SelectDirectory("Enter output directory");
if (outputDir == null)
{
return;
}
var template = _template;
if (template == null)
{
BasicActivator.Show("Pick a template");
template = BasicActivator.SelectFile("Enter metadata template (should have template values in it e.g. $Name, $Description etc)");
}
if (template == null)
{
return;
}
var fileNaming = _fileNaming;
if (fileNaming == null)
{
if (!BasicActivator.TypeText("File naming", "File Naming", 1000, "$Name" + template.Extension, out fileNaming, false))
{
return;
}
}
if (string.IsNullOrWhiteSpace(fileNaming))
{
return;
}
var reporter = new CustomMetadataReport(BasicActivator.RepositoryLocator)
{
NewlineSubstitution = _newlineSub,
CommaSubstitution = _commaSub
};
reporter.GenerateReport(catas.Cast <Catalogue>().OrderBy(c => c.Name).ToArray(), outputDir, template, fileNaming, _oneFile);
}
public void TestCustomMetadataReport_SingleCatalogue_DQEMetrics()
{
var cata = WhenIHaveA <Catalogue>();
cata.Name = "ffff";
cata.Description = null;
cata.SaveToDatabase();
var ei = WhenIHaveA <ExtractionInformation>();
ei.SelectSQL = "[blah]..[mydate]";
ei.SaveToDatabase();
cata.TimeCoverage_ExtractionInformation_ID = ei.ID;
cata.SaveToDatabase();
var template = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "template.md"));
var outDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "outDir"));
if (outDir.Exists)
{
outDir.Delete(true);
}
outDir.Create();
File.WriteAllText(template.FullName,
@"| Name | Desc| StartYear | EndYear | StartMonth | EndMonth | StartDay | EndDay | Range | TimeField |
| $Name | $Description | $DQE_StartYear | $DQE_EndYear | $DQE_StartMonth | $DQE_EndMonth | $DQE_StartDay | $DQE_EndDay | $DQE_StartYear-$DQE_EndYear | $TimeCoverage_ExtractionInformation |");
var reporter = new CustomMetadataReport(RepositoryLocator)
{
NewlineSubstitution = null
};
DateTime?ignore;
var moqDqe = new Mock <IDetermineDatasetTimespan>();
moqDqe.Setup((f) => f.GetMachineReadableTimespanIfKnownOf(cata, true, out ignore))
.Returns(new Tuple <DateTime?, DateTime?>(new DateTime(2001, 02, 01), new DateTime(2002, 04, 03)));
reporter.TimespanCalculator = moqDqe.Object;
reporter.GenerateReport(new[] { cata }, outDir, template, "$Name.md", false);
var outFile = Path.Combine(outDir.FullName, "ffff.md");
FileAssert.Exists(outFile);
var resultText = File.ReadAllText(outFile);
StringAssert.AreEqualIgnoringCase(@"| Name | Desc| StartYear | EndYear | StartMonth | EndMonth | StartDay | EndDay | Range | TimeField |
| ffff | | 2001 | 2002 | 02 | 04 | 01 | 03 | 2001-2002 | mydate |", resultText.TrimEnd());
}
public void TestCustomMetadataReport_Nullability_WithDQERun()
{
var ei1 = WhenIHaveA <ExtractionInformation>();
var ei2 = WhenIHaveA <ExtractionInformation>();
ei1.CatalogueItem.Catalogue.Name = "Cata1";
ei2.CatalogueItem.Catalogue.Name = "Cata2";
ei1.CatalogueItem.Name = "Cata1Col1";
ei2.CatalogueItem.Name = "Cata2Col1";
var template = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "template.md"));
var outDir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "outDir"));
if (outDir.Exists)
{
outDir.Delete(true);
}
outDir.Create();
File.WriteAllText(template.FullName,
@"# Welcome
We love data here, see our datasets:
$foreach Catalogue
## Catalogue '$Name'
$Description
Price: $30
Number of Records: $DQE_CountTotal
| Column | Description | Nullability | Correct | Missing | Wrong | Invalid | Null (inclusive) | Total |
$foreach CatalogueItem
| $Name | $Description | $DQE_PercentNull | $DQE_CountCorrect | $DQE_CountMissing | $DQE_CountWrong | $DQE_CountInvalidatesRow | $DQE_CountDBNull | $DQE_CountTotal |
$end
Accurate as of : $DQE_DateOfEvaluation
$end");
var cata1 = ei1.CatalogueItem.Catalogue;
var cata2 = ei2.CatalogueItem.Catalogue;
var reporter = new CustomMetadataReport(RepositoryLocator);
var eval1 = Mock.Of <Evaluation>();
var eval1_col1 = Mock.Of <ColumnState>();
eval1_col1.TargetProperty = "Cata1Col1";
eval1_col1.CountCorrect = 9;
eval1_col1.CountDBNull = 3; // note that this is seperate from the other counts. A value can be both null and correct.
eval1.ColumnStates = new ColumnState[] { eval1_col1 };
var eval2 = Mock.Of <Evaluation>();
var eval2_col1 = Mock.Of <ColumnState>();
eval2_col1.TargetProperty = "Cata2Col1";
eval2_col1.CountCorrect = 1;
eval2_col1.CountMissing = 2;
eval2_col1.CountWrong = 3;
eval2_col1.CountInvalidatesRow = 4;
eval2_col1.CountDBNull = 5;
eval2.ColumnStates = new ColumnState[] { eval2_col1 };
reporter.EvaluationCache.Add(cata1, eval1);
reporter.EvaluationCache.Add(cata2, eval2);
reporter.GenerateReport(new[] { cata1, cata2 }, outDir, template, "Datasets.md", true);
var outFile = Path.Combine(outDir.FullName, "Datasets.md");
FileAssert.Exists(outFile);
var resultText = File.ReadAllText(outFile);
StringAssert.AreEqualIgnoringCase(@"# Welcome
We love data here, see our datasets:
## Catalogue 'Cata1'
Price: $30
Number of Records: 9
| Column | Description | Nullability | Correct | Missing | Wrong | Invalid | Null (inclusive) | Total |
| Cata1Col1 | | 33% | 9 | 0 | 0 | 0 | 3 | 9 |
Accurate as of : 01/01/0001 00:00:00
## Catalogue 'Cata2'
Price: $30
Number of Records: 10
| Column | Description | Nullability | Correct | Missing | Wrong | Invalid | Null (inclusive) | Total |
| Cata2Col1 | | 50% | 1 | 2 | 3 | 4 | 5 | 10 |
Accurate as of : 01/01/0001 00:00:00", resultText.TrimEnd());
}