public void ExecuteVerification(
[NotNull] string dataQualityXml,
[NotNull] string specificationName,
[NotNull] List <DataSource> dataSourceReplacements,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] string optionsXml,
double tileSize,
string outputDirectoryPath,
IssueRepositoryType issueRepositoryType = IssueRepositoryType.FileGdb,
bool ignoreConditionsForUnknownDatasets = true,
ITrackCancel cancelTracker = null)
{
IList <XmlQualitySpecification> qualitySpecifications;
XmlDataQualityDocument document = ReadXmlDocument(dataQualityXml,
out qualitySpecifications);
_msg.DebugFormat("Available specifications: {0}",
StringUtils.Concatenate(qualitySpecifications.Select(s => s.Name),
", "));
XmlVerificationOptions verificationOptions =
StringUtils.IsNotEmpty(optionsXml)
? VerificationOptionUtils.ReadOptionsXml(optionsXml)
: null;
IList <KeyValuePair <string, string> > properties =
new List <KeyValuePair <string, string> >();
Directory.CreateDirectory(outputDirectoryPath);
try
{
int errorCount;
int warningCount;
int exceptionCount;
int unusedExceptionObjectCount;
int rowCountWithStopConditions;
bool fulfilled = Verify(document, specificationName, dataSourceReplacements,
tileSize, outputDirectoryPath,
issueRepositoryType, properties,
verificationOptions,
areaOfInterest,
cancelTracker,
ignoreConditionsForUnknownDatasets,
out errorCount,
out warningCount,
out exceptionCount,
out unusedExceptionObjectCount,
out rowCountWithStopConditions);
}
catch (Exception)
{
StandaloneVerificationUtils.TryDeleteOutputDirectory(outputDirectoryPath);
throw;
}
}
public static List <string> WriteHtmlReports(
[NotNull] QualitySpecification qualitySpecification,
[NotNull] string directory,
[NotNull] IssueStatistics issueStatistics,
[NotNull] XmlVerificationReport verificationReport,
[NotNull] string verificationReportFileName,
[CanBeNull] string defaultReportTemplatePath = null,
[CanBeNull] XmlVerificationOptions options = null,
[CanBeNull] string issueGdbPath = null,
[CanBeNull] IList <string> issueMapFilePaths = null,
[CanBeNull] IList <string> qualitySpecificationReportFilePaths = null)
{
string defaultTemplateDirectory =
VerificationOptionUtils.GetDefaultTemplateDirectory(options);
var reportDefinitions = new List <HtmlReportDefinition>();
foreach (XmlHtmlReportOptions reportOptions in
VerificationOptionUtils.GetHtmlReportOptions(
options, defaultReportTemplatePath))
{
HtmlReportDefinition reportDefinition =
VerificationOptionUtils.GetReportDefinition(reportOptions,
defaultReportTemplatePath,
defaultTemplateDirectory);
if (!File.Exists(reportDefinition.TemplatePath))
{
_msg.WarnFormat("Template file does not exist: {0}",
reportOptions.TemplatePath);
continue;
}
reportDefinitions.Add(reportDefinition);
}
List <string> filePaths =
reportDefinitions.Select(d => Path.Combine(directory, d.FileName))
.ToList();
foreach (HtmlReportDefinition reportDefinition in reportDefinitions)
{
WriteHtmlReport(qualitySpecification, directory,
reportDefinition,
issueStatistics, verificationReport,
verificationReportFileName,
issueGdbPath,
issueMapFilePaths,
filePaths,
qualitySpecificationReportFilePaths);
}
return(filePaths);
}
public static Func <IObjectDataset, string> GetKeyFieldLookupFunction(
[CanBeNull] XmlVerificationOptions verificationOptions)
{
if (verificationOptions?.KeyFields == null)
{
return(null);
}
var keyFieldLookup = new KeyFieldLookup(verificationOptions.KeyFields);
return(keyFieldLookup.GetKeyField);
}
private bool Verify([NotNull] XmlDataQualityDocument document,
[NotNull] string specificationName,
[NotNull] IEnumerable <DataSource> dataSources,
double tileSize,
[NotNull] string directoryPath,
IssueRepositoryType issureRepositoryType,
[NotNull] IEnumerable <KeyValuePair <string, string> > properties,
[CanBeNull] XmlVerificationOptions verificationOptions,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] ITrackCancel trackCancel,
bool ignoreConditionsForUnknownDatasets,
out int errorCount,
out int warningCount,
out int exceptionCount,
out int unusedExceptionObjectCount,
out int rowCountWithStopConditions)
{
try
{
QualitySpecification qualitySpecification;
using (_msg.IncrementIndentation("Setting up quality specification"))
{
var modelFactory = new VerifiedModelFactory(
CreateSimpleWorkspaceContext, new SimpleVerifiedDatasetHarvester());
var datasetOpener = new SimpleDatasetOpener(new MasterDatabaseDatasetContext());
var factory =
new XmlBasedQualitySpecificationFactory(modelFactory, datasetOpener);
qualitySpecification = factory.CreateQualitySpecification(
document, specificationName, dataSources,
ignoreConditionsForUnknownDatasets);
}
return(Verify(qualitySpecification, tileSize, directoryPath,
issureRepositoryType, properties, verificationOptions,
areaOfInterest, trackCancel,
out errorCount,
out warningCount,
out exceptionCount,
out unusedExceptionObjectCount,
out rowCountWithStopConditions));
}
finally
{
GC.Collect();
}
}
public static ExceptionObjectRepository PrepareExceptionRepository(
[NotNull] QualitySpecification qualitySpecification,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] XmlVerificationOptions verificationOptions)
{
if (verificationOptions == null)
{
return(null);
}
IWorkspace workspace = VerificationOptionUtils.GetExceptionWorkspace(
verificationOptions);
if (workspace == null)
{
return(null);
}
bool useDbfFieldNames = WorkspaceUtils.IsShapefileWorkspace(workspace);
IIssueTableFieldManagement issueTableFields =
IssueTableFieldsFactory.GetIssueTableFields(true, useDbfFieldNames);
var result = new ExceptionObjectRepository(workspace, issueTableFields,
datasetContext, datasetResolver,
areaOfInterest?.Geometry);
List <QualityCondition> qualityConditions =
qualitySpecification.Elements.Select(element => element.QualityCondition)
.ToList();
InvolvedObjectsMatchCriteria involvedObjectsMatchCriteria =
VerificationOptionUtils.GetInvolvedObjectMatchCriteria(verificationOptions);
result.ReadExceptions(qualityConditions,
VerificationOptionUtils.GetDefaultShapeMatchCriterion(
verificationOptions),
VerificationOptionUtils.GetDefaultExceptionObjectStatus(
verificationOptions),
involvedObjectsMatchCriteria);
return(result);
}
public static List <string> WriteQualitySpecificationReport(
[NotNull] QualitySpecification qualitySpecification,
[NotNull] string directory,
[CanBeNull] string defaultTemplateFilePath,
[CanBeNull] XmlVerificationOptions options)
{
string defaultTemplateDirectory =
VerificationOptionUtils.GetDefaultTemplateDirectory(options);
var reportFilePaths = new List <string>();
foreach (XmlSpecificationReportOptions reportOptions in
VerificationOptionUtils.GetSpecificationReportOptions(options,
defaultTemplateFilePath))
{
SpecificationReportDefinition reportDefinition =
VerificationOptionUtils.GetSpecificationReportDefinition(reportOptions,
defaultTemplateFilePath,
defaultTemplateDirectory);
if (!File.Exists(reportDefinition.TemplatePath))
{
_msg.WarnFormat("Template file does not exist: {0}",
reportDefinition.TemplatePath);
continue;
}
string filePath = Path.Combine(directory, reportDefinition.FileName);
HtmlQualitySpecification model =
SpecificationReportUtils.CreateHtmlQualitySpecification(qualitySpecification,
reportDefinition);
SpecificationReportUtils.RenderHtmlQualitySpecification(
model,
reportDefinition.TemplatePath,
filePath);
reportFilePaths.Add(filePath);
}
return(reportFilePaths);
}
private bool Verify([NotNull] QualitySpecification qualitySpecification,
double tileSize,
[NotNull] string directory,
IssueRepositoryType issueRepositoryType,
[NotNull] IEnumerable <KeyValuePair <string, string> > properties,
[CanBeNull] XmlVerificationOptions verificationOptions,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] ITrackCancel trackCancel,
out int errorCount,
out int warningCount,
out int exceptionCount,
out int unusedExceptionObjectCount,
out int rowCountWithStopConditions)
{
Model primaryModel = StandaloneVerificationUtils.GetPrimaryModel(qualitySpecification);
Assert.NotNull(primaryModel, "no primary model found for quality specification");
// TODO disable quality conditions based on primaryModel and DatasetTestParameterValue.UsedAsReferenceData?
// TODO this would probably require an explicit identification of the primary data source
XmlVerificationReportBuilder xmlReportBuilder = GetReportBuilder();
var statisticsBuilder = new IssueStatisticsBuilder();
var datasetsCollector = new InvolvedDatasetsCollector();
var service = new StandaloneQualityVerificationService(
new MultiReportBuilder(xmlReportBuilder,
statisticsBuilder,
datasetsCollector),
(context) => new SimpleDatasetOpener(context));
// This context excludes geometric networks, terrains, topology, etc.:
var datasetContext = new MasterDatabaseDatasetContext();
var datasetResolver =
new QualityConditionObjectDatasetResolver(
new MasterDatabaseWorkspaceContextLookup());
string issueWorkspaceName =
VerificationOptionUtils.GetIssueWorkspaceName(verificationOptions);
string verificationReportFileName =
VerificationOptionUtils.GetXmlReportFileName(verificationOptions);
ISpatialReference spatialReference =
primaryModel.SpatialReferenceDescriptor?.SpatialReference;
var issueGdbWritten = false;
bool fulfilled;
List <string> htmlReportFilePaths;
List <string> specificationReportFilePaths;
string gdbPath = null;
Func <IObjectDataset, string> getKeyField =
StandaloneVerificationUtils.GetKeyFieldLookupFunction(verificationOptions);
ExceptionObjectRepository exceptionObjectRepository =
StandaloneVerificationUtils.PrepareExceptionRepository(
qualitySpecification, datasetContext, datasetResolver, areaOfInterest,
verificationOptions);
using (IIssueRepository issueRepository =
ExternalIssueRepositoryUtils.GetIssueRepository(
directory, issueWorkspaceName, spatialReference, issueRepositoryType,
addExceptionFields: true))
{
fulfilled = service.Verify(qualitySpecification, datasetContext, datasetResolver,
issueRepository, exceptionObjectRepository, tileSize,
getKeyField,
areaOfInterest, trackCancel,
out errorCount,
out warningCount,
out rowCountWithStopConditions);
if (issueRepository != null)
{
issueGdbWritten = true;
gdbPath = ((IWorkspace)issueRepository.FeatureWorkspace).PathName;
_msg.InfoFormat("Issues written to {0}", gdbPath);
issueRepository.CreateIndexes(GetForSubProcess(trackCancel),
ignoreErrors: true);
}
using (_msg.IncrementIndentation("Documenting verification results..."))
{
XmlVerificationReport verificationReport = GetVerificationReport(
xmlReportBuilder, qualitySpecification, properties);
string verificationReportPath = Path.Combine(directory,
verificationReportFileName);
XmlUtils.Serialize(verificationReport, verificationReportPath);
_msg.InfoFormat("Verification report written to {0}", verificationReportPath);
IssueStatistics issueStatistics = statisticsBuilder.IssueStatistics;
if (issueRepository != null)
{
var issueStatisticsWriter =
new IssueStatisticsWriter(issueRepository.FeatureWorkspace);
issueStatisticsWriter.WriteStatistics(issueStatistics);
if (spatialReference != null &&
areaOfInterest != null &&
!areaOfInterest.IsEmpty)
{
var aoiWriter =
new AreaOfInterestWriter(issueRepository.FeatureWorkspace);
aoiWriter.WriteAreaOfInterest(areaOfInterest, spatialReference);
}
}
specificationReportFilePaths =
StandaloneVerificationUtils.WriteQualitySpecificationReport(
qualitySpecification, directory, _qualitySpecificationTemplatePath,
verificationOptions);
htmlReportFilePaths = StandaloneVerificationUtils.WriteHtmlReports(
qualitySpecification, directory, issueStatistics, verificationReport,
verificationReportFileName, _htmlReportTemplatePath, verificationOptions,
issueGdbWritten ? gdbPath : null,
null, specificationReportFilePaths);
}
}
GC.Collect();
GC.WaitForPendingFinalizers();
if (htmlReportFilePaths.Count > 0)
{
using (_msg.IncrementIndentation(htmlReportFilePaths.Count == 1
? "Html report:"
: "Html reports:"))
{
foreach (string path in htmlReportFilePaths)
{
_msg.Info(path);
}
}
}
if (specificationReportFilePaths.Count > 0)
{
using (_msg.IncrementIndentation(specificationReportFilePaths.Count == 1
? "Quality specification report:"
: "Quality specification reports:"))
{
foreach (string path in specificationReportFilePaths)
{
_msg.Info(path);
}
}
}
if (exceptionObjectRepository != null)
{
IExceptionStatistics stats = exceptionObjectRepository.ExceptionStatistics;
exceptionCount = stats.ExceptionCount;
unusedExceptionObjectCount = stats.UnusedExceptionObjectCount;
}
else
{
exceptionCount = 0;
unusedExceptionObjectCount = 0;
}
return(fulfilled);
}
