public void ExecuteVerification( [NotNull] string dataQualityXml, [NotNull] string specificationName, [NotNull] List <DataSource> dataSourceReplacements, [CanBeNull] AreaOfInterest areaOfInterest, [CanBeNull] string optionsXml, double tileSize, string outputDirectoryPath, IssueRepositoryType issueRepositoryType = IssueRepositoryType.FileGdb, bool ignoreConditionsForUnknownDatasets = true, ITrackCancel cancelTracker = null) { IList <XmlQualitySpecification> qualitySpecifications; XmlDataQualityDocument document = ReadXmlDocument(dataQualityXml, out qualitySpecifications); _msg.DebugFormat("Available specifications: {0}", StringUtils.Concatenate(qualitySpecifications.Select(s => s.Name), ", ")); XmlVerificationOptions verificationOptions = StringUtils.IsNotEmpty(optionsXml) ? VerificationOptionUtils.ReadOptionsXml(optionsXml) : null; IList <KeyValuePair <string, string> > properties = new List <KeyValuePair <string, string> >(); Directory.CreateDirectory(outputDirectoryPath); try { int errorCount; int warningCount; int exceptionCount; int unusedExceptionObjectCount; int rowCountWithStopConditions; bool fulfilled = Verify(document, specificationName, dataSourceReplacements, tileSize, outputDirectoryPath, issueRepositoryType, properties, verificationOptions, areaOfInterest, cancelTracker, ignoreConditionsForUnknownDatasets, out errorCount, out warningCount, out exceptionCount, out unusedExceptionObjectCount, out rowCountWithStopConditions); } catch (Exception) { StandaloneVerificationUtils.TryDeleteOutputDirectory(outputDirectoryPath); throw; } }
public static List <string> WriteHtmlReports( [NotNull] QualitySpecification qualitySpecification, [NotNull] string directory, [NotNull] IssueStatistics issueStatistics, [NotNull] XmlVerificationReport verificationReport, [NotNull] string verificationReportFileName, [CanBeNull] string defaultReportTemplatePath = null, [CanBeNull] XmlVerificationOptions options = null, [CanBeNull] string issueGdbPath = null, [CanBeNull] IList <string> issueMapFilePaths = null, [CanBeNull] IList <string> qualitySpecificationReportFilePaths = null) { string defaultTemplateDirectory = VerificationOptionUtils.GetDefaultTemplateDirectory(options); var reportDefinitions = new List <HtmlReportDefinition>(); foreach (XmlHtmlReportOptions reportOptions in VerificationOptionUtils.GetHtmlReportOptions( options, defaultReportTemplatePath)) { HtmlReportDefinition reportDefinition = VerificationOptionUtils.GetReportDefinition(reportOptions, defaultReportTemplatePath, defaultTemplateDirectory); if (!File.Exists(reportDefinition.TemplatePath)) { _msg.WarnFormat("Template file does not exist: {0}", reportOptions.TemplatePath); continue; } reportDefinitions.Add(reportDefinition); } List <string> filePaths = reportDefinitions.Select(d => Path.Combine(directory, d.FileName)) .ToList(); foreach (HtmlReportDefinition reportDefinition in reportDefinitions) { WriteHtmlReport(qualitySpecification, directory, reportDefinition, issueStatistics, verificationReport, verificationReportFileName, issueGdbPath, issueMapFilePaths, filePaths, qualitySpecificationReportFilePaths); } return(filePaths); }
public static Func <IObjectDataset, string> GetKeyFieldLookupFunction( [CanBeNull] XmlVerificationOptions verificationOptions) { if (verificationOptions?.KeyFields == null) { return(null); } var keyFieldLookup = new KeyFieldLookup(verificationOptions.KeyFields); return(keyFieldLookup.GetKeyField); }
private bool Verify([NotNull] XmlDataQualityDocument document, [NotNull] string specificationName, [NotNull] IEnumerable <DataSource> dataSources, double tileSize, [NotNull] string directoryPath, IssueRepositoryType issureRepositoryType, [NotNull] IEnumerable <KeyValuePair <string, string> > properties, [CanBeNull] XmlVerificationOptions verificationOptions, [CanBeNull] AreaOfInterest areaOfInterest, [CanBeNull] ITrackCancel trackCancel, bool ignoreConditionsForUnknownDatasets, out int errorCount, out int warningCount, out int exceptionCount, out int unusedExceptionObjectCount, out int rowCountWithStopConditions) { try { QualitySpecification qualitySpecification; using (_msg.IncrementIndentation("Setting up quality specification")) { var modelFactory = new VerifiedModelFactory( CreateSimpleWorkspaceContext, new SimpleVerifiedDatasetHarvester()); var datasetOpener = new SimpleDatasetOpener(new MasterDatabaseDatasetContext()); var factory = new XmlBasedQualitySpecificationFactory(modelFactory, datasetOpener); qualitySpecification = factory.CreateQualitySpecification( document, specificationName, dataSources, ignoreConditionsForUnknownDatasets); } return(Verify(qualitySpecification, tileSize, directoryPath, issureRepositoryType, properties, verificationOptions, areaOfInterest, trackCancel, out errorCount, out warningCount, out exceptionCount, out unusedExceptionObjectCount, out rowCountWithStopConditions)); } finally { GC.Collect(); } }
public static ExceptionObjectRepository PrepareExceptionRepository( [NotNull] QualitySpecification qualitySpecification, [NotNull] IDatasetContext datasetContext, [NotNull] IQualityConditionObjectDatasetResolver datasetResolver, [CanBeNull] AreaOfInterest areaOfInterest, [CanBeNull] XmlVerificationOptions verificationOptions) { if (verificationOptions == null) { return(null); } IWorkspace workspace = VerificationOptionUtils.GetExceptionWorkspace( verificationOptions); if (workspace == null) { return(null); } bool useDbfFieldNames = WorkspaceUtils.IsShapefileWorkspace(workspace); IIssueTableFieldManagement issueTableFields = IssueTableFieldsFactory.GetIssueTableFields(true, useDbfFieldNames); var result = new ExceptionObjectRepository(workspace, issueTableFields, datasetContext, datasetResolver, areaOfInterest?.Geometry); List <QualityCondition> qualityConditions = qualitySpecification.Elements.Select(element => element.QualityCondition) .ToList(); InvolvedObjectsMatchCriteria involvedObjectsMatchCriteria = VerificationOptionUtils.GetInvolvedObjectMatchCriteria(verificationOptions); result.ReadExceptions(qualityConditions, VerificationOptionUtils.GetDefaultShapeMatchCriterion( verificationOptions), VerificationOptionUtils.GetDefaultExceptionObjectStatus( verificationOptions), involvedObjectsMatchCriteria); return(result); }
public static List <string> WriteQualitySpecificationReport( [NotNull] QualitySpecification qualitySpecification, [NotNull] string directory, [CanBeNull] string defaultTemplateFilePath, [CanBeNull] XmlVerificationOptions options) { string defaultTemplateDirectory = VerificationOptionUtils.GetDefaultTemplateDirectory(options); var reportFilePaths = new List <string>(); foreach (XmlSpecificationReportOptions reportOptions in VerificationOptionUtils.GetSpecificationReportOptions(options, defaultTemplateFilePath)) { SpecificationReportDefinition reportDefinition = VerificationOptionUtils.GetSpecificationReportDefinition(reportOptions, defaultTemplateFilePath, defaultTemplateDirectory); if (!File.Exists(reportDefinition.TemplatePath)) { _msg.WarnFormat("Template file does not exist: {0}", reportDefinition.TemplatePath); continue; } string filePath = Path.Combine(directory, reportDefinition.FileName); HtmlQualitySpecification model = SpecificationReportUtils.CreateHtmlQualitySpecification(qualitySpecification, reportDefinition); SpecificationReportUtils.RenderHtmlQualitySpecification( model, reportDefinition.TemplatePath, filePath); reportFilePaths.Add(filePath); } return(reportFilePaths); }
private bool Verify([NotNull] QualitySpecification qualitySpecification, double tileSize, [NotNull] string directory, IssueRepositoryType issueRepositoryType, [NotNull] IEnumerable <KeyValuePair <string, string> > properties, [CanBeNull] XmlVerificationOptions verificationOptions, [CanBeNull] AreaOfInterest areaOfInterest, [CanBeNull] ITrackCancel trackCancel, out int errorCount, out int warningCount, out int exceptionCount, out int unusedExceptionObjectCount, out int rowCountWithStopConditions) { Model primaryModel = StandaloneVerificationUtils.GetPrimaryModel(qualitySpecification); Assert.NotNull(primaryModel, "no primary model found for quality specification"); // TODO disable quality conditions based on primaryModel and DatasetTestParameterValue.UsedAsReferenceData? // TODO this would probably require an explicit identification of the primary data source XmlVerificationReportBuilder xmlReportBuilder = GetReportBuilder(); var statisticsBuilder = new IssueStatisticsBuilder(); var datasetsCollector = new InvolvedDatasetsCollector(); var service = new StandaloneQualityVerificationService( new MultiReportBuilder(xmlReportBuilder, statisticsBuilder, datasetsCollector), (context) => new SimpleDatasetOpener(context)); // This context excludes geometric networks, terrains, topology, etc.: var datasetContext = new MasterDatabaseDatasetContext(); var datasetResolver = new QualityConditionObjectDatasetResolver( new MasterDatabaseWorkspaceContextLookup()); string issueWorkspaceName = VerificationOptionUtils.GetIssueWorkspaceName(verificationOptions); string verificationReportFileName = VerificationOptionUtils.GetXmlReportFileName(verificationOptions); ISpatialReference spatialReference = primaryModel.SpatialReferenceDescriptor?.SpatialReference; var issueGdbWritten = false; bool fulfilled; List <string> htmlReportFilePaths; List <string> specificationReportFilePaths; string gdbPath = null; Func <IObjectDataset, string> getKeyField = StandaloneVerificationUtils.GetKeyFieldLookupFunction(verificationOptions); ExceptionObjectRepository exceptionObjectRepository = StandaloneVerificationUtils.PrepareExceptionRepository( qualitySpecification, datasetContext, datasetResolver, areaOfInterest, verificationOptions); using (IIssueRepository issueRepository = ExternalIssueRepositoryUtils.GetIssueRepository( directory, issueWorkspaceName, spatialReference, issueRepositoryType, addExceptionFields: true)) { fulfilled = service.Verify(qualitySpecification, datasetContext, datasetResolver, issueRepository, exceptionObjectRepository, tileSize, getKeyField, areaOfInterest, trackCancel, out errorCount, out warningCount, out rowCountWithStopConditions); if (issueRepository != null) { issueGdbWritten = true; gdbPath = ((IWorkspace)issueRepository.FeatureWorkspace).PathName; _msg.InfoFormat("Issues written to {0}", gdbPath); issueRepository.CreateIndexes(GetForSubProcess(trackCancel), ignoreErrors: true); } using (_msg.IncrementIndentation("Documenting verification results...")) { XmlVerificationReport verificationReport = GetVerificationReport( xmlReportBuilder, qualitySpecification, properties); string verificationReportPath = Path.Combine(directory, verificationReportFileName); XmlUtils.Serialize(verificationReport, verificationReportPath); _msg.InfoFormat("Verification report written to {0}", verificationReportPath); IssueStatistics issueStatistics = statisticsBuilder.IssueStatistics; if (issueRepository != null) { var issueStatisticsWriter = new IssueStatisticsWriter(issueRepository.FeatureWorkspace); issueStatisticsWriter.WriteStatistics(issueStatistics); if (spatialReference != null && areaOfInterest != null && !areaOfInterest.IsEmpty) { var aoiWriter = new AreaOfInterestWriter(issueRepository.FeatureWorkspace); aoiWriter.WriteAreaOfInterest(areaOfInterest, spatialReference); } } specificationReportFilePaths = StandaloneVerificationUtils.WriteQualitySpecificationReport( qualitySpecification, directory, _qualitySpecificationTemplatePath, verificationOptions); htmlReportFilePaths = StandaloneVerificationUtils.WriteHtmlReports( qualitySpecification, directory, issueStatistics, verificationReport, verificationReportFileName, _htmlReportTemplatePath, verificationOptions, issueGdbWritten ? gdbPath : null, null, specificationReportFilePaths); } } GC.Collect(); GC.WaitForPendingFinalizers(); if (htmlReportFilePaths.Count > 0) { using (_msg.IncrementIndentation(htmlReportFilePaths.Count == 1 ? "Html report:" : "Html reports:")) { foreach (string path in htmlReportFilePaths) { _msg.Info(path); } } } if (specificationReportFilePaths.Count > 0) { using (_msg.IncrementIndentation(specificationReportFilePaths.Count == 1 ? "Quality specification report:" : "Quality specification reports:")) { foreach (string path in specificationReportFilePaths) { _msg.Info(path); } } } if (exceptionObjectRepository != null) { IExceptionStatistics stats = exceptionObjectRepository.ExceptionStatistics; exceptionCount = stats.ExceptionCount; unusedExceptionObjectCount = stats.UnusedExceptionObjectCount; } else { exceptionCount = 0; unusedExceptionObjectCount = 0; } return(fulfilled); }