public static IList <DataSource> GetDataSources(
[NotNull] XmlDataQualityDocument document,
[NotNull] XmlQualitySpecification xmlQualitySpecification)
{
Assert.ArgumentNotNull(document, nameof(document));
Assert.ArgumentNotNull(xmlQualitySpecification, nameof(xmlQualitySpecification));
IList <XmlQualityCondition> referencedConditions =
XmlDataQualityUtils.GetReferencedXmlQualityConditions(
document, new[] { xmlQualitySpecification })
.Select(p => p.Key)
.ToList();
bool hasUndefinedWorkspaceReference;
IList <XmlWorkspace> xmlWorkspaces = XmlDataQualityUtils.GetReferencedWorkspaces(
document, referencedConditions, out hasUndefinedWorkspaceReference);
var result = new List <DataSource>();
if (hasUndefinedWorkspaceReference)
{
result.Add(new DataSource(GetUniqueDefaultName(xmlWorkspaces),
DataSource.AnonymousId));
}
result.AddRange(
xmlWorkspaces.Select(xmlWorkspace => new DataSource(xmlWorkspace)));
// TODO sort based on reference count?
return(result);
}
GetReferencedTestDescriptorsByName(
[NotNull] IEnumerable <XmlQualityCondition> referencedConditions,
[NotNull] XmlDataQualityDocument document)
{
IDictionary <string, XmlTestDescriptor> xmlTestDescriptorsByName =
GetXmlTestDescriptorsByName(document);
var result = new Dictionary <string, TestDescriptor>(
StringComparer.OrdinalIgnoreCase);
foreach (XmlQualityCondition condition in referencedConditions)
{
string testDescriptorName = condition.TestDescriptorName;
if (testDescriptorName == null || result.ContainsKey(testDescriptorName))
{
continue;
}
XmlTestDescriptor xmlTestDescriptor;
if (!xmlTestDescriptorsByName.TryGetValue(testDescriptorName,
out xmlTestDescriptor))
{
throw new InvalidConfigurationException(
string.Format(
"Test descriptor {0}, referenced in quality condition {1}, not found",
testDescriptorName, condition.Name));
}
result.Add(testDescriptorName,
XmlDataQualityUtils.CreateTestDescriptor(xmlTestDescriptor));
}
return(result);
}
private static IDictionary <string, XmlTestDescriptor> GetXmlTestDescriptorsByName(
[NotNull] XmlDataQualityDocument document)
{
return(document.TestDescriptors?.ToDictionary(
descriptor => descriptor.Name,
StringComparer.OrdinalIgnoreCase)
?? new Dictionary <string, XmlTestDescriptor>());
}
public void ExecuteVerification(
[NotNull] string dataQualityXml,
[NotNull] string specificationName,
[NotNull] List <DataSource> dataSourceReplacements,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] string optionsXml,
double tileSize,
string outputDirectoryPath,
IssueRepositoryType issueRepositoryType = IssueRepositoryType.FileGdb,
bool ignoreConditionsForUnknownDatasets = true,
ITrackCancel cancelTracker = null)
{
IList <XmlQualitySpecification> qualitySpecifications;
XmlDataQualityDocument document = ReadXmlDocument(dataQualityXml,
out qualitySpecifications);
_msg.DebugFormat("Available specifications: {0}",
StringUtils.Concatenate(qualitySpecifications.Select(s => s.Name),
", "));
XmlVerificationOptions verificationOptions =
StringUtils.IsNotEmpty(optionsXml)
? VerificationOptionUtils.ReadOptionsXml(optionsXml)
: null;
IList <KeyValuePair <string, string> > properties =
new List <KeyValuePair <string, string> >();
Directory.CreateDirectory(outputDirectoryPath);
try
{
int errorCount;
int warningCount;
int exceptionCount;
int unusedExceptionObjectCount;
int rowCountWithStopConditions;
bool fulfilled = Verify(document, specificationName, dataSourceReplacements,
tileSize, outputDirectoryPath,
issueRepositoryType, properties,
verificationOptions,
areaOfInterest,
cancelTracker,
ignoreConditionsForUnknownDatasets,
out errorCount,
out warningCount,
out exceptionCount,
out unusedExceptionObjectCount,
out rowCountWithStopConditions);
}
catch (Exception)
{
StandaloneVerificationUtils.TryDeleteOutputDirectory(outputDirectoryPath);
throw;
}
}
public QualitySpecification CreateQualitySpecification(
[NotNull] XmlDataQualityDocument document,
[NotNull] string qualitySpecificationName,
[NotNull] IEnumerable <DataSource> dataSources)
{
const bool ignoreConditionsForUnknownDatasets = false;
return(CreateQualitySpecification(document, qualitySpecificationName,
dataSources,
ignoreConditionsForUnknownDatasets));
}
public QualitySpecification CreateQualitySpecification(
[NotNull] XmlDataQualityDocument document,
[NotNull] IEnumerable <DataSource> dataSources,
bool ignoreConditionsForUnknownDatasets)
{
KeyValuePair <XmlQualitySpecification, XmlDataQualityCategory> keyValuePair =
document.GetAllQualitySpecifications().Single();
string singleSpecificationName = Assert.NotNullOrEmpty(keyValuePair.Key.Name);
return(CreateQualitySpecification(document, singleSpecificationName, dataSources,
ignoreConditionsForUnknownDatasets));
}
private bool Verify([NotNull] XmlDataQualityDocument document,
[NotNull] string specificationName,
[NotNull] IEnumerable <DataSource> dataSources,
double tileSize,
[NotNull] string directoryPath,
IssueRepositoryType issureRepositoryType,
[NotNull] IEnumerable <KeyValuePair <string, string> > properties,
[CanBeNull] XmlVerificationOptions verificationOptions,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] ITrackCancel trackCancel,
bool ignoreConditionsForUnknownDatasets,
out int errorCount,
out int warningCount,
out int exceptionCount,
out int unusedExceptionObjectCount,
out int rowCountWithStopConditions)
{
try
{
QualitySpecification qualitySpecification;
using (_msg.IncrementIndentation("Setting up quality specification"))
{
var modelFactory = new VerifiedModelFactory(
CreateSimpleWorkspaceContext, new SimpleVerifiedDatasetHarvester());
var datasetOpener = new SimpleDatasetOpener(new MasterDatabaseDatasetContext());
var factory =
new XmlBasedQualitySpecificationFactory(modelFactory, datasetOpener);
qualitySpecification = factory.CreateQualitySpecification(
document, specificationName, dataSources,
ignoreConditionsForUnknownDatasets);
}
return(Verify(qualitySpecification, tileSize, directoryPath,
issureRepositoryType, properties, verificationOptions,
areaOfInterest, trackCancel,
out errorCount,
out warningCount,
out exceptionCount,
out unusedExceptionObjectCount,
out rowCountWithStopConditions));
}
finally
{
GC.Collect();
}
}
private static XmlDataQualityDocument ReadXmlDocument(
[NotNull] string xml,
[NotNull] out IList <XmlQualitySpecification> qualitySpecifications)
{
Assert.ArgumentNotNullOrEmpty(xml, nameof(xml));
XmlDataQualityDocument document = XmlDataQualityUtils.DeserializeXml(xml);
Assert.ArgumentCondition(document.GetAllQualitySpecifications().Any(),
"The document does not contain any quality specifications");
XmlDataQualityUtils.AssertUniqueQualitySpecificationNames(document);
XmlDataQualityUtils.AssertUniqueQualityConditionNames(document);
XmlDataQualityUtils.AssertUniqueTestDescriptorNames(document);
qualitySpecifications = document.GetAllQualitySpecifications()
.Select(p => p.Key)
.Where(qs => qs.Elements.Count > 0)
.ToList();
return(document);
}
public QualitySpecification CreateQualitySpecification(
[NotNull] XmlDataQualityDocument document,
[NotNull] string qualitySpecificationName,
[NotNull] IEnumerable <DataSource> dataSources,
bool ignoreConditionsForUnknownDatasets)
{
CultureInfo origCulture = Thread.CurrentThread.CurrentCulture;
try
{
Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
return(CreateQualitySpecificationCore(document,
qualitySpecificationName,
dataSources,
ignoreConditionsForUnknownDatasets));
}
finally
{
Thread.CurrentThread.CurrentCulture = origCulture;
}
}
public void CanCreateEmptyQualitySpecification()
{
var xmlQualitySpecification = new XmlQualitySpecification {
Name = "Empty"
};
var xmlDocument = new XmlDataQualityDocument();
xmlDocument.AddQualitySpecification(xmlQualitySpecification);
var modelFactory =
new VerifiedModelFactory(CreateWorkspaceContext,
new SimpleVerifiedDatasetHarvester());
var factory = new XmlBasedQualitySpecificationFactory(
modelFactory, new SimpleDatasetOpener(new MasterDatabaseDatasetContext()));
QualitySpecification qualitySpecification =
factory.CreateQualitySpecification(xmlDocument, xmlQualitySpecification.Name,
new DataSource[] { });
Assert.AreEqual(xmlQualitySpecification.Name, qualitySpecification.Name);
}
GetCategoryMap([NotNull] XmlDataQualityDocument document)
{
var result = new Dictionary <XmlDataQualityCategory, DataQualityCategory>();
if (document.Categories == null)
{
return(result);
}
foreach (XmlDataQualityCategory xmlCategory in document.Categories)
{
DataQualityCategory category =
XmlDataQualityUtils.CreateDataQualityCategory(xmlCategory, null);
result.Add(xmlCategory, category);
if (xmlCategory.SubCategories != null)
{
AddSubCategories(xmlCategory.SubCategories, category, result);
}
}
return(result);
}
private QualitySpecification CreateQualitySpecificationCore(
[NotNull] XmlDataQualityDocument document,
[NotNull] string qualitySpecificationName,
[NotNull] IEnumerable <DataSource> dataSources,
bool ignoreConditionsForUnknownDatasets)
{
Assert.ArgumentNotNull(document, nameof(document));
Assert.ArgumentNotNull(dataSources, nameof(dataSources));
Assert.ArgumentCondition(document.GetAllQualitySpecifications().Any(),
"The document must contain at least one quality specification");
XmlDataQualityUtils.AssertUniqueWorkspaceIds(document);
XmlDataQualityUtils.AssertUniqueTestDescriptorNames(document);
XmlDataQualityUtils.AssertUniqueQualityConditionNames(document);
XmlDataQualityUtils.AssertUniqueQualitySpecificationNames(document);
XmlDataQualityUtils.AssertUniqueQualifiedCategoryNames(document);
XmlDataQualityCategory xmlSpecificationCategory;
XmlQualitySpecification xmlQualitySpecification =
XmlDataQualityUtils.FindXmlQualitySpecification(document,
qualitySpecificationName,
out xmlSpecificationCategory);
Assert.ArgumentCondition(xmlQualitySpecification != null,
"Specification '{0} not found in document",
qualitySpecificationName);
XmlDataQualityUtils.AssertUniqueElementNames(xmlQualitySpecification);
IDictionary <XmlDataQualityCategory, DataQualityCategory> categoryMap =
GetCategoryMap(document);
IList <KeyValuePair <XmlQualityCondition, XmlDataQualityCategory> >
referencedXmlConditionPairs =
XmlDataQualityUtils.GetReferencedXmlQualityConditions(
document, new[] { xmlQualitySpecification })
.ToList();
IList <XmlQualityCondition> referencedXmlConditions =
referencedXmlConditionPairs.Select(p => p.Key)
.ToList();
IList <XmlWorkspace> referencedXmlWorkspaces =
XmlDataQualityUtils.GetReferencedWorkspaces(document,
referencedXmlConditions);
IDictionary <string, Model> modelsByWorkspaceId = GetModelsByWorkspaceId(
referencedXmlWorkspaces, dataSources, referencedXmlConditions);
IDictionary <string, TestDescriptor> testDescriptorsByName =
GetReferencedTestDescriptorsByName(referencedXmlConditions, document);
var qualityConditions = new Dictionary <string, QualityCondition>(
StringComparer.OrdinalIgnoreCase);
Func <string, IList <Dataset> > getDatasetsByName = name => new List <Dataset>();
foreach (
KeyValuePair <XmlQualityCondition, XmlDataQualityCategory> pair in
referencedXmlConditionPairs)
{
XmlQualityCondition xmlCondition = pair.Key;
XmlDataQualityCategory xmlConditionCategory = pair.Value;
DataQualityCategory conditionCategory = xmlConditionCategory != null
? categoryMap[xmlConditionCategory]
: null;
ICollection <XmlDatasetTestParameterValue> unknownDatasetParameters;
QualityCondition qualityCondition = XmlDataQualityUtils.CreateQualityCondition(
xmlCondition,
testDescriptorsByName[xmlCondition.TestDescriptorName],
modelsByWorkspaceId,
getDatasetsByName,
conditionCategory,
ignoreConditionsForUnknownDatasets,
out unknownDatasetParameters);
if (qualityCondition == null)
{
Assert.True(ignoreConditionsForUnknownDatasets,
"ignoreConditionsForUnknownDatasets");
Assert.True(unknownDatasetParameters.Count > 0,
"Unexpected number of unknown datasets");
_msg.WarnFormat(
unknownDatasetParameters.Count == 1
? "Quality condition '{0}' is ignored because the following dataset is not found: {1}"
: "Quality condition '{0}' is ignored because the following datasets are not found: {1}",
xmlCondition.Name,
XmlDataQualityUtils.ConcatenateUnknownDatasetNames(
unknownDatasetParameters,
modelsByWorkspaceId,
DataSource.AnonymousId));
}
else
{
qualityConditions.Add(qualityCondition.Name, qualityCondition);
}
}
DataQualityCategory specificationCategory =
xmlSpecificationCategory != null
? categoryMap[xmlSpecificationCategory]
: null;
return(XmlDataQualityUtils.CreateQualitySpecification(
qualityConditions, xmlQualitySpecification, specificationCategory,
ignoreConditionsForUnknownDatasets));
}
private void CanCreateQualitySpecificationCore()
{
var locator = TestDataLocator.Create("ProSuite", @"QA\TestData");
string catalogPath = locator.GetPath("QATestData.mdb");
var xmlCategory = new XmlDataQualityCategory {
Name = "Category A"
};
var xmlSubCategory = new XmlDataQualityCategory {
Name = "Category A.1"
};
var xmlSubSubCategory = new XmlDataQualityCategory {
Name = "Category A.1.1"
};
var xmlSubSubCategory2 = new XmlDataQualityCategory {
Name = "Category A.1.2"
};
xmlCategory.AddSubCategory(xmlSubCategory);
xmlSubCategory.AddSubCategory(xmlSubSubCategory);
xmlSubCategory.AddSubCategory(xmlSubSubCategory2);
var xmlQCon = new XmlQualityCondition
{
Name = "MinArea",
TestDescriptorName = _xmlTestDescriptorMinArea.Name
};
xmlQCon.ParameterValues.Add(new XmlDatasetTestParameterValue
{
TestParameterName = "polygonClass",
Value = "polygons",
WorkspaceId = _xmlWorkspace.ID
});
xmlQCon.ParameterValues.Add(new XmlScalarTestParameterValue
{
TestParameterName = "limit",
Value = "12.34"
});
var xmlQSpec = new XmlQualitySpecification {
Name = "qspec"
};
xmlQSpec.Elements.Add(new XmlQualitySpecificationElement
{
QualityConditionName = xmlQCon.Name
});
var xmlDocument = new XmlDataQualityDocument();
xmlDocument.AddCategory(xmlCategory);
xmlDocument.AddWorkspace(_xmlWorkspace);
//xmlDocument.AddQualitySpecification(xmlQSpec);
//xmlDocument.AddQualityCondition(xmlQCon);
xmlDocument.AddTestDescriptor(_xmlTestDescriptorMinArea);
xmlSubCategory.AddQualitySpecification(xmlQSpec);
xmlSubSubCategory.AddQualityCondition(xmlQCon);
var modelFactory =
new VerifiedModelFactory(CreateWorkspaceContext,
new SimpleVerifiedDatasetHarvester());
var factory = new XmlBasedQualitySpecificationFactory(
modelFactory, new SimpleDatasetOpener(new MasterDatabaseDatasetContext()));
var dataSource = new DataSource(_xmlWorkspace)
{
WorkspaceAsText = catalogPath
};
QualitySpecification qualitySpecification =
factory.CreateQualitySpecification(xmlDocument, xmlQSpec.Name,
new[] { dataSource });
Assert.NotNull(qualitySpecification.Category);
Assert.AreEqual(xmlQSpec.Name, qualitySpecification.Name);
Assert.AreEqual(1, qualitySpecification.Elements.Count);
if (qualitySpecification.Category != null)
{
Assert.AreEqual(xmlSubCategory.Name, qualitySpecification.Category.Name);
Assert.NotNull(qualitySpecification.Category.ParentCategory);
if (qualitySpecification.Category.ParentCategory != null)
{
Assert.AreEqual(xmlCategory.Name,
qualitySpecification.Category.ParentCategory.Name);
}
Assert.AreEqual(2, qualitySpecification.Category.SubCategories.Count);
}
QualityCondition qualityCondition =
qualitySpecification.Elements[0].QualityCondition;
Assert.AreEqual(xmlQCon.Name, qualityCondition.Name);
Assert.NotNull(qualityCondition.Category);
if (qualityCondition.Category != null)
{
Assert.AreEqual(xmlSubSubCategory.Name, qualityCondition.Category.Name);
Assert.NotNull(qualityCondition.Category.ParentCategory);
if (qualityCondition.Category.ParentCategory != null)
{
Assert.AreEqual(xmlSubCategory.Name,
qualityCondition.Category.ParentCategory.Name);
}
}
var value =
(ScalarTestParameterValue)qualityCondition.GetParameterValues("limit")[0];
Assert.AreEqual(12.34, value.GetValue(typeof(double)));
}
public void CanCreateQualitySpecificationIgnoringUnkownDatasets()
{
var locator = TestDataLocator.Create("ProSuite", @"QA\TestData");
string catalogPath = locator.GetPath("QATestData.mdb");
var xmlQCon = new XmlQualityCondition
{
Name = "Simple",
TestDescriptorName = _xmlTestDescriptorSimple.Name
};
xmlQCon.ParameterValues.Add(new XmlDatasetTestParameterValue
{
TestParameterName = "featureClass",
Value = "UNKNOWN",
WorkspaceId = _xmlWorkspace.ID
});
var xmlQSpec = new XmlQualitySpecification {
Name = "qspec"
};
xmlQSpec.Elements.Add(new XmlQualitySpecificationElement
{
QualityConditionName = xmlQCon.Name
});
var xmlDocument = new XmlDataQualityDocument();
xmlDocument.AddWorkspace(_xmlWorkspace);
xmlDocument.AddQualitySpecification(xmlQSpec);
xmlDocument.AddQualityCondition(xmlQCon);
xmlDocument.AddTestDescriptor(_xmlTestDescriptorSimple);
var modelFactory =
new VerifiedModelFactory(CreateWorkspaceContext,
new SimpleVerifiedDatasetHarvester());
var factory = new XmlBasedQualitySpecificationFactory(
modelFactory, new SimpleDatasetOpener(new MasterDatabaseDatasetContext()));
var dataSource = new DataSource(_xmlWorkspace)
{
WorkspaceAsText = catalogPath
};
const bool ignoreConditionsForUnknownDatasets = true;
QualitySpecification qualitySpecification;
try
{
qualitySpecification =
factory.CreateQualitySpecification(xmlDocument, xmlQSpec.Name,
new[] { dataSource },
ignoreConditionsForUnknownDatasets);
}
catch (Exception)
{
// TODO: Move test data to different format
if (EnvironmentUtils.Is64BitProcess)
{
Console.WriteLine("Expected exception: PGDB is not supported on x64");
return;
}
throw;
}
Assert.AreEqual(xmlQSpec.Name, qualitySpecification.Name);
Assert.AreEqual(0, qualitySpecification.Elements.Count);
}
