private static IEnumerable <Dataset> GetVerifiedDatasets(
[NotNull] QualitySpecification qualitySpecification,
[NotNull] IDatasetContext datasetContext)
{
var datasets = new SimpleSet <Dataset>();
foreach (
QualitySpecificationElement qualitySpecificationElement in
qualitySpecification.Elements)
{
QualityCondition qualityCondition = qualitySpecificationElement.QualityCondition;
if (qualityCondition == null)
{
continue;
}
foreach (Dataset dataset in qualityCondition.GetDatasetParameterValues())
{
if (!datasets.Contains(dataset) && datasetContext.CanOpen(dataset))
{
datasets.Add(dataset);
}
}
}
return(datasets);
}
private IEnumerable <ITest> CreateTests(
[NotNull] IEnumerable <QualitySpecificationElement> qualitySpecificationElements,
[CanBeNull] IDatasetContext datasetContext,
out IDictionary <ITest, QualitySpecificationElement> qualityConditionsByTest)
{
var result = new List <ITest>();
qualityConditionsByTest = new Dictionary <ITest, QualitySpecificationElement>();
if (datasetContext == null)
{
return(result);
}
IOpenDataset datasetOpener = _openDatasetFactory(datasetContext);
foreach (QualitySpecificationElement element in qualitySpecificationElements)
{
QualityCondition condition = element.QualityCondition;
TestFactory factory = Assert.NotNull(
TestFactoryUtils.CreateTestFactory(condition),
$"Cannot create test factory for condition {condition.Name}");
foreach (ITest test in factory.CreateTests(datasetOpener))
{
result.Add(test);
qualityConditionsByTest.Add(test, element);
}
}
return(result);
}
private static IObject GetInvolvedObject(
[NotNull] InvolvedRow involvedRow,
[NotNull] QualityCondition qualityCondition,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[NotNull] out IObjectDataset objectDataset)
{
Assert.ArgumentNotNull(involvedRow, nameof(involvedRow));
Assert.ArgumentNotNull(qualityCondition, nameof(qualityCondition));
Assert.ArgumentCondition(!involvedRow.RepresentsEntireTable,
"involved row represents entire table");
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
//string gdbTableName = involvedRow.TableName;
//objectDataset = Assert.NotNull(
// QualityVerificationUtils.GetInvolvedObjectDataset(gdbTableName,
// qualityCondition,
// datasetResolver),
// "object dataset not found for {0}", involvedRow.TableName);
string gdbTableName = involvedRow.TableName;
objectDataset = Assert.NotNull(
datasetResolver.GetDatasetByGdbTableName(gdbTableName, qualityCondition),
"object dataset not found for {0}", involvedRow.TableName);
ITable table = datasetContext.OpenTable(objectDataset);
// TODO REFACTORMODEL revise null handling
Assert.NotNull(table, "Dataset not found in workspace: {0}", objectDataset.Name);
// TODO batch!
return((IObject)table.GetRow(involvedRow.OID));
}
public static IObjectClass TryOpenFromMasterDatabase(
[NotNull] IObjectDataset dataset, bool allowAlways = false)
{
IDatasetContext context = GetMasterDatabaseWorkspaceContext(dataset,
allowAlways);
return(context?.OpenObjectClass(dataset));
}
public static IRasterDataset TryOpenFromMasterDatabase(IDdxRasterDataset dataset,
bool allowAlways = false)
{
IDatasetContext context = GetMasterDatabaseWorkspaceContext(dataset,
allowAlways);
return(context?.OpenRasterDataset(dataset));
}
private static bool CanWriteToDataset([NotNull] IObjectDataset objectDataset,
[NotNull] IDatasetContext datasetContext)
{
IObjectClass objectClass = datasetContext.OpenObjectClass(objectDataset);
// TODO check for file geodatabase/personal geodatabase
return(objectClass != null && DatasetUtils.UserHasWriteAccess(objectClass));
}
public AlternateKeyConverterProvider(
[NotNull] IDictionary <Guid, QualityCondition> conditionsByGuid,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[NotNull] IDatasetContext datasetContext)
{
_conditionsByGuid = conditionsByGuid;
_datasetResolver = datasetResolver;
_datasetContext = datasetContext;
}
private static void InvalidateAllowedErrorsByInvolvedObjectState(
[NotNull] IObjectDataset involvedObjectDataset,
[NotNull] IDatasetContext datasetContext,
[NotNull] IDictionary <int, List <AllowedError> > allowedErrorsByInvolvedObjectID,
bool invalidateIfAnyInvolvedObjectChanged)
{
ICollection <int> existingInvolvedObjectIds;
try
{
existingInvolvedObjectIds = GetExistingInvolvedObjectIds(
involvedObjectDataset, datasetContext,
allowedErrorsByInvolvedObjectID,
invalidateIfAnyInvolvedObjectChanged);
}
catch (InvolvedTableOpenException e)
{
_msg.WarnFormat(e.Message);
// don't invalidate the allowed errors that involve a table which (currently?) can't be opened
return;
}
// invalidate the allowed errors if any of the involved objects no longer exists
foreach (
KeyValuePair <int, List <AllowedError> > involvedObjectId in
allowedErrorsByInvolvedObjectID)
{
int objectId = involvedObjectId.Key;
if (objectId <= 0)
{
// QUICK FIX for https://issuetracker02.eggits.net/browse/COM-171
// this seems to be the case for allowed errors without any involved objects
// TODO find out why
continue;
}
if (existingInvolvedObjectIds.Contains(objectId))
{
// the involved object still exists
continue;
}
// object with rowId does not exist any more in table,
// so it is deleted and an implicit change
foreach (AllowedError error in
allowedErrorsByInvolvedObjectID[objectId])
{
error.Invalidated = true;
}
}
}
public AlternateKeyConverter(
[NotNull] QualityCondition qualityCondition,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[NotNull] IDatasetContext datasetContext)
{
Assert.ArgumentNotNull(qualityCondition, nameof(qualityCondition));
Assert.ArgumentNotNull(datasetResolver, nameof(datasetResolver));
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
_qualityCondition = qualityCondition;
_datasetResolver = datasetResolver;
_datasetContext = datasetContext;
}
private static IssueDatasetWriter CreateIssueWriter(
[NotNull] IErrorDataset objectDataset,
[NotNull] IDatasetContext datasetContext,
[NotNull] IFieldIndexCache fieldIndexCache)
{
Assert.ArgumentNotNull(objectDataset, nameof(objectDataset));
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
Assert.ArgumentNotNull(fieldIndexCache, nameof(fieldIndexCache));
ITable table = datasetContext.OpenTable(objectDataset);
return(table == null
? null
: new IssueDatasetWriter(table, objectDataset, fieldIndexCache));
}
/// <summary>
/// Assigns allowedError.Invalidated = true for all errors, where the context changed.
/// In the base implementation, Invalidated is assigned to errors, where involved rows changed.
/// </summary>
/// <param name="allowedErrors">The allowed errors.</param>
/// <param name="qualityConditions">The quality conditions.</param>
/// <param name="datasetContext">The model context.</param>
/// <param name="invalidateIfAnyInvolvedObjectChanged">if set to <c>true</c> [invalidate if involved object changed].</param>
/// <param name="invalidateIfQualityConditionWasUpdated">if set to <c>true</c> [invalidate if quality condition was updated].</param>
public static void InvalidateAllowedErrors(
[NotNull] IEnumerable <AllowedError> allowedErrors,
[NotNull] IEnumerable <QualityCondition> qualityConditions,
[NotNull] IDatasetContext datasetContext,
bool invalidateIfAnyInvolvedObjectChanged,
bool invalidateIfQualityConditionWasUpdated)
{
Assert.ArgumentNotNull(allowedErrors, nameof(allowedErrors));
Assert.ArgumentNotNull(qualityConditions, nameof(qualityConditions));
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
ICollection <AllowedError> allowedErrorsCollection =
CollectionUtils.GetCollection(allowedErrors);
IDictionary <string, IDictionary <int, List <AllowedError> > >
errorsByMissingTableNameAndInvolvedObjectID;
IEnumerable <KeyValuePair <IObjectDataset, IDictionary <int, List <AllowedError> > > >
errorsByObjectDatasetAndInvolvedObjectID =
GetAllowedErrorsByObjectDatasetAndInvolvedObjectID(
allowedErrorsCollection,
out errorsByMissingTableNameAndInvolvedObjectID);
foreach (string tableName in errorsByMissingTableNameAndInvolvedObjectID.Keys)
{
_msg.WarnFormat("Table '{0}' referenced by allowed errors not registered",
tableName);
}
foreach (KeyValuePair <IObjectDataset, IDictionary <int, List <AllowedError> > > pair
in errorsByObjectDatasetAndInvolvedObjectID)
{
InvalidateAllowedErrorsByInvolvedObjectState(
pair.Key,
datasetContext,
pair.Value,
invalidateIfAnyInvolvedObjectChanged);
}
if (invalidateIfQualityConditionWasUpdated)
{
InvalidateAllowedErrorsWithUpdatedQualityCondition(allowedErrorsCollection,
qualityConditions);
}
}
public static ExceptionObjectRepository PrepareExceptionRepository(
[NotNull] QualitySpecification qualitySpecification,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] XmlVerificationOptions verificationOptions)
{
if (verificationOptions == null)
{
return(null);
}
IWorkspace workspace = VerificationOptionUtils.GetExceptionWorkspace(
verificationOptions);
if (workspace == null)
{
return(null);
}
bool useDbfFieldNames = WorkspaceUtils.IsShapefileWorkspace(workspace);
IIssueTableFieldManagement issueTableFields =
IssueTableFieldsFactory.GetIssueTableFields(true, useDbfFieldNames);
var result = new ExceptionObjectRepository(workspace, issueTableFields,
datasetContext, datasetResolver,
areaOfInterest?.Geometry);
List <QualityCondition> qualityConditions =
qualitySpecification.Elements.Select(element => element.QualityCondition)
.ToList();
InvolvedObjectsMatchCriteria involvedObjectsMatchCriteria =
VerificationOptionUtils.GetInvolvedObjectMatchCriteria(verificationOptions);
result.ReadExceptions(qualityConditions,
VerificationOptionUtils.GetDefaultShapeMatchCriterion(
verificationOptions),
VerificationOptionUtils.GetDefaultExceptionObjectStatus(
verificationOptions),
involvedObjectsMatchCriteria);
return(result);
}
public static bool DetermineIfIssuesCanBeWritten(
[NotNull] IDatasetContext datasetContext,
[NotNull] IEnumerable <IErrorDataset> errorDatasets,
[CanBeNull] NotificationCollection notifications = null)
{
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
Assert.ArgumentNotNull(errorDatasets, nameof(errorDatasets));
var unableToWrite = false;
List <IErrorDataset> list = errorDatasets.ToList();
if (list.Count == 0)
{
unableToWrite = true;
NotificationUtils.Add(notifications, "No error datasets exist in model");
}
foreach (IErrorDataset dataset in list)
{
if (dataset == null)
{
continue;
}
if (!CanWriteToDataset(dataset, datasetContext))
{
unableToWrite = true;
NotificationUtils.Add(notifications, "No write access to error dataset {0}",
dataset.Name);
}
}
// ok if there is at least dataset for rows without geometry
if (!list.OfType <ErrorTableDataset>().Any())
{
unableToWrite = true;
NotificationUtils.Add(notifications,
"Error dataset for errors without geometry is required");
}
return(!unableToWrite);
}
public static IEnumerable <IObjectClass> GetObjectClasses(
[NotNull] IEnumerable <Dataset> datasets,
[NotNull] IDatasetContext datasetContext)
{
var result = new List <IObjectClass>();
foreach (Dataset dataset in datasets)
{
if (dataset.Deleted || dataset.Model == null)
{
continue;
}
var objectDataset = dataset as IObjectDataset;
if (objectDataset == null)
{
continue;
}
IObjectClass objectClass;
try
{
objectClass = datasetContext.OpenObjectClass(objectDataset);
}
catch (Exception e)
{
_msg.DebugFormat("Error opening object class for dataset {0} ({1}): {2}",
dataset.Name, dataset.Model.Name, e.Message);
objectClass = null;
}
if (objectClass != null)
{
result.Add(objectClass);
}
}
return(result);
}
/// <summary>
/// Verifies the specified object classes.
/// </summary>
/// <param name="qualitySpecification">The quality specification to verify.</param>
/// <param name="datasetContext">The model context.</param>
/// <param name="datasetResolver">The resolver for getting the object dataset based on a table name, in the context of a quality condition</param>
/// <param name="issueRepository">The issue repository.</param>
/// <param name="exceptionObjectRepository">The exception object repository</param>
/// <param name="tileSize">Tile size for the quality verification.</param>
/// <param name="getKeyFieldName">Function for getting the key field name for an object dataset</param>
/// <param name="areaOfInterest">The area of interest for the verification (optional).</param>
/// <param name="trackCancel">The cancel tracker.</param>
/// <returns></returns>
public bool Verify(
[NotNull] QualitySpecification qualitySpecification,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[CanBeNull] IIssueRepository issueRepository,
[CanBeNull] IExceptionObjectRepository exceptionObjectRepository,
double tileSize,
[CanBeNull] Func <IObjectDataset, string> getKeyFieldName,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] ITrackCancel trackCancel)
{
int errorCount;
int warningCount;
int rowCountWithStopConditions;
return(Verify(qualitySpecification, datasetContext, datasetResolver,
issueRepository, exceptionObjectRepository, tileSize,
getKeyFieldName, areaOfInterest, trackCancel,
out errorCount,
out warningCount,
out rowCountWithStopConditions));
}
public BufferedIssueWriter(
[NotNull] IVerificationReportBuilder verificationReportBuilder,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[CanBeNull] IIssueRepository issueRepository,
[CanBeNull] Func <IObjectDataset, string> getAlternateKeyField,
int maximumErrorCount = 10000,
int maximumVertexCount = 1000000)
{
Assert.ArgumentNotNull(verificationReportBuilder,
nameof(verificationReportBuilder));
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
Assert.ArgumentNotNull(datasetResolver, nameof(datasetResolver));
_verificationReportBuilder = verificationReportBuilder;
_datasetContext = datasetContext;
_issueRepository = issueRepository;
_getAlternateKeyField = getAlternateKeyField;
_maximumErrorCount = maximumErrorCount;
_maximumVertexCount = maximumVertexCount;
_datasetResolver = datasetResolver;
}
private static ICollection <int> GetExistingInvolvedObjectIds(
[NotNull] IObjectDataset involvedObjectDataset,
[NotNull] IDatasetContext datasetContext,
[NotNull] IDictionary <int, List <AllowedError> > allowedErrorsByInvolvedObjectID,
bool invalidateIfAnyInvolvedObjectChanged)
{
var result = new SimpleSet <int>(allowedErrorsByInvolvedObjectID.Count);
IQueryFilter queryFilter;
int dateOfChangeFieldIndex;
ITable involvedTable = GetInvolvedTableAndQueryFilter(
involvedObjectDataset, datasetContext,
invalidateIfAnyInvolvedObjectChanged,
out queryFilter, out dateOfChangeFieldIndex);
const bool recycle = true;
foreach (IRow involvedRow in GdbQueryUtils.GetRowsInList(
involvedTable,
involvedTable.OIDFieldName,
allowedErrorsByInvolvedObjectID.Keys,
recycle, queryFilter))
{
// these are all involved rows in all the allowed errors.
int oid = involvedRow.OID;
result.TryAdd(oid);
if (dateOfChangeFieldIndex >= 0)
{
InvalidateIfAnyInvolvedObjectChanged(allowedErrorsByInvolvedObjectID[oid],
involvedRow,
dateOfChangeFieldIndex);
}
}
return(result);
}
private static IEnumerable <IRow> GetInvolvedRows(
[NotNull] IErrorObject errorObject,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionRepository qualityConditionRepository,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver)
{
IList <InvolvedRow> involved = RowParser.Parse(errorObject.RawInvolvedObjects);
QualityCondition qualityCondition = GetQualityCondition(errorObject,
qualityConditionRepository);
if (qualityCondition == null)
{
yield break;
}
foreach (KeyValuePair <string, IList <int> > pair in
GetInvolvedObjectIDsByTableName(involved))
{
string tableName = pair.Key;
IList <int> objectIDs = pair.Value;
IObjectDataset dataset =
datasetResolver.GetDatasetByInvolvedRowTableName(tableName, qualityCondition);
if (dataset == null)
{
continue;
}
ITable table;
try
{
table = datasetContext.OpenTable(dataset);
}
catch (Exception e)
{
_msg.WarnFormat("Error getting involved rows for table {0}: {1}", tableName,
e.Message);
continue;
}
if (table == null)
{
continue;
}
IEnumerable <IRow> rows = TryGetRows(table, objectIDs);
if (rows == null)
{
// error already logged in TryGetRows()
continue;
}
foreach (IRow row in rows)
{
if (GdbObjectUtils.IsDeleted(row))
{
// don't return deleted rows
continue;
}
yield return(row);
}
}
}
/// <summary>
/// Verifies the specified object classes.
/// </summary>
/// <param name="qualitySpecification">The quality specification to verify.</param>
/// <param name="datasetContext">The model context.</param>
/// <param name="datasetResolver">The resolver for getting the object dataset based on a table name, in the context of a quality condition</param>
/// <param name="issueRepository">The issue repository.</param>
/// <param name="exceptionObjectRepository">The exception object repository</param>
/// <param name="tileSize">Tile size for the quality verification.</param>
/// <param name="getKeyFieldName">Function for getting the key field name for an object dataset</param>
/// <param name="areaOfInterest">The test run perimeter (optional).</param>
/// <param name="trackCancel">The cancel tracker.</param>
/// <param name="errorCount">The number of (hard) errors.</param>
/// <param name="warningCount">The number of warnings.</param>
/// <param name="rowCountWithStopConditions">The number of rows for which a stop condition was violated - those rows may not be completely tested.</param>
/// <returns></returns>
public bool Verify([NotNull] QualitySpecification qualitySpecification,
[NotNull] IDatasetContext datasetContext,
[NotNull] IQualityConditionObjectDatasetResolver datasetResolver,
[CanBeNull] IIssueRepository issueRepository,
[CanBeNull] IExceptionObjectRepository exceptionObjectRepository,
double tileSize,
[CanBeNull] Func <IObjectDataset, string> getKeyFieldName,
[CanBeNull] AreaOfInterest areaOfInterest,
[CanBeNull] ITrackCancel trackCancel,
out int errorCount,
out int warningCount,
out int rowCountWithStopConditions)
{
Assert.ArgumentNotNull(qualitySpecification, nameof(qualitySpecification));
Assert.ArgumentNotNull(datasetContext, nameof(datasetContext));
Assert.ArgumentNotNull(datasetResolver, nameof(datasetResolver));
Assert.ArgumentCondition(tileSize > 0, "Invalid tile size: {0}", tileSize);
_verificationReportBuilder.BeginVerification(areaOfInterest);
IEnumerable <QualitySpecificationElement> elements =
GetOrderedElements(qualitySpecification).ToList();
IDictionary <ITest, QualitySpecificationElement> elementsByTest;
IEnumerable <ITest> tests =
CreateTests(elements, datasetContext, out elementsByTest).ToList();
var qualityConditionCount = 0;
foreach (QualitySpecificationElement element in elements)
{
qualityConditionCount++;
_verificationReportBuilder.AddVerifiedQualityCondition(element);
}
var datasetCount = 0;
foreach (Dataset dataset in
GetVerifiedDatasets(qualitySpecification, datasetContext))
{
datasetCount++;
_verificationReportBuilder.AddVerifiedDataset(dataset);
}
Stopwatch watch = _msg.DebugStartTiming();
LogTests(tests, elementsByTest);
TestContainer testContainer = CreateTestContainer(tests, tileSize);
LogBeginVerification(qualitySpecification, tileSize, areaOfInterest);
IssueProcessor issueProcessor;
ProgressProcessor progressProcessor;
using (var issueWriter = new BufferedIssueWriter(_verificationReportBuilder,
datasetContext, datasetResolver,
issueRepository,
getKeyFieldName))
{
issueProcessor = CreateIssueProcessor(testContainer, issueWriter,
areaOfInterest, exceptionObjectRepository,
elementsByTest);
progressProcessor = new ProgressProcessor(testContainer, elementsByTest,
trackCancel);
testContainer.QaError += (sender, args) => issueProcessor.Process(args);
testContainer.TestingRow +=
delegate(object o, RowEventArgs args)
{
if (issueProcessor.HasStopCondition(args.Row))
{
args.Cancel = true;
}
};
testContainer.ProgressChanged +=
(sender, args) => progressProcessor.Process(args);
// run the tests
TestExecutionUtils.Execute(testContainer, areaOfInterest);
}
_verificationReportBuilder.AddRowsWithStopConditions(
issueProcessor.GetRowsWithStopConditions());
if (exceptionObjectRepository != null)
{
_verificationReportBuilder.AddExceptionStatistics(
exceptionObjectRepository.ExceptionStatistics);
}
_verificationReportBuilder.EndVerification(progressProcessor.Cancelled);
_msg.DebugStopTiming(watch, "Verification");
errorCount = issueProcessor.ErrorCount;
warningCount = issueProcessor.WarningCount;
rowCountWithStopConditions = issueProcessor.RowsWithStopConditionsCount;
bool fulfilled = errorCount == 0 && !progressProcessor.Cancelled;
LogResults(elements, issueProcessor,
qualityConditionCount, datasetCount,
fulfilled, progressProcessor.Cancelled,
exceptionObjectRepository?.ExceptionStatistics);
return(fulfilled);
}
private static ITable GetInvolvedTableAndQueryFilter(
[NotNull] IObjectDataset involvedObjectDataset,
[NotNull] IDatasetContext datasetContext,
bool includeDateOfChangeField,
[NotNull] out IQueryFilter queryFilter,
out int dateOfChangeFieldIndex)
{
ITable result;
try
{
result = datasetContext.OpenTable(involvedObjectDataset);
Assert.NotNull(result, "Dataset not found in current context: {0}",
involvedObjectDataset.Name);
}
catch (Exception e)
{
string message;
if (involvedObjectDataset.Deleted)
{
message = string.Format(
"The dataset '{0}' referenced in allowed errors is registered as deleted, unable to open",
involvedObjectDataset.Name);
}
else
{
message = string.Format(
"Error opening dataset '{0}' referenced in allowed errors: {1}",
involvedObjectDataset.Name,
e.Message);
}
throw new InvolvedTableOpenException(message, e);
}
queryFilter = new QueryFilterClass();
var subfields = new List <string> {
result.OIDFieldName
};
if (includeDateOfChangeField)
{
ObjectAttribute dateOfChangeAttribute =
involvedObjectDataset.GetAttribute(AttributeRole.DateOfChange);
dateOfChangeFieldIndex =
dateOfChangeAttribute != null
? AttributeUtils.GetFieldIndex(result, dateOfChangeAttribute)
: -1;
if (dateOfChangeAttribute != null && dateOfChangeFieldIndex >= 0)
{
subfields.Add(dateOfChangeAttribute.Name);
}
}
else
{
dateOfChangeFieldIndex = -1;
}
GdbQueryUtils.SetSubFields(queryFilter, subfields);
return(result);
}
public SimpleDatasetOpener(IDatasetContext datasetContext)
{
_datasetContext = datasetContext;
}
