private async Task <RecordCatalog> Load(LogDataSource <Dictionary <String, Object> > src)
{
return(await Task <RecordCatalog> .Run(() =>
{
var retVal = new RecordCatalog();
foreach (var record in src)
{
if (!CanFormKey(UniqueIdentifierKeys, record))
{
OutputWriter.WriteLine($"Could not form unique key with elements {Stringify(UniqueIdentifierKeys, ",")} at {src.Location}");
continue;
}
String uniqueId = BuildKey(UniqueIdentifierKeys, record);
retVal.AddRecord(uniqueId, src.Location, record);
}
return retVal;
}));
}
private void Reconcile(RecordCatalog oldRecords, RecordCatalog newRecords)
{
OutputWriter.WriteLine($"START TEST: {TestName}");
OutputWriter.WriteLine($"Unique key format: {Stringify(UniqueIdentifierKeys, "::")}");
if (FilteredKeys.Length > 0)
{
OutputWriter.WriteLine($"Elements not used in record equality evaluation: {Stringify(FilteredKeys, ",")}");
}
OutputWriter.WriteLine("*----+ Checking if there are duplicate records in the old data set +----*");
ReportDuplicateRecords(oldRecords.Duplicates);
OutputWriter.WriteLine("*----+ Checking if there are duplicate records in the new data set +----*");
ReportDuplicateRecords(newRecords.Duplicates);
OutputWriter.WriteLine("*----+ Checking for any record element differences +----*");
foreach (var elem in oldRecords.ValueKeys.Except(newRecords.ValueKeys))
{
OutputWriter.WriteLine($"{elem} appears in the old data set, but is not found in the new data set");
}
var newFieldFilter = newRecords.ValueKeys.Except(oldRecords.ValueKeys);
foreach (var elem in newFieldFilter)
{
OutputWriter.WriteLine($"{elem} appears to be a new data element");
}
OutputWriter.WriteLine("*----+ Checking that the number of records in old and new data sets are equal +----*");
OutputWriter.WriteLine($"Old records {oldRecords.RecordCount}, New records {newRecords.RecordCount}");
if (oldRecords.RecordCount != newRecords.RecordCount)
{
var diff = new List <String>(oldRecords.Records.Keys.Except(newRecords.Records.Keys));
if (diff.Count > 0)
{
OutputWriter.WriteLine($"----+ {diff.Count} records missing from the new record +----*");
foreach (var key in diff)
{
OutputWriter.WriteLine($"{oldRecords.Records[key].UniqueId} : {oldRecords.Records[key].Location}");
}
}
diff = new List <String>(newRecords.Records.Keys.Except(oldRecords.Records.Keys));
if (diff.Count > 0)
{
OutputWriter.WriteLine($"----+ {diff.Count} records missing from the old record +----*");
foreach (var key in diff)
{
OutputWriter.WriteLine($"{newRecords.Records[key].UniqueId} : {newRecords.Records[key].Location}");
}
}
}
OutputWriter.WriteLine("*----+ Verifying records match in data sets +----*");
foreach (var recordId in oldRecords.Records.Keys)
{
if (!newRecords.Records.ContainsKey(recordId))
{
OutputWriter.WriteLine($"Record {recordId} not found in the new data set");
continue;
}
var comparisonElements = oldRecords.Records[recordId].Record.Keys.Except(FilteredKeys);
var oldHash = oldRecords.Records[recordId].HashByKeys(comparisonElements);
var newHash = newRecords.Records[recordId].HashByKeys(comparisonElements);
if (oldHash.CompareTo(newHash) != 0)
{
OutputWriter.WriteLine($"Unique Key {recordId} did not match");
}
}
OutputWriter.WriteLine($"END TEST: {TestName}");
}