private Dictionary <string, string> GetEdtFieldValues(StandardMapping mappingUnderTest)
{
if (mappingUnderTest.IsEmailField())
{
return(GetEmailFieldValues(_idxDocumentIds, mappingUnderTest.EdtName));
}
if (mappingUnderTest.EdtType == "MultiValueList")
{
var allFieldValues = EdtDocumentRepository.GetMultiValueFieldValues(_idxDocumentIds, _idxToEdtConversionService.MappedEdtDatabaseColumn);
var combinedValues = allFieldValues.GroupBy(x => x.DocNumber)
.Select(group => new
{
DocNumber = group.Key,
Values = group.Select(x => x.FieldValue).OrderBy(x => x)
});
var dictionaryValues = combinedValues.ToDictionary(x => (string)x.DocNumber, x => string.Join(";", x.Values));
return(dictionaryValues);
}
else
{
return((_idxToEdtConversionService.MappedEdtDatabaseColumnType == ColumnType.Date) ? EdtDocumentRepository.GetDocumentDateField(_idxDocumentIds, _idxToEdtConversionService.MappedEdtDatabaseColumn)
: EdtDocumentRepository.GetDocumentField(_idxDocumentIds, _idxToEdtConversionService.MappedEdtDatabaseColumn));
}
}
public StandardMapping Map(string url)
{
var mapping = new StandardMapping(_currentMapping == null ? "" : _currentMapping.BasePath(), _routeHandler).Map(url);
AddMapping(mapping);
return(mapping);
}
public IdxToEdtConversionService(StandardMapping standardMapping)
{
_standardMapping = standardMapping;
if (!_standardMapping.IsEmailField())
{
_edtColumnDetails = GetEdtColumnDetailsFromDisplayName(standardMapping.EdtName);
}
}
public IEnumerable <StandardMapping> GetStandardMappings()
{
var fieldsToIgnore = Settings.IgnoredIdxFieldsFromComparison.ToList();
using (var sr = new StreamReader(Settings.StandardMapPath))
{
using (var swIgnored = new StreamWriter(Path.Combine(Settings.LogDirectory, "standard_mappings_ignored.csv")))
{
var headers = sr.ReadLine();
ValidateHeadersAreInExpectedPositions(headers);
var mappings = new List <StandardMapping>();
while (!sr.EndOfStream)
{
var newMappingTokens = sr.ReadLine()?.Split(_delimiter, StringSplitOptions.None);
if (newMappingTokens == null)
{
continue;
}
var mapping = new StandardMapping()
{
EdtName = newMappingTokens[EdtFieldNameColumn],
IdxName = newMappingTokens[IdxNameColumn],
EdtType = newMappingTokens[StoredEdtTypeColumn],
ImportGroup = newMappingTokens[ImporterGroupColumn]
};
if (mapping.EdtName.Equals(mapping.IdxName) || fieldsToIgnore.Contains(mapping.IdxName) || mapping.EdtName.EndsWith("_unmapped"))
{
swIgnored.WriteLine($"{mapping.EdtName},{mapping.IdxName}");
}
else
{
if (string.IsNullOrEmpty(mapping.EdtType))
{
TestContext.Out.WriteLine(
$"Warning: EDT Field {mapping.EdtName} has no type, thus will NOT be converted in processing.");
}
mappings.Add(mapping);
}
}
if (mappings.Count <= 0)
{
throw new Exception("Failed to read mappings from standard mapping doc, count is 0");
}
return(mappings);
}
}
}
public void ValidateFieldPopulation(StandardMapping mappingUnderTest)
{
long populated = 0;
long different = 0;
long documentsInIdxButNotInEdt = 0;
long documentsInEdtButNotInIdx = 0;
long idxUnfound = 0;
long unexpectedErrors = 0;
long matched = 0;
long totalsampled = 0;
//initiliase conversion service for field under test
_idxToEdtConversionService = new IdxToEdtConversionService(mappingUnderTest);
TestLogger.Debug($"Using EDT database column for comparison: {_idxToEdtConversionService.MappedEdtDatabaseColumn}");
//Get
var edtValues = GetEdtFieldValues(mappingUnderTest);
//loop thru each sample document
foreach (var idxDocument in _idxSample)
{
totalsampled++;
var idxField = GetIdxFieldValue(idxDocument, mappingUnderTest);
if (!edtValues.TryGetValue(idxDocument.DocumentId, out var edtValueForIdxRecord) && !string.IsNullOrEmpty(idxField))
{
documentsInIdxButNotInEdt++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxDocument.DocumentId, "Document not found in Edt's document table"));
}
else
{
if (string.IsNullOrEmpty(idxField))
{
if (!string.IsNullOrEmpty(edtValueForIdxRecord))
{
documentsInEdtButNotInIdx++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxDocument.DocumentId, $"Edt had value {edtValueForIdxRecord} for field {mappingUnderTest.EdtName} when Idx had no value."));
}
else
{
idxUnfound++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxDocument.DocumentId, $"Field { mappingUnderTest.IdxName } not found in Idx for document"));
}
}
else
{
try
{
if (!string.IsNullOrEmpty(edtValueForIdxRecord))
{
populated++;
}
var expectedEdtValue = _idxToEdtConversionService.ConvertValueToEdtForm(idxField);
if (!edtValueForIdxRecord.Equals(expectedEdtValue, StringComparison.InvariantCultureIgnoreCase))
{
different++;
ComparisonResults.Add(new Framework.Models.Reporting.ComparisonResult(idxDocument.DocumentId, edtValueForIdxRecord, expectedEdtValue, idxField));
}
else
{
matched++;
}
}
catch (Exception ex)
{
unexpectedErrors++;
var error = $"{ex.Message}<br></br>{ex.StackTrace}";
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxDocument.DocumentId, error));
}
}
}
}
PrintStats(different, matched, documentsInIdxButNotInEdt, documentsInEdtButNotInIdx, idxUnfound, unexpectedErrors, populated, totalsampled);
if (ComparisonErrors.Count() > 0 || ComparisonResults.Count() > 0)
{
var diffFile = PrintComparisonTables(mappingUnderTest.EdtName);
TestLogger.Info($"Difference and error details written to: <a href=\"{diffFile}\">{diffFile}</a>");
}
Assert.Zero(different, $"Differences were seen between expected value and actual value for this Edt field {mappingUnderTest.EdtName}");
Assert.Zero(unexpectedErrors, $"Unexpected errors experienced during processing {mappingUnderTest.EdtName}");
Assert.Zero(documentsInIdxButNotInEdt, $"Idx documents were not found in EDT.");
Assert.Zero(documentsInEdtButNotInIdx, "Edt was found to have field populated for instances where Idx was null");
if (idxUnfound > 0)
{
TestLogger.Info($"The Idx was found to not have a value for field {mappingUnderTest.IdxName} in {idxUnfound} documents/instances.");
}
if (populated == 0)
{
TestLogger.Info($"No sampled documents had the Edt field {mappingUnderTest.EdtName} populated.");
}
}
private string GetIdxFieldValue(Framework.Models.IdxLoadFile.Document idxDocument, StandardMapping mappingUnderTest)
{
if (mappingUnderTest.EdtType != "MultiValueList" && !mappingUnderTest.IsEmailField())
{
return(idxDocument.AllFields.FirstOrDefault(x => x.Key.Equals(mappingUnderTest.IdxName))?.Value);
}
var allFieldValues = idxDocument.AllFields.Where(x => x.Key.Equals(mappingUnderTest.IdxName))
.Select(x => x.Value)
.OrderBy(x => x);
return(string.Join(";", allFieldValues));
}
