private void PrintOutIdDiffs()
{
var EdtIds = EdtDocumentRepository.GetDocumentNumbers();
var IdxIds = new IdxDocumentsRepository().GetDocumentIds();
var edtExceptIdx = EdtIds.Except(IdxIds);
var IdxExceptEdt = IdxIds.Except(EdtIds);
using (var sw = new StreamWriter($"{Settings.ReportingDirectory}\\DocumentCountDifferences_IdLists.csv"))
{
sw.WriteLine("In Edt but not in Idx:");
foreach (var id in edtExceptIdx)
{
sw.WriteLine(id);
}
sw.WriteLine("In Idx but not Edt");
foreach (var id in IdxExceptEdt)
{
sw.WriteLine(id);
}
}
TestLogger.Info(
$"Document Id differences output to <a href=\"DocumentCountDifferences_IdLists.csv\"> DocumentCountDifferences_IdLists.csv</a>");
}
private Dictionary <string, string> GetEdtFieldValues(StandardMapping mappingUnderTest)
{
if (mappingUnderTest.IsEmailField())
{
return(GetEmailFieldValues(_idxDocumentIds, mappingUnderTest.EdtName));
}
if (mappingUnderTest.EdtType == "MultiValueList")
{
var allFieldValues = EdtDocumentRepository.GetMultiValueFieldValues(_idxDocumentIds, _idxToEdtConversionService.MappedEdtDatabaseColumn);
var combinedValues = allFieldValues.GroupBy(x => x.DocNumber)
.Select(group => new
{
DocNumber = group.Key,
Values = group.Select(x => x.FieldValue).OrderBy(x => x)
});
var dictionaryValues = combinedValues.ToDictionary(x => (string)x.DocNumber, x => string.Join(";", x.Values));
return(dictionaryValues);
}
else
{
return((_idxToEdtConversionService.MappedEdtDatabaseColumnType == ColumnType.Date) ? EdtDocumentRepository.GetDocumentDateField(_idxDocumentIds, _idxToEdtConversionService.MappedEdtDatabaseColumn)
: EdtDocumentRepository.GetDocumentField(_idxDocumentIds, _idxToEdtConversionService.MappedEdtDatabaseColumn));
}
}
private Dictionary <string, string> GetEmailFieldValues(List <string> documentIds, string fieldType)
{
var type = fieldType.Replace(Settings.EmailFieldIdentifyingPrefix, string.Empty);
var allFields = EdtDocumentRepository.GetDocumentCorrespondances(documentIds)
.Where(x => x.CorrespondanceType.Equals(type, StringComparison.InvariantCultureIgnoreCase));
var desiredParties = from field in allFields
group field.PartyName by field.DocumentNumber into correspondants
select new { DocumentId = correspondants.Key, Value = string.Join(";", correspondants.ToList().OrderBy(x => x)) };
return(desiredParties.ToDictionary(x => (string)x.DocumentId, x => x.Value));
}
public static long GetDocumentCountForBatch()
{
var allDocumentIds = EdtDocumentRepository.GetAllDocumentIds();
var allNativeFilesInEdt = Directory
.GetFiles(Path.Combine(Settings.EdtCfsDirectory, $"Site01_Case{Settings.EdtCaseId.ToString().PadLeft(4, '0')}"), "*.*", SearchOption.AllDirectories)
.Select(x => new FileInfo(x).Name)
.Select(x => x.Substring(0, x.IndexOf('_')))
.Distinct();
var presentDocuments = allDocumentIds.Intersect(allNativeFilesInEdt);
return(presentDocuments.LongCount());
}
public void CaseLevelCounts()
{
var EdtDocumentCounts = EdtDocumentRepository.GetDocumentCountPerBatch();
var tableData = new List <string[]>()
{
new string[] { "Batch", "Count of Documents" }
};
tableData.AddRange(EdtDocumentCounts.Select(x => new string[] { x.BatchName, x.DocumentCount.ToString() }));
tableData.Add(new string[] { "Total", EdtDocumentCounts.Sum(x => x.DocumentCount).ToString() });
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(tableData.ToArray()));
}
public void NativeCountsAreEqualBetweenIdxAndQuarantineFolder()
{
int lppDocCount = EdtDocumentRepository.GetDocumentQuarantineDocumentCount();
int idxLppDocCount = new IdxDocumentsRepository().GetNumberOfLppDocs();
string[][] data =
{
new[] { "Item Evaluated", "Count of LPP Documents" },
new[] { "Idx file", idxLppDocCount.ToString() },
new[] { "Quarantine Folder", lppDocCount.ToString() }
};
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(data));
Assert.AreEqual(lppDocCount, idxLppDocCount,
"File counts should be equal between IDX and EDT Quarantine folder");
}
public void TextCountsAreEqualBetweenIdxAndEdtFileStore()
{
//For each Document in Batch, Count where Body is not null
var edtDocsWithBody = EdtDocumentRepository.GetDocuentNumbersWithABody();
//compare against Text count in microfocus dir
var edtIds = EdtDocumentRepository.GetDocumentNumbers();
var textFileDocsIds = Directory
.GetFiles(Settings.MicroFocusStagingDirectoryTextPath, "*.txt", SearchOption.AllDirectories)
.Select(x => GetDocumentIdFromFilePath(x)).Where(x => edtIds.Contains(x));
var mircoFocusDocCount = textFileDocsIds.Count();
//output counts
string[][] data =
{
new[] { "Item Evaluated", "Count of Documents" },
new[] { "MicroFocus Export text(s)", mircoFocusDocCount.ToString() },
new[] { "Edt Document.Body", edtDocsWithBody.Count().ToString() }
};
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(data));
if (mircoFocusDocCount != edtDocsWithBody.Count())
{
//output diff list
var outputFile = Path.Combine(Settings.ReportingDirectory, "TextContentMissing.csv");
using (var sw = new StreamWriter(outputFile))
{
var missingBodies = textFileDocsIds.Where(x => !edtDocsWithBody.Contains(x));
foreach (var missing in missingBodies)
{
sw.WriteLine(missing);
}
TestLogger.Info($"List of Ids without body output to: {new FileInfo(outputFile).FullName}");
}
}
Assert.AreEqual(mircoFocusDocCount, edtDocsWithBody.Count(),
"File counts should be equal for Microfocus load and EDT");
}
public void DatabaseDocumentsCountsAreEqualBetweenIdxAndEdtDatabase()
{
var EdtDocumentCount = EdtDocumentRepository.GetDocumentCount();
string[][] data = new string[][]
{
new string[] { "Item Evaluated", "Count of Documents" },
new string[] { "Idx file", _idxDocumentCount.ToString() },
new string[] { "Edt Database", EdtDocumentCount.ToString() }
};
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(data));
if (_idxDocumentCount != EdtDocumentCount)
{
PrintOutIdDiffs();
}
Assert.AreEqual(_idxDocumentCount, EdtDocumentCount, "File counts should be equal for Idx and Load file");
}
private ColumnDetails GetEdtColumnDetailsFromDisplayName(string displayName)
{
var lowerDisplayName = displayName.ToLower();
var edtColumnDetails = EdtDocumentRepository.GetColumnDetails().ToList();
var matchedDbName = edtColumnDetails.FirstOrDefault(x => x.DisplayName.ToLower().Equals(lowerDisplayName));
if (matchedDbName != null)
{
return(matchedDbName);
}
Regex rgx = new Regex("[^a-zA-Z0-9]");
lowerDisplayName = rgx.Replace(lowerDisplayName, "");
matchedDbName = edtColumnDetails.Find(x => x.GetAlphaNumbericOnlyDisplayName().ToLower()
.Replace(" ", string.Empty).Equals(lowerDisplayName));
return(matchedDbName ?? throw new Exception($"Unable to determine Edt Db column name from mapped display name {displayName}"));
}
public void RedactedCountsAreEqualBetweenRedactionLoadFile()
{
int edtRedactionCount = 0;
int redactionDocCount = 0;
if (!string.IsNullOrWhiteSpace(Settings.RedactionsFilePath))
{
edtRedactionCount = EdtDocumentRepository.GetDocumentRedactedDocumentCount();
redactionDocCount = new RedactionLoadFileReader(Settings.RedactionsFilePath).GetRecordCount();
}
string[][] data =
{
new[] { "Item Evaluated", "Count of LPP Documents" },
new[] { "Redaction Load file", redactionDocCount.ToString() },
new[] { "Document Table", edtRedactionCount.ToString() }
};
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(data));
Assert.AreEqual(redactionDocCount, edtRedactionCount,
"File counts should be equal between Redaction Load files and EDT");
}
public void Tags()
{
long idxUnpopulated = 0;
long edtUnexpectedlyPopulated = 0;
long errors = 0;
long matched = 0;
long different = 0;
var workbookRecords = AunWorkbookReader.Read();
var allEdtTags = EdtDocumentRepository.GetDocumentTags(_idxDocumentIds);
foreach (var idxRecord in _idxSample)
{
var aunWorkbookIds = idxRecord.AllFields.Where(x => x.Key.Equals("AUN_WORKBOOK_NUMERIC"));
var foundEdtValue = allEdtTags.TryGetValue(idxRecord.DocumentId, out var relatedEdTags);
if (aunWorkbookIds.Count() == 0)
{
idxUnpopulated++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxRecord.DocumentId, "AUN_WORKBOOK_NUMERIC field was not present for idx record"));
if (relatedEdTags != null && relatedEdTags.Count() > 0)
{
edtUnexpectedlyPopulated++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxRecord.DocumentId, $"EDT has value(s) {string.Join(",", relatedEdTags)} when Idx record had no value"));
}
continue;
}
foreach (var aunWorkbookId in aunWorkbookIds)
{
var foundWorkbookName = workbookRecords.TryGetValue(aunWorkbookId.Value, out var aunWorkbookName);
if (foundWorkbookName)
{
if (!foundEdtValue || (relatedEdTags != null && !relatedEdTags.Any(x => x != null && x.Equals(aunWorkbookName, System.StringComparison.InvariantCultureIgnoreCase))))
{
different++;
var edtLogValue = relatedEdTags != null?string.Join(";", relatedEdTags) : "none found";
ComparisonResults.Add(new Framework.Models.Reporting.ComparisonResult(idxRecord.DocumentId, edtLogValue, aunWorkbookName, aunWorkbookId.Value.ToString()));
}
else
{
matched++;
}
}
else
{
errors++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxRecord.DocumentId, $"Couldnt convert aun workbook id {aunWorkbookId} to name"));
}
}
}
var diffFile = PrintComparisonTables("Tags");
TestLogger.Info($"Difference and error details written to: <a href=\"{diffFile}\">{diffFile}</a>");
//print table of stats
string[][] data = new string[][] {
new string[] { "<b>Comparison Statistics:</b>" },
new string[] { "Statistic", "Count" },
new string[] { "Differences", different.ToString() },
new string[] { "Matched", matched.ToString() },
new string[] { "Unexpected Errors", errors.ToString() },
new string[] { "Edt document(s) incorrectly have a value when Idx is null", edtUnexpectedlyPopulated.ToString() },
new string[] { "Idx document(s) not populated for field under test (and EDt is also null)", idxUnpopulated.ToString() }
};
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(data));
Assert.Zero(different, $"Differences were seen between expected value and actual value");
Assert.Zero(edtUnexpectedlyPopulated, "Edt was found to have field populated for instances where Idx was null");
Assert.Zero(errors, "Expected errors encountered during processing");
if (idxUnpopulated > 0)
{
TestLogger.Info($"The Idx was found to not have a value for field AUN_WORKBOOK_NUMERIC in {idxUnpopulated} documents/instances.");
}
}
public void Locations()
{
long idxUnpopulated = 0;
long edtUnexpectedlyPopulated = 0;
long errors = 0;
long matched = 0;
long different = 0;
var allEdtLocations = EdtDocumentRepository.GetDocumentLocations(_idxDocumentIds);
foreach (var idxDocument in _idxSample)
{
allEdtLocations.TryGetValue(idxDocument.DocumentId, out var edtLocation);
if (string.IsNullOrWhiteSpace(edtLocation))
{
different++;
ComparisonErrors.Add(new Framework.Models.Reporting.ComparisonError(idxDocument.DocumentId, "Location not present in EDT"));
}
string group = idxDocument.AllFields.SingleOrDefault(x => x.Key.Equals(Settings.LocationIdxFields[0])).Value;
string custodian = idxDocument.AllFields.SingleOrDefault(x => x.Key.Equals(Settings.LocationIdxFields[1])).Value;
string source = idxDocument.AllFields.SingleOrDefault(x => x.Key.Equals(Settings.LocationIdxFields[2])).Value;
string location = $@"{group}\{custodian}\{source}";
idxDocument.AllFields.Where(c => c.Key.StartsWith(Settings.LocationIdxFields[3])).OrderBy(c => c.Key).ToList().ForEach(
c =>
{
if (!string.IsNullOrWhiteSpace(c.Value) && !c.Value.Contains(".msg:"))
{
location += @"\" + c.Value.Replace(":", "-");
}
});
if (!location.Equals(edtLocation, StringComparison.InvariantCultureIgnoreCase))
{
different++;
ComparisonResults.Add(new Framework.Models.Reporting.ComparisonResult(idxDocument.DocumentId, edtLocation, location, location));
}
else
{
matched++;
}
}
var diffFile = PrintComparisonTables("Locations");
TestLogger.Info($"Difference and error details written to: <a href=\"{diffFile}\">Report\\{diffFile}</a>");
//print table of stats
string[][] data = new string[][] {
new string[] { "<b>Comparison Statistics:</b>" },
new string[] { "Statistic", "Count" },
new string[] { "Differences", different.ToString() },
new string[] { "Matched", matched.ToString() },
new string[] { "Unexpected Errors", errors.ToString() },
new string[] { "Edt document(s) incorrectly have a value when Idx is null", edtUnexpectedlyPopulated.ToString() },
new string[] { "Idx document(s) not populated for field under test (and EDt is also null)", idxUnpopulated.ToString() }
};
TestLogger.Log(AventStack.ExtentReports.Status.Info, MarkupHelper.CreateTable(data));
Assert.Zero(different, $"Differences were seen between expected value and actual value");
Assert.Zero(edtUnexpectedlyPopulated, "Edt was found to have field populated for instances where Idx was null");
Assert.Zero(errors, "Expected errors encountered during processing");
}
