public void TestReconstructionFromCsv()
{
var opts = new IsIdentifiableRelationalDatabaseOptions();
var dir = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory);
foreach (var f in dir.GetFiles("*HappyOzz*.csv"))
{
f.Delete();
}
opts.DestinationCsvFolder = dir.FullName;
opts.TableName = "HappyOzz";
opts.StoreReport = true;
FailureStoreReport report = new FailureStoreReport("HappyOzz", 1000);
report.AddDestinations(opts);
var failure = new Failure(
new FailurePart[]
{
new FailurePart("Kansas", FailureClassification.Location, 12),
new FailurePart("Toto", FailureClassification.Location, 28)
})
{
ProblemValue = "We aren't in Kansas anymore Toto",
ProblemField = "Narrative",
ResourcePrimaryKey = "1.2.3",
Resource = "FunBooks.HappyOzz"
};
report.Add(failure);
report.CloseReport();
var created = dir.GetFiles("*HappyOzz*.csv").SingleOrDefault();
Assert.IsNotNull(created);
var report2 = new FailureStoreReport("", 0);
var failures2 = report2.Deserialize(created).ToArray();
//read failure ok
Assert.AreEqual(1, failures2.Length);
Assert.AreEqual(failure.ProblemValue, failures2[0].ProblemValue);
Assert.AreEqual(failure.ProblemField, failures2[0].ProblemField);
Assert.AreEqual(failure.ResourcePrimaryKey, failures2[0].ResourcePrimaryKey);
Assert.AreEqual(failure.Resource, failures2[0].Resource);
//read parts ok
Assert.AreEqual(2, failures2[0].Parts.Count);
Assert.AreEqual(failure.Parts[0].Classification, failures2[0].Parts[0].Classification);
Assert.AreEqual(failure.Parts[0].Offset, failures2[0].Parts[0].Offset);
Assert.AreEqual(failure.Parts[0].Word, failures2[0].Parts[0].Word);
Assert.AreEqual(failure.Parts[1].Classification, failures2[0].Parts[1].Classification);
Assert.AreEqual(failure.Parts[1].Offset, failures2[0].Parts[1].Offset);
Assert.AreEqual(failure.Parts[1].Word, failures2[0].Parts[1].Word);
}
/// <summary>
/// Connects to the database and runs all failures through the rules base performing redactions as required
/// </summary>
/// <returns></returns>
public int Run()
{
//In RulesOnly mode this will be null
var server = _target?.Discover();
List <Exception> errors = new List <Exception>();
var storeReport = new FailureStoreReport(_outputFile.Name, 100);
Stopwatch sw = new Stopwatch();
sw.Start();
using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile))
{
IsIdentifiableRule updateRule;
storeReport.AddDestination(storeReportDestination);
while (_reportReader.Next())
{
bool noUpdate;
try
{
noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule);
}
catch (Exception e)
{
errors.Add(e);
continue;
}
//is it novel for updater
if (noUpdate)
{
//is it novel for ignorer
if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule))
{
//we can't process it unattended
storeReport.Add(_reportReader.Current);
Unresolved++;
}
else
{
if (!_ignoreRulesUsed.ContainsKey(ignoreRule))
{
_ignoreRulesUsed.Add(ignoreRule, 1);
}
else
{
_ignoreRulesUsed[ignoreRule]++;
}
Ignores++;
}
}
else
{
if (!_updateRulesUsed.ContainsKey(updateRule))
{
_updateRulesUsed.Add(updateRule, 1);
}
else
{
_updateRulesUsed[updateRule]++;
}
Updates++;
}
Total++;
if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000)
{
Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true);
sw.Restart();
}
}
storeReport.CloseReport();
}
Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
_ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);
Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
_updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);
Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false);
Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true);
return(0);
}