public void Test_RegexUpdateStrategy(DatabaseType dbType, bool provideCaptureGroup)
{
var db = GetCleanedServer(dbType);
var dbname = db.GetRuntimeName();
//the Failure was about Kansas and Toto
var failure = new Failure(
new FailurePart[]
{
new FailurePart("Kansas", FailureClassification.Location, 13),
new FailurePart("Toto", FailureClassification.Location, 28)
})
{
ProblemValue = "We aren't in Kansas anymore Toto",
ProblemField = "Narrative",
ResourcePrimaryKey = "1.2.3.4",
Resource = dbname + ".HappyOzz"
};
DataTable dt = new DataTable();
dt.Columns.Add("MyPk");
dt.Columns.Add("Narrative");
dt.PrimaryKey = new[] { dt.Columns["MyPk"] };
dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto");
var tbl = db.CreateTable("HappyOzz", dt);
//redacted string will be longer!
var col = tbl.DiscoverColumn("Narrative");
col.DataType.Resize(1000);
var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml"));
//make sure repeat test runs work properly
if (File.Exists(newRules.FullName))
{
File.Delete(newRules.FullName);
}
RowUpdater updater = new RowUpdater(newRules);
//But the user told us that only the Toto bit is a problem
string rule = provideCaptureGroup ? "(Toto)$" : "Toto$";
updater.RulesFactory = Mock.Of <IRulePatternFactory>(m => m.GetPattern(It.IsAny <object>(), It.IsAny <Failure>()) == rule);
//this is the thing we are actually testing, where we update based on the usersRule not the failing parts
updater.UpdateStrategy = new RegexUpdateStrategy();
//it should be novel i.e. require user decision
Assert.IsTrue(updater.OnLoad(db.Server, failure, out _));
updater.Update(db.Server, failure, null);//<- null here will trigger the rule pattern factory to prompt 'user' for pattern which is "(Toto)$"
var result = tbl.GetDataTable();
if (provideCaptureGroup)
{
Assert.AreEqual("We aren't in Kansas anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Expected update to only affect the capture group ToTo");
}
else
{
Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Because regex had no capture group we expected the update strategy to fallback on Failure Part matching");
}
//it should be updated automatically and not require user decision
Assert.IsFalse(updater.OnLoad(db.Server, failure, out _));
}
/// <summary>
/// Connects to the database and runs all failures through the rules base performing redactions as required
/// </summary>
/// <returns></returns>
public int Run()
{
//In RulesOnly mode this will be null
var server = _target?.Discover();
List <Exception> errors = new List <Exception>();
var storeReport = new FailureStoreReport(_outputFile.Name, 100);
Stopwatch sw = new Stopwatch();
sw.Start();
using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile))
{
IsIdentifiableRule updateRule;
storeReport.AddDestination(storeReportDestination);
while (_reportReader.Next())
{
bool noUpdate;
try
{
noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule);
}
catch (Exception e)
{
errors.Add(e);
continue;
}
//is it novel for updater
if (noUpdate)
{
//is it novel for ignorer
if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule))
{
//we can't process it unattended
storeReport.Add(_reportReader.Current);
Unresolved++;
}
else
{
if (!_ignoreRulesUsed.ContainsKey(ignoreRule))
{
_ignoreRulesUsed.Add(ignoreRule, 1);
}
else
{
_ignoreRulesUsed[ignoreRule]++;
}
Ignores++;
}
}
else
{
if (!_updateRulesUsed.ContainsKey(updateRule))
{
_updateRulesUsed.Add(updateRule, 1);
}
else
{
_updateRulesUsed[updateRule]++;
}
Updates++;
}
Total++;
if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000)
{
Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true);
sw.Restart();
}
}
storeReport.CloseReport();
}
Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
_ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);
Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
_updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);
Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false);
Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true);
return(0);
}
public void Test(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var dbname = db.GetRuntimeName();
var failure = new Failure(
new FailurePart[]
{
new FailurePart("Kansas", FailureClassification.Location, 13),
new FailurePart("Toto", FailureClassification.Location, 28)
})
{
ProblemValue = "We aren't in Kansas anymore Toto",
ProblemField = "Narrative",
ResourcePrimaryKey = "1.2.3.4",
Resource = dbname + ".HappyOzz"
};
DataTable dt = new DataTable();
dt.Columns.Add("MyPk");
dt.Columns.Add("Narrative");
dt.PrimaryKey = new[] { dt.Columns["MyPk"] };
dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto");
var tbl = db.CreateTable("HappyOzz", dt);
//redacted string will be longer!
var col = tbl.DiscoverColumn("Narrative");
col.DataType.Resize(1000);
var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml"));
//make sure repeat test runs work properly
if (File.Exists(newRules.FullName))
{
File.Delete(newRules.FullName);
}
RowUpdater updater = new RowUpdater(newRules);
updater.UpdateStrategy = new ProblemValuesUpdateStrategy();
//it should be novel i.e. require user decision
Assert.IsTrue(updater.OnLoad(db.Server, failure, out _));
updater.Update(db.Server, failure, null);
var result = tbl.GetDataTable();
Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"]);
TestHelpers.Contains(
@"- Action: Report
IfColumn: Narrative
As: Location
IfPattern: ^We\ aren't\ in\ Kansas\ anymore\ Toto$
", File.ReadAllText(newRules.FullName));//btw slash space is a 'literal space' so legit
//it should be updated automatically and not require user decision
Assert.IsFalse(updater.OnLoad(db.Server, failure, out _));
}