private void Test(bool[] rowsToKeep)
{
int count = rowsToKeep.Length;
// Build an identity column
NumberColumn <int> column = new NumberColumn <int>(-1);
for (int i = 0; i < rowsToKeep.Length; ++i)
{
column[i] = i;
}
// Build a RowUpdater and fake mapping to temp to check those results
TableStub table = new TableStub();
RowUpdater updater = new RowUpdater(table, table);
int[] rowsToTemp = Enumerable.Range(10, count).ToArray();
// Request Garbage Collection
GarbageCollector.Collect <int>(column, null, rowsToKeep, updater, rowsToTemp);
// Verify correct values were kept
StringBuilder expected = new StringBuilder();
int expectedCount = 0;
for (int i = 0; i < count; ++i)
{
if (rowsToKeep[i])
{
expectedCount++;
if (expected.Length > 0)
{
expected.Append(", ");
}
expected.Append(i);
}
}
Assert.Equal(expectedCount, column.Count);
Assert.Equal(expected.ToString(), String.Join(", ", column.OrderBy((i) => i)));
// Verify rows removed are reported in the correct indices in temp or swapped in original column
RowStub stub = new RowStub(table, 0);
for (int i = 0; i < count; ++i)
{
stub.Index = i;
updater.Update(stub);
if (!rowsToKeep[i])
{
Assert.Equal(10 + i, stub.Index);
}
else
{
Assert.Equal(i, column[stub.Index]);
}
}
}
/// <summary>
/// Creates a new instance that will connect to the database server (<paramref name="target"/>) and perform redactions using the <paramref name="updater"/>
/// </summary>
/// <param name="opts">CLI options for the process</param>
/// <param name="target">DBMS to connect to for redacting</param>
/// <param name="ignorer">Rules base for detecting false positives</param>
/// <param name="updater">Rules base for redacting true positives</param>
public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, IgnoreRuleGenerator ignorer, RowUpdater updater)
{
_log = LogManager.GetCurrentClassLogger();
if (string.IsNullOrWhiteSpace(opts.FailuresCsv))
{
throw new Exception("Unattended requires a file of errors to process");
}
var fi = new FileInfo(opts.FailuresCsv);
if (!fi.Exists)
{
throw new FileNotFoundException($"Could not find Failures file '{fi.FullName}'");
}
if (!opts.OnlyRules)
{
_target = target ?? throw new Exception("A single Target must be supplied for database updates");
}
_reportReader = new ReportReader(fi);
if (string.IsNullOrWhiteSpace(opts.UnattendedOutputPath))
{
throw new Exception("An output path must be specified for Failures that could not be resolved");
}
_outputFile = new FileInfo(opts.UnattendedOutputPath);
_ignorer = ignorer;
_updater = updater;
}
public RulesView(ReportReader currentReport, IgnoreRuleGenerator ignorer, RowUpdater updater)
{
CurrentReport = currentReport;
Ignorer = ignorer;
Updater = updater;
Modal = true;
var lblInitialSummary = new Label($"There are {ignorer.Rules.Count} ignore rules and {updater.Rules.Count} update rules. Current report contains {CurrentReport.Failures.Length:N0} Failures");
Add(lblInitialSummary);
var lblEvaluate = new Label($"Evaluate:")
{
Y = Pos.Bottom(lblInitialSummary) + 1
};
Add(lblEvaluate);
var ruleCollisions = new Button("Rule Coverage")
{
Y = Pos.Bottom(lblEvaluate)
};
ruleCollisions.Clicked += () => EvaluateRuleCoverage();
Add(ruleCollisions);
_treeView = new TreeView()
{
Y = Pos.Bottom(ruleCollisions) + 1,
Width = Dim.Fill(),
Height = Dim.Fill(1)
};
_treeView.KeyPress += _treeView_KeyPress;
Add(_treeView);
var close = new Button("Close", true)
{
Y = Pos.Bottom(_treeView)
};
close.Clicked += () => Quit();
Add(close);
}
private static int OnParse(GlobalOptions globals, IsIdentifiableReviewerOptions opts)
{
FansiImplementations.Load();
Deserializer d = new Deserializer();
List <Target> targets;
try
{
var file = new FileInfo(opts.TargetsFile);
if (!file.Exists)
{
Console.Write($"Could not find '{file.FullName}'");
return(1);
}
var contents = File.ReadAllText(file.FullName);
if (string.IsNullOrWhiteSpace(contents))
{
Console.Write($"Targets file is empty '{file.FullName}'");
return(2);
}
targets = d.Deserialize <List <Target> >(contents);
if (!targets.Any())
{
Console.Write($"Targets file did not contain any valid targets '{file.FullName}'");
return(3);
}
}
catch (Exception e)
{
Console.WriteLine($"Error deserializing '{opts.TargetsFile}'");
Console.WriteLine(e.Message);
return(4);
}
if (opts.OnlyRules)
{
Console.WriteLine("Skipping Connection Tests");
}
else
{
Console.WriteLine("Running Connection Tests");
try
{
foreach (Target t in targets)
{
Console.WriteLine(t.Discover().Exists()
? $"Successfully connected to {t.Name}"
: $"Failed to connect to {t.Name}");
}
}
catch (Exception e)
{
Console.WriteLine("Error Validating Targets");
Console.WriteLine(e.ToString());
return(10);
}
}
//for updater try to match the ProblemValue words
var updater = new RowUpdater(new FileInfo(opts.RedList))
{
RulesOnly = opts.OnlyRules,
RulesFactory = new MatchProblemValuesPatternFactory()
};
//for Ignorer match the whole string
var ignorer = new IgnoreRuleGenerator(new FileInfo(opts.IgnoreList));
try
{
if (!string.IsNullOrWhiteSpace(opts.UnattendedOutputPath))
{
//run unattended
if (targets.Count != 1)
{
throw new Exception("Unattended requires a single entry in Targets");
}
var unattended = new UnattendedReviewer(opts, targets.Single(), ignorer, updater);
return(unattended.Run());
}
else
{
Console.WriteLine("Press any key to launch GUI");
Console.ReadKey();
//run interactive
Application.Init();
var mainWindow = new MainWindow(opts, ignorer, updater);
Application.Top.Add(mainWindow);
Application.Run();
return(0);
}
}
catch (Exception e)
{
Console.Write(e.Message);
int tries = 5;
while (Application.Top != null && tries-- > 0)
{
try
{
Application.RequestStop();
}
catch (Exception)
{
Console.WriteLine("Failed to terminate GUI on crash");
}
}
return(99);
}
}
public MainWindow(IsIdentifiableReviewerOptions opts, IgnoreRuleGenerator ignorer, RowUpdater updater)
{
Ignorer = ignorer;
Updater = updater;
_origUpdaterRulesFactory = updater.RulesFactory;
_origIgnorerRulesFactory = ignorer.RulesFactory;
X = 0;
Y = 1;
Width = Dim.Fill();
Height = Dim.Fill();
var top = Application.Top;
var menu = new MenuBar(new MenuBarItem [] {
new MenuBarItem("_File (F9)", new MenuItem [] {
new MenuItem("_Open Report", null, OpenReport),
new MenuItem("_Quit", null, () => { top.Running = false; })
}),
new MenuBarItem("_Options", new MenuItem [] {
miCustomPatterns = new MenuItem("_Custom Patterns", null, ToggleCustomPatterns)
{
CheckType = MenuItemCheckStyle.Checked, Checked = false
}
}),
new MenuBarItem("_View", new MenuItem [] {
new MenuItem("_Rules", null, ViewRules),
})
});
_info = new Label("Info")
{
X = 0,
Y = 0,
Width = Dim.Fill(),
Height = 1
};
_info.ColorScheme = _greyOnBlack;
_valuePane = new FailureView()
{
X = 0,
Y = 1,
Width = Dim.Fill(),
Height = 10,
};
var frame = new FrameView("Options")
{
X = 0,
Y = 11,
Width = Dim.Fill(),
Height = Dim.Fill()
};
var ignoreButton = new Button("Ignore")
{
X = 0
};
ignoreButton.Clicked += Ignore;
frame.Add(ignoreButton);
var updateButton = new Button("Update")
{
X = 11
};
updateButton.Clicked += Update;
frame.Add(updateButton);
_gotoTextField = new TextField("1")
{
X = 28,
Width = 5
};
_gotoTextField.TextChanged += (s) => GoTo();
frame.Add(_gotoTextField);
frame.Add(new Label(23, 0, "GoTo:"));
var prevButton = new Button("Prev")
{
X = 0,
Y = 1
};
prevButton.Clicked += () => GoToRelative(-1);
frame.Add(prevButton);
var nextButton = new Button("Next")
{
X = 11,
Y = 1
};
nextButton.Clicked += () => GoToRelative(1);
frame.Add(nextButton);
var undoButton = new Button("unDo")
{
X = 11,
Y = 2
};
undoButton.Clicked += () => Undo();
frame.Add(undoButton);
frame.Add(new Label(0, 4, "Default Patterns"));
_ignoreRuleLabel = new Label(0, 5, "Ignore:");
_updateRuleLabel = new Label(0, 6, "Update:");;
frame.Add(_ignoreRuleLabel);
frame.Add(_updateRuleLabel);
// always run rules only mode for the manual gui
Updater.RulesOnly = true;
top.Add(menu);
Add(_info);
Add(_valuePane);
Add(frame);
if (!string.IsNullOrWhiteSpace(opts.FailuresCsv))
{
OpenReport(opts.FailuresCsv, (e) => throw e);
}
}
public void Test_RegexUpdateStrategy(DatabaseType dbType, bool provideCaptureGroup)
{
var db = GetCleanedServer(dbType);
var dbname = db.GetRuntimeName();
//the Failure was about Kansas and Toto
var failure = new Failure(
new FailurePart[]
{
new FailurePart("Kansas", FailureClassification.Location, 13),
new FailurePart("Toto", FailureClassification.Location, 28)
})
{
ProblemValue = "We aren't in Kansas anymore Toto",
ProblemField = "Narrative",
ResourcePrimaryKey = "1.2.3.4",
Resource = dbname + ".HappyOzz"
};
DataTable dt = new DataTable();
dt.Columns.Add("MyPk");
dt.Columns.Add("Narrative");
dt.PrimaryKey = new[] { dt.Columns["MyPk"] };
dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto");
var tbl = db.CreateTable("HappyOzz", dt);
//redacted string will be longer!
var col = tbl.DiscoverColumn("Narrative");
col.DataType.Resize(1000);
var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml"));
//make sure repeat test runs work properly
if (File.Exists(newRules.FullName))
{
File.Delete(newRules.FullName);
}
RowUpdater updater = new RowUpdater(newRules);
//But the user told us that only the Toto bit is a problem
string rule = provideCaptureGroup ? "(Toto)$" : "Toto$";
updater.RulesFactory = Mock.Of <IRulePatternFactory>(m => m.GetPattern(It.IsAny <object>(), It.IsAny <Failure>()) == rule);
//this is the thing we are actually testing, where we update based on the usersRule not the failing parts
updater.UpdateStrategy = new RegexUpdateStrategy();
//it should be novel i.e. require user decision
Assert.IsTrue(updater.OnLoad(db.Server, failure, out _));
updater.Update(db.Server, failure, null);//<- null here will trigger the rule pattern factory to prompt 'user' for pattern which is "(Toto)$"
var result = tbl.GetDataTable();
if (provideCaptureGroup)
{
Assert.AreEqual("We aren't in Kansas anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Expected update to only affect the capture group ToTo");
}
else
{
Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Because regex had no capture group we expected the update strategy to fallback on Failure Part matching");
}
//it should be updated automatically and not require user decision
Assert.IsFalse(updater.OnLoad(db.Server, failure, out _));
}
public void Test(DatabaseType dbType)
{
var db = GetCleanedServer(dbType);
var dbname = db.GetRuntimeName();
var failure = new Failure(
new FailurePart[]
{
new FailurePart("Kansas", FailureClassification.Location, 13),
new FailurePart("Toto", FailureClassification.Location, 28)
})
{
ProblemValue = "We aren't in Kansas anymore Toto",
ProblemField = "Narrative",
ResourcePrimaryKey = "1.2.3.4",
Resource = dbname + ".HappyOzz"
};
DataTable dt = new DataTable();
dt.Columns.Add("MyPk");
dt.Columns.Add("Narrative");
dt.PrimaryKey = new[] { dt.Columns["MyPk"] };
dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto");
var tbl = db.CreateTable("HappyOzz", dt);
//redacted string will be longer!
var col = tbl.DiscoverColumn("Narrative");
col.DataType.Resize(1000);
var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml"));
//make sure repeat test runs work properly
if (File.Exists(newRules.FullName))
{
File.Delete(newRules.FullName);
}
RowUpdater updater = new RowUpdater(newRules);
updater.UpdateStrategy = new ProblemValuesUpdateStrategy();
//it should be novel i.e. require user decision
Assert.IsTrue(updater.OnLoad(db.Server, failure, out _));
updater.Update(db.Server, failure, null);
var result = tbl.GetDataTable();
Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"]);
TestHelpers.Contains(
@"- Action: Report
IfColumn: Narrative
As: Location
IfPattern: ^We\ aren't\ in\ Kansas\ anymore\ Toto$
", File.ReadAllText(newRules.FullName));//btw slash space is a 'literal space' so legit
//it should be updated automatically and not require user decision
Assert.IsFalse(updater.OnLoad(db.Server, failure, out _));
}