private void Test(bool[] rowsToKeep) { int count = rowsToKeep.Length; // Build an identity column NumberColumn <int> column = new NumberColumn <int>(-1); for (int i = 0; i < rowsToKeep.Length; ++i) { column[i] = i; } // Build a RowUpdater and fake mapping to temp to check those results TableStub table = new TableStub(); RowUpdater updater = new RowUpdater(table, table); int[] rowsToTemp = Enumerable.Range(10, count).ToArray(); // Request Garbage Collection GarbageCollector.Collect <int>(column, null, rowsToKeep, updater, rowsToTemp); // Verify correct values were kept StringBuilder expected = new StringBuilder(); int expectedCount = 0; for (int i = 0; i < count; ++i) { if (rowsToKeep[i]) { expectedCount++; if (expected.Length > 0) { expected.Append(", "); } expected.Append(i); } } Assert.Equal(expectedCount, column.Count); Assert.Equal(expected.ToString(), String.Join(", ", column.OrderBy((i) => i))); // Verify rows removed are reported in the correct indices in temp or swapped in original column RowStub stub = new RowStub(table, 0); for (int i = 0; i < count; ++i) { stub.Index = i; updater.Update(stub); if (!rowsToKeep[i]) { Assert.Equal(10 + i, stub.Index); } else { Assert.Equal(i, column[stub.Index]); } } }
/// <summary> /// Creates a new instance that will connect to the database server (<paramref name="target"/>) and perform redactions using the <paramref name="updater"/> /// </summary> /// <param name="opts">CLI options for the process</param> /// <param name="target">DBMS to connect to for redacting</param> /// <param name="ignorer">Rules base for detecting false positives</param> /// <param name="updater">Rules base for redacting true positives</param> public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, IgnoreRuleGenerator ignorer, RowUpdater updater) { _log = LogManager.GetCurrentClassLogger(); if (string.IsNullOrWhiteSpace(opts.FailuresCsv)) { throw new Exception("Unattended requires a file of errors to process"); } var fi = new FileInfo(opts.FailuresCsv); if (!fi.Exists) { throw new FileNotFoundException($"Could not find Failures file '{fi.FullName}'"); } if (!opts.OnlyRules) { _target = target ?? throw new Exception("A single Target must be supplied for database updates"); } _reportReader = new ReportReader(fi); if (string.IsNullOrWhiteSpace(opts.UnattendedOutputPath)) { throw new Exception("An output path must be specified for Failures that could not be resolved"); } _outputFile = new FileInfo(opts.UnattendedOutputPath); _ignorer = ignorer; _updater = updater; }
public RulesView(ReportReader currentReport, IgnoreRuleGenerator ignorer, RowUpdater updater) { CurrentReport = currentReport; Ignorer = ignorer; Updater = updater; Modal = true; var lblInitialSummary = new Label($"There are {ignorer.Rules.Count} ignore rules and {updater.Rules.Count} update rules. Current report contains {CurrentReport.Failures.Length:N0} Failures"); Add(lblInitialSummary); var lblEvaluate = new Label($"Evaluate:") { Y = Pos.Bottom(lblInitialSummary) + 1 }; Add(lblEvaluate); var ruleCollisions = new Button("Rule Coverage") { Y = Pos.Bottom(lblEvaluate) }; ruleCollisions.Clicked += () => EvaluateRuleCoverage(); Add(ruleCollisions); _treeView = new TreeView() { Y = Pos.Bottom(ruleCollisions) + 1, Width = Dim.Fill(), Height = Dim.Fill(1) }; _treeView.KeyPress += _treeView_KeyPress; Add(_treeView); var close = new Button("Close", true) { Y = Pos.Bottom(_treeView) }; close.Clicked += () => Quit(); Add(close); }
private static int OnParse(GlobalOptions globals, IsIdentifiableReviewerOptions opts) { FansiImplementations.Load(); Deserializer d = new Deserializer(); List <Target> targets; try { var file = new FileInfo(opts.TargetsFile); if (!file.Exists) { Console.Write($"Could not find '{file.FullName}'"); return(1); } var contents = File.ReadAllText(file.FullName); if (string.IsNullOrWhiteSpace(contents)) { Console.Write($"Targets file is empty '{file.FullName}'"); return(2); } targets = d.Deserialize <List <Target> >(contents); if (!targets.Any()) { Console.Write($"Targets file did not contain any valid targets '{file.FullName}'"); return(3); } } catch (Exception e) { Console.WriteLine($"Error deserializing '{opts.TargetsFile}'"); Console.WriteLine(e.Message); return(4); } if (opts.OnlyRules) { Console.WriteLine("Skipping Connection Tests"); } else { Console.WriteLine("Running Connection Tests"); try { foreach (Target t in targets) { Console.WriteLine(t.Discover().Exists() ? $"Successfully connected to {t.Name}" : $"Failed to connect to {t.Name}"); } } catch (Exception e) { Console.WriteLine("Error Validating Targets"); Console.WriteLine(e.ToString()); return(10); } } //for updater try to match the ProblemValue words var updater = new RowUpdater(new FileInfo(opts.RedList)) { RulesOnly = opts.OnlyRules, RulesFactory = new MatchProblemValuesPatternFactory() }; //for Ignorer match the whole string var ignorer = new IgnoreRuleGenerator(new FileInfo(opts.IgnoreList)); try { if (!string.IsNullOrWhiteSpace(opts.UnattendedOutputPath)) { //run unattended if (targets.Count != 1) { throw new Exception("Unattended requires a single entry in Targets"); } var unattended = new UnattendedReviewer(opts, targets.Single(), ignorer, updater); return(unattended.Run()); } else { Console.WriteLine("Press any key to launch GUI"); Console.ReadKey(); //run interactive Application.Init(); var mainWindow = new MainWindow(opts, ignorer, updater); Application.Top.Add(mainWindow); Application.Run(); return(0); } } catch (Exception e) { Console.Write(e.Message); int tries = 5; while (Application.Top != null && tries-- > 0) { try { Application.RequestStop(); } catch (Exception) { Console.WriteLine("Failed to terminate GUI on crash"); } } return(99); } }
public MainWindow(IsIdentifiableReviewerOptions opts, IgnoreRuleGenerator ignorer, RowUpdater updater) { Ignorer = ignorer; Updater = updater; _origUpdaterRulesFactory = updater.RulesFactory; _origIgnorerRulesFactory = ignorer.RulesFactory; X = 0; Y = 1; Width = Dim.Fill(); Height = Dim.Fill(); var top = Application.Top; var menu = new MenuBar(new MenuBarItem [] { new MenuBarItem("_File (F9)", new MenuItem [] { new MenuItem("_Open Report", null, OpenReport), new MenuItem("_Quit", null, () => { top.Running = false; }) }), new MenuBarItem("_Options", new MenuItem [] { miCustomPatterns = new MenuItem("_Custom Patterns", null, ToggleCustomPatterns) { CheckType = MenuItemCheckStyle.Checked, Checked = false } }), new MenuBarItem("_View", new MenuItem [] { new MenuItem("_Rules", null, ViewRules), }) }); _info = new Label("Info") { X = 0, Y = 0, Width = Dim.Fill(), Height = 1 }; _info.ColorScheme = _greyOnBlack; _valuePane = new FailureView() { X = 0, Y = 1, Width = Dim.Fill(), Height = 10, }; var frame = new FrameView("Options") { X = 0, Y = 11, Width = Dim.Fill(), Height = Dim.Fill() }; var ignoreButton = new Button("Ignore") { X = 0 }; ignoreButton.Clicked += Ignore; frame.Add(ignoreButton); var updateButton = new Button("Update") { X = 11 }; updateButton.Clicked += Update; frame.Add(updateButton); _gotoTextField = new TextField("1") { X = 28, Width = 5 }; _gotoTextField.TextChanged += (s) => GoTo(); frame.Add(_gotoTextField); frame.Add(new Label(23, 0, "GoTo:")); var prevButton = new Button("Prev") { X = 0, Y = 1 }; prevButton.Clicked += () => GoToRelative(-1); frame.Add(prevButton); var nextButton = new Button("Next") { X = 11, Y = 1 }; nextButton.Clicked += () => GoToRelative(1); frame.Add(nextButton); var undoButton = new Button("unDo") { X = 11, Y = 2 }; undoButton.Clicked += () => Undo(); frame.Add(undoButton); frame.Add(new Label(0, 4, "Default Patterns")); _ignoreRuleLabel = new Label(0, 5, "Ignore:"); _updateRuleLabel = new Label(0, 6, "Update:");; frame.Add(_ignoreRuleLabel); frame.Add(_updateRuleLabel); // always run rules only mode for the manual gui Updater.RulesOnly = true; top.Add(menu); Add(_info); Add(_valuePane); Add(frame); if (!string.IsNullOrWhiteSpace(opts.FailuresCsv)) { OpenReport(opts.FailuresCsv, (e) => throw e); } }
public void Test_RegexUpdateStrategy(DatabaseType dbType, bool provideCaptureGroup) { var db = GetCleanedServer(dbType); var dbname = db.GetRuntimeName(); //the Failure was about Kansas and Toto var failure = new Failure( new FailurePart[] { new FailurePart("Kansas", FailureClassification.Location, 13), new FailurePart("Toto", FailureClassification.Location, 28) }) { ProblemValue = "We aren't in Kansas anymore Toto", ProblemField = "Narrative", ResourcePrimaryKey = "1.2.3.4", Resource = dbname + ".HappyOzz" }; DataTable dt = new DataTable(); dt.Columns.Add("MyPk"); dt.Columns.Add("Narrative"); dt.PrimaryKey = new[] { dt.Columns["MyPk"] }; dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto"); var tbl = db.CreateTable("HappyOzz", dt); //redacted string will be longer! var col = tbl.DiscoverColumn("Narrative"); col.DataType.Resize(1000); var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml")); //make sure repeat test runs work properly if (File.Exists(newRules.FullName)) { File.Delete(newRules.FullName); } RowUpdater updater = new RowUpdater(newRules); //But the user told us that only the Toto bit is a problem string rule = provideCaptureGroup ? "(Toto)$" : "Toto$"; updater.RulesFactory = Mock.Of <IRulePatternFactory>(m => m.GetPattern(It.IsAny <object>(), It.IsAny <Failure>()) == rule); //this is the thing we are actually testing, where we update based on the usersRule not the failing parts updater.UpdateStrategy = new RegexUpdateStrategy(); //it should be novel i.e. require user decision Assert.IsTrue(updater.OnLoad(db.Server, failure, out _)); updater.Update(db.Server, failure, null);//<- null here will trigger the rule pattern factory to prompt 'user' for pattern which is "(Toto)$" var result = tbl.GetDataTable(); if (provideCaptureGroup) { Assert.AreEqual("We aren't in Kansas anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Expected update to only affect the capture group ToTo"); } else { Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Because regex had no capture group we expected the update strategy to fallback on Failure Part matching"); } //it should be updated automatically and not require user decision Assert.IsFalse(updater.OnLoad(db.Server, failure, out _)); }
public void Test(DatabaseType dbType) { var db = GetCleanedServer(dbType); var dbname = db.GetRuntimeName(); var failure = new Failure( new FailurePart[] { new FailurePart("Kansas", FailureClassification.Location, 13), new FailurePart("Toto", FailureClassification.Location, 28) }) { ProblemValue = "We aren't in Kansas anymore Toto", ProblemField = "Narrative", ResourcePrimaryKey = "1.2.3.4", Resource = dbname + ".HappyOzz" }; DataTable dt = new DataTable(); dt.Columns.Add("MyPk"); dt.Columns.Add("Narrative"); dt.PrimaryKey = new[] { dt.Columns["MyPk"] }; dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto"); var tbl = db.CreateTable("HappyOzz", dt); //redacted string will be longer! var col = tbl.DiscoverColumn("Narrative"); col.DataType.Resize(1000); var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml")); //make sure repeat test runs work properly if (File.Exists(newRules.FullName)) { File.Delete(newRules.FullName); } RowUpdater updater = new RowUpdater(newRules); updater.UpdateStrategy = new ProblemValuesUpdateStrategy(); //it should be novel i.e. require user decision Assert.IsTrue(updater.OnLoad(db.Server, failure, out _)); updater.Update(db.Server, failure, null); var result = tbl.GetDataTable(); Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"]); TestHelpers.Contains( @"- Action: Report IfColumn: Narrative As: Location IfPattern: ^We\ aren't\ in\ Kansas\ anymore\ Toto$ ", File.ReadAllText(newRules.FullName));//btw slash space is a 'literal space' so legit //it should be updated automatically and not require user decision Assert.IsFalse(updater.OnLoad(db.Server, failure, out _)); }