示例#1
0
        public void Test_RegexUpdateStrategy(DatabaseType dbType, bool provideCaptureGroup)
        {
            var db     = GetCleanedServer(dbType);
            var dbname = db.GetRuntimeName();

            //the Failure was about Kansas and Toto
            var failure = new Failure(
                new FailurePart[]
            {
                new FailurePart("Kansas", FailureClassification.Location, 13),
                new FailurePart("Toto", FailureClassification.Location, 28)
            })
            {
                ProblemValue       = "We aren't in Kansas anymore Toto",
                ProblemField       = "Narrative",
                ResourcePrimaryKey = "1.2.3.4",
                Resource           = dbname + ".HappyOzz"
            };

            DataTable dt = new DataTable();

            dt.Columns.Add("MyPk");
            dt.Columns.Add("Narrative");

            dt.PrimaryKey = new[] { dt.Columns["MyPk"] };

            dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto");

            var tbl = db.CreateTable("HappyOzz", dt);

            //redacted string will be longer!
            var col = tbl.DiscoverColumn("Narrative");

            col.DataType.Resize(1000);

            var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml"));

            //make sure repeat test runs work properly
            if (File.Exists(newRules.FullName))
            {
                File.Delete(newRules.FullName);
            }

            RowUpdater updater = new RowUpdater(newRules);

            //But the user told us that only the Toto bit is a problem
            string rule = provideCaptureGroup ? "(Toto)$" : "Toto$";

            updater.RulesFactory = Mock.Of <IRulePatternFactory>(m => m.GetPattern(It.IsAny <object>(), It.IsAny <Failure>()) == rule);

            //this is the thing we are actually testing, where we update based on the usersRule not the failing parts
            updater.UpdateStrategy = new RegexUpdateStrategy();

            //it should be novel i.e. require user decision
            Assert.IsTrue(updater.OnLoad(db.Server, failure, out _));

            updater.Update(db.Server, failure, null);//<- null here will trigger the rule pattern factory to prompt 'user' for pattern which is "(Toto)$"

            var result = tbl.GetDataTable();

            if (provideCaptureGroup)
            {
                Assert.AreEqual("We aren't in Kansas anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Expected update to only affect the capture group ToTo");
            }
            else
            {
                Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"], "Because regex had no capture group we expected the update strategy to fallback on Failure Part matching");
            }

            //it should be updated automatically and not require user decision
            Assert.IsFalse(updater.OnLoad(db.Server, failure, out _));
        }
示例#2
0
        /// <summary>
        /// Connects to the database and runs all failures through the rules base performing redactions as required
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            //In RulesOnly mode this will be null
            var server = _target?.Discover();
            List <Exception> errors = new List <Exception>();

            var storeReport = new FailureStoreReport(_outputFile.Name, 100);

            Stopwatch sw = new Stopwatch();

            sw.Start();

            using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile))
            {
                IsIdentifiableRule updateRule;

                storeReport.AddDestination(storeReportDestination);

                while (_reportReader.Next())
                {
                    bool noUpdate;

                    try
                    {
                        noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule);
                    }
                    catch (Exception e)
                    {
                        errors.Add(e);
                        continue;
                    }

                    //is it novel for updater
                    if (noUpdate)
                    {
                        //is it novel for ignorer
                        if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule))
                        {
                            //we can't process it unattended
                            storeReport.Add(_reportReader.Current);
                            Unresolved++;
                        }
                        else
                        {
                            if (!_ignoreRulesUsed.ContainsKey(ignoreRule))
                            {
                                _ignoreRulesUsed.Add(ignoreRule, 1);
                            }
                            else
                            {
                                _ignoreRulesUsed[ignoreRule]++;
                            }

                            Ignores++;
                        }
                    }
                    else
                    {
                        if (!_updateRulesUsed.ContainsKey(updateRule))
                        {
                            _updateRulesUsed.Add(updateRule, 1);
                        }
                        else
                        {
                            _updateRulesUsed[updateRule]++;
                        }

                        Updates++;
                    }

                    Total++;

                    if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000)
                    {
                        Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true);
                        sw.Restart();
                    }
                }

                storeReport.CloseReport();
            }

            Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
                                                                          _ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);

            Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
                                                                          _updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);

            Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false);

            Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true);
            return(0);
        }
示例#3
0
        public void Test(DatabaseType dbType)
        {
            var db     = GetCleanedServer(dbType);
            var dbname = db.GetRuntimeName();

            var failure = new Failure(
                new FailurePart[]
            {
                new FailurePart("Kansas", FailureClassification.Location, 13),
                new FailurePart("Toto", FailureClassification.Location, 28)
            })
            {
                ProblemValue       = "We aren't in Kansas anymore Toto",
                ProblemField       = "Narrative",
                ResourcePrimaryKey = "1.2.3.4",
                Resource           = dbname + ".HappyOzz"
            };

            DataTable dt = new DataTable();

            dt.Columns.Add("MyPk");
            dt.Columns.Add("Narrative");

            dt.PrimaryKey = new[] { dt.Columns["MyPk"] };

            dt.Rows.Add("1.2.3.4", "We aren't in Kansas anymore Toto");

            var tbl = db.CreateTable("HappyOzz", dt);

            //redacted string will be longer!
            var col = tbl.DiscoverColumn("Narrative");

            col.DataType.Resize(1000);

            var newRules = new FileInfo(Path.Combine(TestContext.CurrentContext.WorkDirectory, "Redlist.yaml"));

            //make sure repeat test runs work properly
            if (File.Exists(newRules.FullName))
            {
                File.Delete(newRules.FullName);
            }

            RowUpdater updater = new RowUpdater(newRules);

            updater.UpdateStrategy = new ProblemValuesUpdateStrategy();

            //it should be novel i.e. require user decision
            Assert.IsTrue(updater.OnLoad(db.Server, failure, out _));

            updater.Update(db.Server, failure, null);

            var result = tbl.GetDataTable();

            Assert.AreEqual("We aren't in SMI_REDACTED anymore SMI_REDACTED", result.Rows[0]["Narrative"]);

            TestHelpers.Contains(
                @"- Action: Report
  IfColumn: Narrative
  As: Location
  IfPattern: ^We\ aren't\ in\ Kansas\ anymore\ Toto$
", File.ReadAllText(newRules.FullName));//btw slash space is a 'literal space' so legit

            //it should be updated automatically and not require user decision
            Assert.IsFalse(updater.OnLoad(db.Server, failure, out _));
        }