Exemplo n.º 1
0
        public void TestReconstructionFromCsv()
        {
            var opts = new IsIdentifiableRelationalDatabaseOptions();
            var dir  = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory);

            foreach (var f in dir.GetFiles("*HappyOzz*.csv"))
            {
                f.Delete();
            }

            opts.DestinationCsvFolder = dir.FullName;
            opts.TableName            = "HappyOzz";
            opts.StoreReport          = true;

            FailureStoreReport report = new FailureStoreReport("HappyOzz", 1000);

            report.AddDestinations(opts);

            var failure = new Failure(
                new FailurePart[]
            {
                new FailurePart("Kansas", FailureClassification.Location, 12),
                new FailurePart("Toto", FailureClassification.Location, 28)
            })
            {
                ProblemValue       = "We aren't in Kansas anymore Toto",
                ProblemField       = "Narrative",
                ResourcePrimaryKey = "1.2.3",
                Resource           = "FunBooks.HappyOzz"
            };

            report.Add(failure);

            report.CloseReport();

            var created = dir.GetFiles("*HappyOzz*.csv").SingleOrDefault();

            Assert.IsNotNull(created);

            var report2   = new FailureStoreReport("", 0);
            var failures2 = report2.Deserialize(created).ToArray();

            //read failure ok
            Assert.AreEqual(1, failures2.Length);
            Assert.AreEqual(failure.ProblemValue, failures2[0].ProblemValue);
            Assert.AreEqual(failure.ProblemField, failures2[0].ProblemField);
            Assert.AreEqual(failure.ResourcePrimaryKey, failures2[0].ResourcePrimaryKey);
            Assert.AreEqual(failure.Resource, failures2[0].Resource);

            //read parts ok
            Assert.AreEqual(2, failures2[0].Parts.Count);

            Assert.AreEqual(failure.Parts[0].Classification, failures2[0].Parts[0].Classification);
            Assert.AreEqual(failure.Parts[0].Offset, failures2[0].Parts[0].Offset);
            Assert.AreEqual(failure.Parts[0].Word, failures2[0].Parts[0].Word);

            Assert.AreEqual(failure.Parts[1].Classification, failures2[0].Parts[1].Classification);
            Assert.AreEqual(failure.Parts[1].Offset, failures2[0].Parts[1].Offset);
            Assert.AreEqual(failure.Parts[1].Word, failures2[0].Parts[1].Word);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Connects to the database and runs all failures through the rules base performing redactions as required
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            //In RulesOnly mode this will be null
            var server = _target?.Discover();
            List <Exception> errors = new List <Exception>();

            var storeReport = new FailureStoreReport(_outputFile.Name, 100);

            Stopwatch sw = new Stopwatch();

            sw.Start();

            using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile))
            {
                IsIdentifiableRule updateRule;

                storeReport.AddDestination(storeReportDestination);

                while (_reportReader.Next())
                {
                    bool noUpdate;

                    try
                    {
                        noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule);
                    }
                    catch (Exception e)
                    {
                        errors.Add(e);
                        continue;
                    }

                    //is it novel for updater
                    if (noUpdate)
                    {
                        //is it novel for ignorer
                        if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule))
                        {
                            //we can't process it unattended
                            storeReport.Add(_reportReader.Current);
                            Unresolved++;
                        }
                        else
                        {
                            if (!_ignoreRulesUsed.ContainsKey(ignoreRule))
                            {
                                _ignoreRulesUsed.Add(ignoreRule, 1);
                            }
                            else
                            {
                                _ignoreRulesUsed[ignoreRule]++;
                            }

                            Ignores++;
                        }
                    }
                    else
                    {
                        if (!_updateRulesUsed.ContainsKey(updateRule))
                        {
                            _updateRulesUsed.Add(updateRule, 1);
                        }
                        else
                        {
                            _updateRulesUsed[updateRule]++;
                        }

                        Updates++;
                    }

                    Total++;

                    if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000)
                    {
                        Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true);
                        sw.Restart();
                    }
                }

                storeReport.CloseReport();
            }

            Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
                                                                          _ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);

            Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine,
                                                                          _updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false);

            Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false);

            Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true);
            return(0);
        }