public void TestReconstructionFromCsv() { var opts = new IsIdentifiableRelationalDatabaseOptions(); var dir = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory); foreach (var f in dir.GetFiles("*HappyOzz*.csv")) { f.Delete(); } opts.DestinationCsvFolder = dir.FullName; opts.TableName = "HappyOzz"; opts.StoreReport = true; FailureStoreReport report = new FailureStoreReport("HappyOzz", 1000); report.AddDestinations(opts); var failure = new Failure( new FailurePart[] { new FailurePart("Kansas", FailureClassification.Location, 12), new FailurePart("Toto", FailureClassification.Location, 28) }) { ProblemValue = "We aren't in Kansas anymore Toto", ProblemField = "Narrative", ResourcePrimaryKey = "1.2.3", Resource = "FunBooks.HappyOzz" }; report.Add(failure); report.CloseReport(); var created = dir.GetFiles("*HappyOzz*.csv").SingleOrDefault(); Assert.IsNotNull(created); var report2 = new FailureStoreReport("", 0); var failures2 = report2.Deserialize(created).ToArray(); //read failure ok Assert.AreEqual(1, failures2.Length); Assert.AreEqual(failure.ProblemValue, failures2[0].ProblemValue); Assert.AreEqual(failure.ProblemField, failures2[0].ProblemField); Assert.AreEqual(failure.ResourcePrimaryKey, failures2[0].ResourcePrimaryKey); Assert.AreEqual(failure.Resource, failures2[0].Resource); //read parts ok Assert.AreEqual(2, failures2[0].Parts.Count); Assert.AreEqual(failure.Parts[0].Classification, failures2[0].Parts[0].Classification); Assert.AreEqual(failure.Parts[0].Offset, failures2[0].Parts[0].Offset); Assert.AreEqual(failure.Parts[0].Word, failures2[0].Parts[0].Word); Assert.AreEqual(failure.Parts[1].Classification, failures2[0].Parts[1].Classification); Assert.AreEqual(failure.Parts[1].Offset, failures2[0].Parts[1].Offset); Assert.AreEqual(failure.Parts[1].Word, failures2[0].Parts[1].Word); }
public ReportReader(FileInfo csvFile, Action <int> loadedRows, CancellationToken token) { var report = new FailureStoreReport("", 0); Failures = report.Deserialize(csvFile, loadedRows, token).ToArray(); }
public ReportReader(FileInfo csvFile) { var report = new FailureStoreReport("", 0); Failures = report.Deserialize(csvFile).ToArray(); }
/// <summary> /// Connects to the database and runs all failures through the rules base performing redactions as required /// </summary> /// <returns></returns> public int Run() { //In RulesOnly mode this will be null var server = _target?.Discover(); List <Exception> errors = new List <Exception>(); var storeReport = new FailureStoreReport(_outputFile.Name, 100); Stopwatch sw = new Stopwatch(); sw.Start(); using (var storeReportDestination = new CsvDestination(new IsIdentifiableDicomFileOptions(), _outputFile)) { IsIdentifiableRule updateRule; storeReport.AddDestination(storeReportDestination); while (_reportReader.Next()) { bool noUpdate; try { noUpdate = _updater.OnLoad(server, _reportReader.Current, out updateRule); } catch (Exception e) { errors.Add(e); continue; } //is it novel for updater if (noUpdate) { //is it novel for ignorer if (_ignorer.OnLoad(_reportReader.Current, out IsIdentifiableRule ignoreRule)) { //we can't process it unattended storeReport.Add(_reportReader.Current); Unresolved++; } else { if (!_ignoreRulesUsed.ContainsKey(ignoreRule)) { _ignoreRulesUsed.Add(ignoreRule, 1); } else { _ignoreRulesUsed[ignoreRule]++; } Ignores++; } } else { if (!_updateRulesUsed.ContainsKey(updateRule)) { _updateRulesUsed.Add(updateRule, 1); } else { _updateRulesUsed[updateRule]++; } Updates++; } Total++; if (Total % 10000 == 0 || sw.ElapsedMilliseconds > 5000) { Log($"Done {Total:N0} u={Updates:N0} i={Ignores:N0} o={Unresolved:N0} err={errors.Count:N0}", true); sw.Restart(); } } storeReport.CloseReport(); } Log($"Ignore Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, _ignoreRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false); Log($"Update Rules Used:" + Environment.NewLine + string.Join(Environment.NewLine, _updateRulesUsed.OrderBy(k => k.Value).Select(k => $"{k.Key.IfPattern} - {k.Value:N0}")), false); Log("Errors:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => e.ToString())), false); Log($"Finished {Total:N0} updates={Updates:N0} ignored={Ignores:N0} out={Unresolved:N0} err={errors.Count:N0}", true); return(0); }