public void TestAreIdentical() { var rule1 = new IsIdentifiableRule(); var rule2 = new IsIdentifiableRule(); Assert.IsTrue(rule1.AreIdentical(rule2)); rule2.IfPattern = "\r\n"; Assert.IsFalse(rule1.AreIdentical(rule2)); rule1.IfPattern = "\r\n"; Assert.IsTrue(rule1.AreIdentical(rule2)); rule2.IfColumn = "MyCol"; Assert.IsFalse(rule1.AreIdentical(rule2)); rule1.IfColumn = "MyCol"; Assert.IsTrue(rule1.AreIdentical(rule2)); rule2.Action = RuleAction.Ignore; rule1.Action = RuleAction.Report; Assert.IsFalse(rule1.AreIdentical(rule2, true)); Assert.IsTrue(rule1.AreIdentical(rule2, false)); rule2.Action = RuleAction.Report; rule1.Action = RuleAction.Report; Assert.IsTrue(rule1.AreIdentical(rule2, false)); Assert.IsTrue(rule1.AreIdentical(rule2, true)); }
/// <summary> /// Adds a new rule (both to the <see cref="RulesFile"/> and the in memory <see cref="Rules"/> collection). /// </summary> /// <param name="f"></param> /// <param name="action"></param> /// <returns>The new / existing rule that covers failure</returns> protected IsIdentifiableRule Add(Failure f, RuleAction action) { var rule = new IsIdentifiableRule { Action = action, IfColumn = f.ProblemField, IfPattern = RulesFactory.GetPattern(this, f), As = action == RuleAction.Ignore? FailureClassification.None : f.Parts.Select(p => p.Classification).FirstOrDefault() }; //don't add identical rules if (Rules.Any(r => r.AreIdentical(rule))) { return(rule); } Rules.Add(rule); var contents = Serialize(rule, true); File.AppendAllText(RulesFile.FullName, contents); History.Push(new OutBaseHistory(rule, contents)); return(rule); }
public void Test_RegexMultipleMatches(bool isReport) { var rule = new IsIdentifiableRule() { Action = isReport ? RuleAction.Report : RuleAction.Ignore, IfColumn = "Modality", IfPattern = "[0-9],", As = FailureClassification.Date }; Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("MODALITY", "1,2,3", out IEnumerable <FailurePart> bad)); if (isReport) { var b = bad.ToArray(); Assert.AreEqual(2, b.Length); Assert.AreEqual("1,", b[0].Word); Assert.AreEqual(FailureClassification.Date, b[0].Classification); Assert.AreEqual(0, b[0].Offset); Assert.AreEqual("2,", b[1].Word); Assert.AreEqual(FailureClassification.Date, b[1].Classification); Assert.AreEqual(2, b[1].Offset); } else { Assert.IsEmpty(bad); } Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _)); }
public void TestOneRule_NoColumn_NoPattern(bool isReport) { //rule is to ignore everything var rule = new IsIdentifiableRule() { Action = isReport ? RuleAction.Report : RuleAction.Ignore, }; var ex = Assert.Throws <Exception>(() => rule.Apply("Modality", "CT", out _)); Assert.AreEqual("Illegal rule setup. You must specify either a column or a pattern (or both)", ex.Message); }
/// <summary> /// Update the database <paramref name="server"/> to redact the <paramref name="failure"/>. /// </summary> /// <param name="server">Where to connect to get the data, can be null if <see cref="RulesOnly"/> is true</param> /// <param name="failure">The failure to redact/create a rule for</param> /// <param name="usingRule">Pass null to create a new rule or give value to reuse an existing rule</param> public void Update(DiscoveredServer server, Failure failure, IsIdentifiableRule usingRule) { //theres no rule yet so create one (and add to RedList.yaml) if (usingRule == null) { usingRule = Add(failure, RuleAction.Report); } //if we are running in rules only mode we don't need to also update the database if (RulesOnly) { return; } var syntax = server.GetQuerySyntaxHelper(); //the fully specified name e.g. [mydb]..[mytbl] string tableName = failure.Resource; var tokens = tableName.Split('.', StringSplitOptions.RemoveEmptyEntries); var db = tokens.First(); tableName = tokens.Last(); if (string.IsNullOrWhiteSpace(db) || string.IsNullOrWhiteSpace(tableName) || string.Equals(db, tableName)) { throw new NotSupportedException($"Could not understand table name {failure.Resource}, maybe it is not full specified with a valid database and table name?"); } db = syntax.GetRuntimeName(db); tableName = syntax.GetRuntimeName(tableName); DiscoveredTable table = server.ExpectDatabase(db).ExpectTable(tableName); //if we've never seen this table before if (!_primaryKeys.ContainsKey(table)) { var pk = table.DiscoverColumns().SingleOrDefault(k => k.IsPrimaryKey); _primaryKeys.Add(table, pk); } using (var con = server.GetConnection()) { con.Open(); foreach (var sql in UpdateStrategy.GetUpdateSql(table, _primaryKeys, failure, usingRule)) { var cmd = server.GetCommand(sql, con); cmd.ExecuteNonQuery(); } } }
public void TestOneRule_NoColumn_WithPattern(bool isReport) { var rule = new IsIdentifiableRule() { Action = isReport ? RuleAction.Report : RuleAction.Ignore, IfPattern = "^CT$", As = FailureClassification.Date }; Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("Modality", "CT", out _)); Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("ImageType", "CT", out _)); //ignore both because no restriction on column Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _)); }
public void TestOneRule_IsColumnMatch_WithPattern(bool isReport) { var rule = new IsIdentifiableRule() { Action = isReport ? RuleAction.Report : RuleAction.Ignore, IfColumn = "Modality", IfPattern = "^CT$", As = FailureClassification.Date }; Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("Modality", "CT", out _)); Assert.AreEqual(RuleAction.None, rule.Apply("Modality", "MR", out _)); Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _)); }
/// <summary> /// Serializes the <paramref name="rule"/> into yaml optionally with a comment at the start /// </summary> /// <param name="rule"></param> /// <param name="addCreatorComment"></param> /// <returns></returns> private string Serialize(IsIdentifiableRule rule, bool addCreatorComment) { var serializer = new SerializerBuilder() .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) .Build(); string yaml = serializer.Serialize(new List <IsIdentifiableRule> { rule }); if (!addCreatorComment) { return(yaml); } return($"#{Environment.UserName} - {DateTime.Now}" + Environment.NewLine + yaml); }
/// <summary> /// Handler for loading <paramref name="failure"/>. If the user previously made an update decision an /// update will transparently happen for this record and false is returned. /// </summary> /// <param name="server"></param> /// <param name="failure"></param> /// <param name="rule">The first rule that covered the <paramref name="failure"/></param> /// <returns>True if <paramref name="failure"/> is novel and not seen before</returns> public bool OnLoad(DiscoveredServer server, Failure failure, out IsIdentifiableRule rule) { rule = null; //we have bigger problems than if this is novel! if (server == null) { return(true); } //if we have seen this before if (IsCoveredByExistingRule(failure, out rule)) { //since user has issued an update for this exact problem before we can update this one too Update(server, failure, rule); //and return false to indicate that it is not a novel issue return(false); } return(true); }
public void TestOneRule_IsColumnMatch_NoPattern(bool isReport) { var rule = new IsIdentifiableRule() { Action = isReport ? RuleAction.Report : RuleAction.Ignore, IfColumn = "Modality", As = FailureClassification.Date }; Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("MODALITY", "CT", out IEnumerable <FailurePart> bad)); if (isReport) { Assert.AreEqual(FailureClassification.Date, bad.Single().Classification); } else { Assert.IsEmpty(bad); } Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _)); }
/// <summary> /// Generates 1 UPDATE statement per <see cref="Failure.Parts"/> for redacting the current <paramref name="failure"/> /// </summary> /// <param name="table"></param> /// <param name="primaryKeys"></param> /// <param name="failure"></param> /// <param name="usingRule"></param> /// <returns></returns> public override IEnumerable <string> GetUpdateSql(DiscoveredTable table, Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule) { var syntax = table.GetQuerySyntaxHelper(); foreach (var part in failure.Parts) { yield return(GetUpdateWordSql(table, primaryKeys, syntax, failure, part.Word)); } }
public RuleUsageNode(OutBase rulebase, IsIdentifiableRule rule, int numberOfTimesUsed) { Rulebase = rulebase; Rule = rule; NumberOfTimesUsed = numberOfTimesUsed; }
/// <summary> /// Override to generate one or more SQL statements that will fully redact a given <see cref="Failure"/> in the <paramref name="table"/> /// </summary> /// <param name="table"></param> /// <param name="primaryKeys"></param> /// <param name="failure"></param> /// <param name="usingRule"></param> /// <returns></returns> public abstract IEnumerable <string> GetUpdateSql(DiscoveredTable table, Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule);
/// <summary> /// Removes an existing rule and flushes out to disk /// </summary> /// <param name="rule"></param> /// <returns>True if the rule existed and was successfully deleted in memory and on disk</returns> public bool Delete(IsIdentifiableRule rule) { return(Rules.Remove(rule) && Purge(Serialize(rule, false), $"# Rule deleted by {Environment.UserName} - {DateTime.Now}{Environment.NewLine}")); }
/// <summary> /// Returns true if there are any rules that already exactly cover the given <paramref name="failure"/> /// </summary> /// <param name="failure"></param> /// <param name="match">The first rule that matches the <paramref name="failure"/></param> /// <returns></returns> protected bool IsCoveredByExistingRule(Failure failure, out IsIdentifiableRule match) { //if any rule matches then we are covered by an existing rule match = Rules.FirstOrDefault(r => r.Apply(failure.ProblemField, failure.ProblemValue, out _) != RuleAction.None); return(match != null); }
/// <summary> /// When a new <paramref name="failure"/> is loaded, is it already covered by existing rules i.e. rules you are /// working on now that have been added since the report was generated /// </summary> /// <param name="failure"></param> /// <param name="existingRule">The rule which already covers this failure</param> /// <returns>true if it is novel</returns> public bool OnLoad(Failure failure, out IsIdentifiableRule existingRule) { //get user ot make a decision only if it is NOT covered by an existing rule return(!IsCoveredByExistingRule(failure, out existingRule)); }
/// <summary> /// Returns SQL for updating the <paramref name="table"/> to redact the capture groups in <see cref="IsIdentifiableRule.IfPattern"/>. If no capture groups are represented in the <paramref name="usingRule"/> then this class falls back on <see cref="ProblemValuesUpdateStrategy"/> /// </summary> /// <param name="table"></param> /// <param name="primaryKeys"></param> /// <param name="failure"></param> /// <param name="usingRule"></param> /// <returns></returns> public override IEnumerable <string> GetUpdateSql(DiscoveredTable table, Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule) { if (usingRule == null || string.IsNullOrWhiteSpace(usingRule.IfPattern)) { return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule)); } try { Regex r = new Regex(usingRule.IfPattern); var match = r.Match(failure.ProblemValue); //Group 1 (index 0) is always the full match, we want selective updates if (match.Success && match.Groups.Count > 1) { var syntax = table.GetQuerySyntaxHelper(); //update the capture groups of the Regex return(match.Groups.Cast <Group>().Skip(1).Select(m => GetUpdateWordSql(table, primaryKeys, syntax, failure, m.Value))); } //The Regex did not have capture groups or did not match the failure return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule)); } catch (Exception) { //The Regex pattern was bad or something else bad went wrong return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule)); } }
public CollidingRulesNode(IsIdentifiableRule ignoreRule, IsIdentifiableRule updateRule, Failure f) { this.IgnoreRule = ignoreRule; this.UpdateRule = updateRule; this.CollideOn = new List <Failure>(new [] { f }); }
/// <summary> /// Records a serialized <see cref="Rule"/> /// </summary> /// <param name="rule"></param> /// <param name="yaml"></param> public OutBaseHistory(IsIdentifiableRule rule, string yaml) { Rule = rule; Yaml = yaml; }