public void TestAreIdentical()
{
var rule1 = new IsIdentifiableRule();
var rule2 = new IsIdentifiableRule();
Assert.IsTrue(rule1.AreIdentical(rule2));
rule2.IfPattern = "\r\n";
Assert.IsFalse(rule1.AreIdentical(rule2));
rule1.IfPattern = "\r\n";
Assert.IsTrue(rule1.AreIdentical(rule2));
rule2.IfColumn = "MyCol";
Assert.IsFalse(rule1.AreIdentical(rule2));
rule1.IfColumn = "MyCol";
Assert.IsTrue(rule1.AreIdentical(rule2));
rule2.Action = RuleAction.Ignore;
rule1.Action = RuleAction.Report;
Assert.IsFalse(rule1.AreIdentical(rule2, true));
Assert.IsTrue(rule1.AreIdentical(rule2, false));
rule2.Action = RuleAction.Report;
rule1.Action = RuleAction.Report;
Assert.IsTrue(rule1.AreIdentical(rule2, false));
Assert.IsTrue(rule1.AreIdentical(rule2, true));
}
/// <summary>
/// Adds a new rule (both to the <see cref="RulesFile"/> and the in memory <see cref="Rules"/> collection).
/// </summary>
/// <param name="f"></param>
/// <param name="action"></param>
/// <returns>The new / existing rule that covers failure</returns>
protected IsIdentifiableRule Add(Failure f, RuleAction action)
{
var rule = new IsIdentifiableRule
{
Action = action,
IfColumn = f.ProblemField,
IfPattern = RulesFactory.GetPattern(this, f),
As =
action == RuleAction.Ignore?
FailureClassification.None :
f.Parts.Select(p => p.Classification).FirstOrDefault()
};
//don't add identical rules
if (Rules.Any(r => r.AreIdentical(rule)))
{
return(rule);
}
Rules.Add(rule);
var contents = Serialize(rule, true);
File.AppendAllText(RulesFile.FullName, contents);
History.Push(new OutBaseHistory(rule, contents));
return(rule);
}
public void Test_RegexMultipleMatches(bool isReport)
{
var rule = new IsIdentifiableRule()
{
Action = isReport ? RuleAction.Report : RuleAction.Ignore,
IfColumn = "Modality",
IfPattern = "[0-9],",
As = FailureClassification.Date
};
Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("MODALITY", "1,2,3", out IEnumerable <FailurePart> bad));
if (isReport)
{
var b = bad.ToArray();
Assert.AreEqual(2, b.Length);
Assert.AreEqual("1,", b[0].Word);
Assert.AreEqual(FailureClassification.Date, b[0].Classification);
Assert.AreEqual(0, b[0].Offset);
Assert.AreEqual("2,", b[1].Word);
Assert.AreEqual(FailureClassification.Date, b[1].Classification);
Assert.AreEqual(2, b[1].Offset);
}
else
{
Assert.IsEmpty(bad);
}
Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _));
}
public void TestOneRule_NoColumn_NoPattern(bool isReport)
{
//rule is to ignore everything
var rule = new IsIdentifiableRule()
{
Action = isReport ? RuleAction.Report : RuleAction.Ignore,
};
var ex = Assert.Throws <Exception>(() => rule.Apply("Modality", "CT", out _));
Assert.AreEqual("Illegal rule setup. You must specify either a column or a pattern (or both)", ex.Message);
}
/// <summary>
/// Update the database <paramref name="server"/> to redact the <paramref name="failure"/>.
/// </summary>
/// <param name="server">Where to connect to get the data, can be null if <see cref="RulesOnly"/> is true</param>
/// <param name="failure">The failure to redact/create a rule for</param>
/// <param name="usingRule">Pass null to create a new rule or give value to reuse an existing rule</param>
public void Update(DiscoveredServer server, Failure failure, IsIdentifiableRule usingRule)
{
//theres no rule yet so create one (and add to RedList.yaml)
if (usingRule == null)
{
usingRule = Add(failure, RuleAction.Report);
}
//if we are running in rules only mode we don't need to also update the database
if (RulesOnly)
{
return;
}
var syntax = server.GetQuerySyntaxHelper();
//the fully specified name e.g. [mydb]..[mytbl]
string tableName = failure.Resource;
var tokens = tableName.Split('.', StringSplitOptions.RemoveEmptyEntries);
var db = tokens.First();
tableName = tokens.Last();
if (string.IsNullOrWhiteSpace(db) || string.IsNullOrWhiteSpace(tableName) || string.Equals(db, tableName))
{
throw new NotSupportedException($"Could not understand table name {failure.Resource}, maybe it is not full specified with a valid database and table name?");
}
db = syntax.GetRuntimeName(db);
tableName = syntax.GetRuntimeName(tableName);
DiscoveredTable table = server.ExpectDatabase(db).ExpectTable(tableName);
//if we've never seen this table before
if (!_primaryKeys.ContainsKey(table))
{
var pk = table.DiscoverColumns().SingleOrDefault(k => k.IsPrimaryKey);
_primaryKeys.Add(table, pk);
}
using (var con = server.GetConnection())
{
con.Open();
foreach (var sql in UpdateStrategy.GetUpdateSql(table, _primaryKeys, failure, usingRule))
{
var cmd = server.GetCommand(sql, con);
cmd.ExecuteNonQuery();
}
}
}
public void TestOneRule_NoColumn_WithPattern(bool isReport)
{
var rule = new IsIdentifiableRule()
{
Action = isReport ? RuleAction.Report : RuleAction.Ignore,
IfPattern = "^CT$",
As = FailureClassification.Date
};
Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("Modality", "CT", out _));
Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("ImageType", "CT", out _)); //ignore both because no restriction on column
Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _));
}
public void TestOneRule_IsColumnMatch_WithPattern(bool isReport)
{
var rule = new IsIdentifiableRule()
{
Action = isReport ? RuleAction.Report : RuleAction.Ignore,
IfColumn = "Modality",
IfPattern = "^CT$",
As = FailureClassification.Date
};
Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("Modality", "CT", out _));
Assert.AreEqual(RuleAction.None, rule.Apply("Modality", "MR", out _));
Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _));
}
/// <summary>
/// Serializes the <paramref name="rule"/> into yaml optionally with a comment at the start
/// </summary>
/// <param name="rule"></param>
/// <param name="addCreatorComment"></param>
/// <returns></returns>
private string Serialize(IsIdentifiableRule rule, bool addCreatorComment)
{
var serializer = new SerializerBuilder()
.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults)
.Build();
string yaml = serializer.Serialize(new List <IsIdentifiableRule> {
rule
});
if (!addCreatorComment)
{
return(yaml);
}
return($"#{Environment.UserName} - {DateTime.Now}" + Environment.NewLine +
yaml);
}
/// <summary>
/// Handler for loading <paramref name="failure"/>. If the user previously made an update decision an
/// update will transparently happen for this record and false is returned.
/// </summary>
/// <param name="server"></param>
/// <param name="failure"></param>
/// <param name="rule">The first rule that covered the <paramref name="failure"/></param>
/// <returns>True if <paramref name="failure"/> is novel and not seen before</returns>
public bool OnLoad(DiscoveredServer server, Failure failure, out IsIdentifiableRule rule)
{
rule = null;
//we have bigger problems than if this is novel!
if (server == null)
{
return(true);
}
//if we have seen this before
if (IsCoveredByExistingRule(failure, out rule))
{
//since user has issued an update for this exact problem before we can update this one too
Update(server, failure, rule);
//and return false to indicate that it is not a novel issue
return(false);
}
return(true);
}
public void TestOneRule_IsColumnMatch_NoPattern(bool isReport)
{
var rule = new IsIdentifiableRule()
{
Action = isReport ? RuleAction.Report : RuleAction.Ignore,
IfColumn = "Modality",
As = FailureClassification.Date
};
Assert.AreEqual(isReport? RuleAction.Report : RuleAction.Ignore, rule.Apply("MODALITY", "CT", out IEnumerable <FailurePart> bad));
if (isReport)
{
Assert.AreEqual(FailureClassification.Date, bad.Single().Classification);
}
else
{
Assert.IsEmpty(bad);
}
Assert.AreEqual(RuleAction.None, rule.Apply("ImageType", "PRIMARY", out _));
}
/// <summary>
/// Generates 1 UPDATE statement per <see cref="Failure.Parts"/> for redacting the current <paramref name="failure"/>
/// </summary>
/// <param name="table"></param>
/// <param name="primaryKeys"></param>
/// <param name="failure"></param>
/// <param name="usingRule"></param>
/// <returns></returns>
public override IEnumerable <string> GetUpdateSql(DiscoveredTable table,
Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule)
{
var syntax = table.GetQuerySyntaxHelper();
foreach (var part in failure.Parts)
{
yield return(GetUpdateWordSql(table, primaryKeys, syntax, failure, part.Word));
}
}
public RuleUsageNode(OutBase rulebase, IsIdentifiableRule rule, int numberOfTimesUsed)
{
Rulebase = rulebase;
Rule = rule;
NumberOfTimesUsed = numberOfTimesUsed;
}
/// <summary> /// Override to generate one or more SQL statements that will fully redact a given <see cref="Failure"/> in the <paramref name="table"/> /// </summary> /// <param name="table"></param> /// <param name="primaryKeys"></param> /// <param name="failure"></param> /// <param name="usingRule"></param> /// <returns></returns> public abstract IEnumerable <string> GetUpdateSql(DiscoveredTable table, Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule);
/// <summary>
/// Removes an existing rule and flushes out to disk
/// </summary>
/// <param name="rule"></param>
/// <returns>True if the rule existed and was successfully deleted in memory and on disk</returns>
public bool Delete(IsIdentifiableRule rule)
{
return(Rules.Remove(rule) && Purge(Serialize(rule, false), $"# Rule deleted by {Environment.UserName} - {DateTime.Now}{Environment.NewLine}"));
}
/// <summary>
/// Returns true if there are any rules that already exactly cover the given <paramref name="failure"/>
/// </summary>
/// <param name="failure"></param>
/// <param name="match">The first rule that matches the <paramref name="failure"/></param>
/// <returns></returns>
protected bool IsCoveredByExistingRule(Failure failure, out IsIdentifiableRule match)
{
//if any rule matches then we are covered by an existing rule
match = Rules.FirstOrDefault(r => r.Apply(failure.ProblemField, failure.ProblemValue, out _) != RuleAction.None);
return(match != null);
}
/// <summary>
/// When a new <paramref name="failure"/> is loaded, is it already covered by existing rules i.e. rules you are
/// working on now that have been added since the report was generated
/// </summary>
/// <param name="failure"></param>
/// <param name="existingRule">The rule which already covers this failure</param>
/// <returns>true if it is novel</returns>
public bool OnLoad(Failure failure, out IsIdentifiableRule existingRule)
{
//get user ot make a decision only if it is NOT covered by an existing rule
return(!IsCoveredByExistingRule(failure, out existingRule));
}
/// <summary>
/// Returns SQL for updating the <paramref name="table"/> to redact the capture groups in <see cref="IsIdentifiableRule.IfPattern"/>. If no capture groups are represented in the <paramref name="usingRule"/> then this class falls back on <see cref="ProblemValuesUpdateStrategy"/>
/// </summary>
/// <param name="table"></param>
/// <param name="primaryKeys"></param>
/// <param name="failure"></param>
/// <param name="usingRule"></param>
/// <returns></returns>
public override IEnumerable <string> GetUpdateSql(DiscoveredTable table, Dictionary <DiscoveredTable, DiscoveredColumn> primaryKeys, Failure failure, IsIdentifiableRule usingRule)
{
if (usingRule == null || string.IsNullOrWhiteSpace(usingRule.IfPattern))
{
return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule));
}
try
{
Regex r = new Regex(usingRule.IfPattern);
var match = r.Match(failure.ProblemValue);
//Group 1 (index 0) is always the full match, we want selective updates
if (match.Success && match.Groups.Count > 1)
{
var syntax = table.GetQuerySyntaxHelper();
//update the capture groups of the Regex
return(match.Groups.Cast <Group>().Skip(1).Select(m => GetUpdateWordSql(table, primaryKeys, syntax, failure, m.Value)));
}
//The Regex did not have capture groups or did not match the failure
return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule));
}
catch (Exception)
{
//The Regex pattern was bad or something else bad went wrong
return(_fallback.GetUpdateSql(table, primaryKeys, failure, usingRule));
}
}
public CollidingRulesNode(IsIdentifiableRule ignoreRule, IsIdentifiableRule updateRule, Failure f)
{
this.IgnoreRule = ignoreRule;
this.UpdateRule = updateRule;
this.CollideOn = new List <Failure>(new [] { f });
}
/// <summary>
/// Records a serialized <see cref="Rule"/>
/// </summary>
/// <param name="rule"></param>
/// <param name="yaml"></param>
public OutBaseHistory(IsIdentifiableRule rule, string yaml)
{
Rule = rule;
Yaml = yaml;
}
