public static string GetFactsOfRule(ValidationRule rule)
{
var sb = new StringBuilder();
foreach (var p in rule.Parameters)
{
sb.AppendLine(p.Name);
sb.AppendLine(TaxonomyEngine.CurrentEngine.CurrentTaxonomy.GetFactStringKeys(p.TaxFacts));
sb.AppendLine("________________________________________________");
}
return(sb.ToString());
}
public static void CheckConsistency(Taxonomy taxonomy, ValidationRule rule)
{
var sb = new StringBuilder();
var inconsistent = false;
sb.AppendLine(rule.RawInfo);
foreach (var p in rule.Parameters)
{
var tfcs = "";
var tfc = p.TaxFacts.Count;
var tfall = p.TaxFacts.SelectMany(i => i).Count();
if (tfall > tfc && !p.BindAsSequence)
{
}
tfcs = String.Format("{0}: {2}/{1}", p.Name, tfc, tfall);
sb.AppendLine(p.ToString() + " TFCS: " + tfcs);
foreach (var factgroup in p.TaxFacts)
{
if (!p.BindAsSequence && factgroup.Count > 1)
{
inconsistent = true;
sb.AppendLine(taxonomy.GetFactStringKeys(p.TaxFacts));
break;
}
}
}
sb.AppendLine("___________");
if (rule.ID == "eba_v0204_m")
{
}
if (inconsistent)
{
Logger.WriteLine(String.Format("Can't resolve rule {0}", rule.ID));
var validationfolder = TaxonomyEngine.LogFolder + "Validation\\";
var logfilename = Utilities.Strings.GetFileName(Logger.LogPath);
var outputpath = validationfolder + "Issues_" + logfilename;
Utilities.FS.AppendAllText(outputpath, sb.ToString());
}
}
public static void SetParamerterTypes(Taxonomy taxonomy, ValidationRule rule)
{
foreach (var p in rule.Parameters)
{
if (String.IsNullOrEmpty(p.Concept))
{
var firstfactgroup = p.TaxFacts.FirstOrDefault(i => i.Count > 0);
int firstfactindex = firstfactgroup != null ? firstfactgroup[0] : -2;
if (firstfactindex != -2)
{
var factkey = taxonomy.FactsManager.GetFactKey(firstfactindex);
if (factkey.Length > 0 && taxonomy.CounterFactParts.ContainsKey(factkey[0]))
{
p.Concept = taxonomy.CounterFactParts[factkey[0]];
}
}
}
if (p.Type == TypeEnum.Unknown)
{
p.Type = GetType(taxonomy, p.Concept);
}
}
}
public LogicalModel.Validation.ValidationRule GetLogicalRule(Hierarchy <XbrlIdentifiable> hrule, XbrlTaxonomyDocument document)
{
this.Document = document;
var tmp_rule = hrule.Copy();
//FixRule(tmp_rule);
var logicalrule = new LogicalModel.Validation.ValidationRule();
var valueassertion = tmp_rule.Item as ValueAssertion;
logicalrule.ID = valueassertion.ID;
//Utilities.Logger.WriteLine("Getting rule for " + logicalrule.ID);
logicalrule.LabelID = valueassertion.LabelID;
logicalrule.OriginalExpression = valueassertion.Test.Replace("\r\n", " ").Replace("\r", " ").Replace("\n", " ");
logicalrule.SetTaxonomy(this.Taxonomy);
var sb = new StringBuilder();
sb.AppendLine(logicalrule.DisplayText);
sb.AppendLine(valueassertion.Test);
var rawval = document.FileName + "\r\n" + document.LocalPath + "\r\n" + logicalrule.DisplayText + "\r\n" + logicalrule.OriginalExpression + "\r\n" + hrule.ToHierarchyString(i => i.ToString()) + "\r\n";
logicalrule.RawInfo = rawval;
Utilities.FS.AppendAllText(Taxonomy.TaxonomyValidationFolder + "Validations_XML.txt", rawval);
if (logicalrule.ID.Contains("1058"))/*,"es_v354"*/
{
}
var factvariables = tmp_rule.Where(i => i.Item is FactVariable);
foreach (var fv in factvariables)
{
tmp_rule.Remove(fv);
}
logicalrule.BaseQuery = GetQuery(tmp_rule);
logicalrule.BaseQuery.GetString(Taxonomy);
foreach (var fv in factvariables)
{
var factvariable = fv.Item as FactVariable;
var name = factvariable.Name;
var parameter = new LogicalModel.Validation.ValidationParameter(name, logicalrule.ID);
logicalrule.Parameters.Add(parameter);
parameter.BindAsSequence = factvariable.BindAsSequence;
parameter.FallBackValue = factvariable.FallbackValue;
parameter.BaseQuery = GetQuery(fv);
parameter.Concept = parameter.BaseQuery.GetConcept();
if (String.IsNullOrEmpty(parameter.Concept))
{
parameter.Concept = logicalrule.BaseQuery.GetConcept();
}
if (parameter.BaseQuery.HasDictFilter("find:filingIndicator"))
{
parameter.IsGeneral = true;
parameter.StringValue = "filingindicators";
}
ValidationRuleHelper.SetParamerterTypes(Taxonomy, logicalrule);
}
var factparameterqueries = logicalrule.Parameters.Where(i => !i.IsGeneral).Select(i => i.BaseQuery).ToArray();
var commconparameterquery = FactBaseQuery.GetCommonQuery(factparameterqueries);
if (commconparameterquery != null && commconparameterquery.HasFilters())
{
FactBaseQuery.MergeQueries(logicalrule.BaseQuery, commconparameterquery);
foreach (var pquery in factparameterqueries)
{
FactBaseQuery.RemoveQuery(pquery, commconparameterquery);
}
}
var sbquery = new StringBuilder();
if (1 == 1)
{
sbquery.AppendLine("RuleQuery: ");
sbquery.AppendLine(logicalrule.BaseQuery != null? logicalrule.BaseQuery.ToString():"");
sbquery.AppendLine();
foreach (var p in logicalrule.Parameters)
{
sbquery.AppendLine(p + " Query: ");
if (p.BaseQuery != null)
{
sbquery.AppendLine(p.BaseQuery != null ? p.BaseQuery.ToString() : "");
}
}
}
foreach (var gp in Taxonomy.ValidationParameters)
{
if (valueassertion.Test.Contains(gp.Value.XPathName))
{
if (gp.Key == "ReportingLevel")
{
var p_rl1 = new LogicalModel.Validation.ValidationParameter("ReportingLevel", logicalrule.ID);
p_rl1.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("_con") ? "con" : "ind";
p_rl1.Type = LogicalModel.TypeEnum.String;
p_rl1.IsGeneral = true;
logicalrule.Parameters.Add(p_rl1);
continue;
}
if (gp.Key == "AccountingStandard")
{
var p_rl2 = new LogicalModel.Validation.ValidationParameter("AccountingStandard", logicalrule.ID);
p_rl2.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("GAAP") ? "GAAP" : "IFRS";
p_rl2.Type = LogicalModel.TypeEnum.String;
p_rl2.IsGeneral = true;
logicalrule.Parameters.Add(p_rl2);
continue;
}
var p_rl = new LogicalModel.Validation.ValidationParameter(gp.Key, logicalrule.ID);
p_rl.StringValue = "Unresolved";
p_rl.Type = LogicalModel.TypeEnum.String;
p_rl.IsGeneral = true;
logicalrule.Parameters.Add(p_rl);
}
}
/*
* if (valueassertion.Test.Contains("$ReportingLevel"))
* {
* var p_rl1 = new LogicalModel.Validation.ValidationParameter("ReportingLevel", logicalrule.ID);
* p_rl1.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("_con") ? "con" : "ind";
* p_rl1.Type = LogicalModel.TypeEnum.String;
* p_rl1.IsGeneral = true;
* logicalrule.Parameters.Add(p_rl1);
* }
* if (valueassertion.Test.Contains("$AccountingStandard"))
* {
* var p_rl2 = new LogicalModel.Validation.ValidationParameter("AccountingStandard", logicalrule.ID);
* p_rl2.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("GAAP") ? "GAAP" : "IFRS";
* p_rl2.Type = LogicalModel.TypeEnum.String;
* p_rl2.IsGeneral = true;
* logicalrule.Parameters.Add(p_rl2);
* }
*/
logicalrule.SetTaxonomy(this.Taxonomy);
ValidationRuleHelper.ExecuteExplicitFiltering(this.Taxonomy, logicalrule);
ValidationRuleHelper.ExecuteImplicitFiltering(this.Taxonomy, logicalrule);
ValidationRuleHelper.ExecuteMatching(this.Taxonomy, logicalrule);
ValidationRuleHelper.CheckConsistency(this.Taxonomy, logicalrule);
//SetFacts(logicalrule);
return(logicalrule);
}
public static void GroupTaxFacts(Taxonomy taxonomy, Dictionary <int, int[]> factkeydict, ValidationRule rule, List <int> groupaspects)
{
if (groupaspects.Count == 0)
{
return;
}
var p1 = rule.Parameters.FirstOrDefault(i => !i.IsGeneral && !i.BindAsSequence);
var factparameters = rule.Parameters.Where(i => !i.IsGeneral).ToList();
var nonsequencedfactparameters = factparameters.Where(i => !i.BindAsSequence).ToList();
p1 = nonsequencedfactparameters.OrderByDescending(i => GetFactCount(i)).FirstOrDefault();
var sb = new StringBuilder();
foreach (var p in factparameters)
{
var tfc = p.TaxFacts.Count;
var tfall = p.TaxFacts.SelectMany(i => i).Count();
if (tfall > tfc && !p.BindAsSequence)
{
}
sb.AppendLine(String.Format("{0}: {2}/{1}", p.Name, tfc, tfall));
}
var groups = new List <Dictionary <string, List <int> > >();
var paramtaxfactdict = new Dictionary <string, List <List <int> > >();
if ((Object)p1 != null)
{
foreach (var fp in factparameters)
{
paramtaxfactdict.Add(fp.Name, new List <List <int> >());
}
var otherfactparameters = rule.Parameters.Where(i => !i.IsGeneral && !i.Equals(p1)).ToList();
var groupix = 0;
foreach (var factgroup in p1.TaxFacts)
{
var groupdictionary = new Dictionary <int[], Dictionary <string, List <int> > >(new IntArrayEqualityComparer());
foreach (var factid in factgroup.AsEnumerable())
{
var groupkey = AddFactToGroup(factid, p1.Name, taxonomy, groupaspects, groupdictionary, factkeydict);
foreach (var p in otherfactparameters)
{
var factids = p.TaxFacts[groupix];
//if (factids.Count > 1)
//{
//}
foreach (var p_factid in factids)
{
//AddFactToGroup(p_factid, p.Name, groupkey, groupdictionary, factkeydict);
AddFactToGroup(p_factid, p.Name, taxonomy, groupaspects, groupdictionary, factkeydict);
}
if (factids.Count == 0)
{
AddEmptyFactToGroup(p.Name, groupkey, groupdictionary, factkeydict);
}
factids.Clear();
}
}
foreach (var group in groupdictionary)
{
foreach (var item in group.Value)
{
var il = new List <int>();
if (item.Key == "a" && paramtaxfactdict[item.Key].Count > 0)
{
}
paramtaxfactdict[item.Key].Add(il);
foreach (var fact in item.Value)
{
il.Add(fact);
}
}
}
//}
groupix++;
}
foreach (var fp in factparameters)
{
var taxfacts = paramtaxfactdict[fp.Name];
fp.TaxFacts = taxfacts;
}
}
}
public static void ExecuteExplicitFiltering(Taxonomy taxonomy, ValidationRule rule)
{
if (rule.ID.Contains("es_v308"))
{
}
var tables = rule.Tables.Select(i => taxonomy.Tables.FirstOrDefault(t => t.ID == i)).ToList();
IntervalList tableintevallist = null;
foreach (var table in tables)
{
tableintevallist = tableintevallist == null ? new IntervalList() : tableintevallist;
tableintevallist = Utilities.Objects.MergeSorted(tableintevallist, table.FactindexList, null);
//tableintevallist.Clear();
}
var hastableinfo = tableintevallist != null ? tableintevallist.Count > 0 : false;
IList <int> allfactsintevallist = new IntervalList(0, taxonomy.FactsManager.FactsOfPages.Count);
if (hastableinfo)
{
allfactsintevallist = tableintevallist;
}
var ruletypeddimension = rule.BaseQuery.DictFilterIndexes.Where(i => taxonomy.IsTyped(i));
foreach (var parameter in rule.Parameters)
{
parameter.TaxFacts.Clear();
IList <int> sdata = tableintevallist;
if (!parameter.IsGeneral)
{
if (parameter.BaseQuery.DictFilterIndexes.Count == 0)
{
sdata = allfactsintevallist;
}
parameter.Data = parameter.BaseQuery.ToIntervalList(taxonomy.FactsOfParts, sdata);
parameter.TypedDimensions = parameter.BaseQuery.DictFilterIndexes.Where(i => taxonomy.IsTyped(i)).ToList();
parameter.TypedDimensions = parameter.TypedDimensions.Concat(ruletypeddimension).Distinct().ToList();
parameter.CoveredParts = parameter.BaseQuery.GetAspects(taxonomy);
if (parameter.FallBackValue == "()"
/*&& parameter.TypedDimensions.Count>0 */
&& !parameter.BindAsSequence)
{
parameter.BindAsSequence = true;
}
}
}
bool hasfacts = false;
var ix = 0;
IList <int> data = tableintevallist;
if (rule.BaseQuery.DictFilterIndexes.Count == 0)
{
data = allfactsintevallist;
}
rule.TypedDimensions = rule.BaseQuery.DictFilterIndexes.Where(i => taxonomy.IsTyped(i)).ToList();
rule.CoveredParts = rule.BaseQuery.GetAspects(taxonomy);
var singlefactparameters = rule.Parameters.Where(i => !i.IsGeneral && !i.BindAsSequence).ToList();
var multifactparameters = rule.Parameters.Where(i => !i.IsGeneral && i.BindAsSequence).ToList();
//Utilities.Logger.WriteToFile(String.Format("EnumerateIntervals {0} on {1}", rule.BaseQuery, data));
foreach (var group in rule.BaseQuery.EnumerateIntervals(taxonomy.FactsOfParts, 0, data, false))
{
//Utilities.Logger.WriteToFile("Joining rule with parameters...");
foreach (var parameter in rule.Parameters)
{
if (!parameter.IsGeneral)
{
IntervalList facts = (IntervalList)Utilities.Objects.IntersectSorted(parameter.Data, group, null);
//var facts = factsq.ToList();
if (facts.Count > 0)
{
hasfacts = true;
}
else
{
}
parameter.TaxFacts.Add(facts.ToList());
}
}
ix++;
//Utilities.Logger.WriteToFile("End joining rule with parameters");
}
ValidationRuleHelper.SetParamerterTypes(taxonomy, rule);
if (!hasfacts)
{
Utilities.Logger.WriteLine(String.Format("{0}: Rule has no facts!", rule.ID));
}
}
public static List <int> GetUncoveredDomains(Taxonomy taxonomy, Dictionary <int, int[]> factkeydict, ValidationRule rule)
{
var result = new List <int>();
var ruleparameterdictionary = rule.Parameters.ToDictionary(i => i.Name);
var factparameterids = ruleparameterdictionary.Where(i => !i.Value.IsGeneral).Select(i => i.Key).ToList();
var mainaspectsofparameters = new List <List <int> >();
var factindexes = rule.Parameters.SelectMany(i => i.TaxFacts).SelectMany(i => i).OrderBy(i => i).ToList();
foreach (var factindex in factindexes)
{
var key = taxonomy.FactsManager.GetFactKey(factindex);
if (!factkeydict.ContainsKey(factindex))
{
factkeydict.Add(factindex, null);
}
factkeydict[factindex] = key;
}
foreach (var p in rule.Parameters)
{
var rulep = ruleparameterdictionary[p.Name];
var mainaspects = new HashSet <int>();
foreach (var factgroup in p.TaxFacts)
{
foreach (var factid in factgroup.AsEnumerable())
{
var taxonomykey = factkeydict[factid];
foreach (var part in taxonomykey)
{
var mainaspectid = taxonomy.DimensionDomainsOfMembers.ContainsKey(part) ? taxonomy.DimensionDomainsOfMembers[part] : part;
if (!mainaspects.Contains(mainaspectid))
{
mainaspects.Add(mainaspectid);
}
}
}
}
var aspectsofparameter = mainaspects.Except(rulep.CoveredParts).ToList();
aspectsofparameter = aspectsofparameter.Except(rule.CoveredParts).ToList();
mainaspectsofparameters.Add(aspectsofparameter);
}
List <int> commonmainaspects = null;
foreach (var parameteraspects in mainaspectsofparameters)
{
if (commonmainaspects == null)
{
commonmainaspects = parameteraspects.ToList();
}
else
{
commonmainaspects = commonmainaspects.Intersect(parameteraspects).ToList();
}
}
result = commonmainaspects;
return(result);
}
public void SetFacts(LogicalModel.Validation.ValidationRule rule)
{
if (rule.ID.Contains("0147"))
{
}
var tables = rule.Tables.Select(i => Taxonomy.Tables.FirstOrDefault(t => t.ID == i)).ToList();
IList <int> tableintevallist = null;
foreach (var table in tables)
{
tableintevallist = tableintevallist == null ? new IntervalList() : tableintevallist;
tableintevallist = Utilities.Objects.MergeSorted(tableintevallist, table.FactindexList, null);
}
var hastableinfo = tableintevallist != null ? tableintevallist.Count > 0 : false;
IList <int> allfactsintevallist = new IntervalList(0, Taxonomy.FactsManager.FactsOfPages.Count);
if (hastableinfo)
{
allfactsintevallist = tableintevallist;
}
var ruletypeddimension = rule.BaseQuery.DictFilterIndexes.Where(i => Taxonomy.IsTyped(i));
foreach (var parameter in rule.Parameters)
{
parameter.TaxFacts.Clear();
IList <int> sdata = tableintevallist;
if (!parameter.IsGeneral)
{
if (parameter.BaseQuery.DictFilterIndexes.Count == 0)
{
sdata = allfactsintevallist;
}
parameter.Data = parameter.BaseQuery.ToIntervalList(this.Taxonomy.FactsOfParts, sdata);
parameter.TypedDimensions = parameter.BaseQuery.DictFilterIndexes.Where(i => Taxonomy.IsTyped(i)).ToList();
parameter.TypedDimensions = parameter.TypedDimensions.Concat(ruletypeddimension).Distinct().ToList();
parameter.CoveredParts = parameter.BaseQuery.GetAspects(this.Taxonomy);
if (parameter.FallBackValue == "()"
/*&& parameter.TypedDimensions.Count>0 */
&& !parameter.BindAsSequence)
{
parameter.BindAsSequence = true;
}
}
}
bool hasfacts = false;
var ix = 0;
IList <int> data = tableintevallist;
if (rule.BaseQuery.DictFilterIndexes.Count == 0)
{
// && rule.BaseQuery.Pools.Count == 0
//var interval = allfactsinteval;
//var intervallist = new IntervalList();
//intervallist.AddInterval(interval);
data = allfactsintevallist;
}
rule.TypedDimensions = rule.BaseQuery.DictFilterIndexes.Where(i => Taxonomy.IsTyped(i)).ToList();
rule.CoveredParts = rule.BaseQuery.GetAspects(this.Taxonomy);
var singlefactparameters = rule.Parameters.Where(i => !i.IsGeneral && !i.BindAsSequence).ToList();
var multifactparameters = rule.Parameters.Where(i => !i.IsGeneral && i.BindAsSequence).ToList();
var mffnspissue = false;
LogicalModel.Validation.ValidationParameter p_mffnspissue = null;
//Utilities.Logger.WriteToFile(String.Format("EnumerateIntervals {0} on {1}", rule.BaseQuery, data));
foreach (var group in rule.BaseQuery.EnumerateIntervals(this.Taxonomy.FactsOfParts, 0, data, false))
{
//Utilities.Logger.WriteToFile("Joining rule with parameters...");
foreach (var parameter in rule.Parameters)
{
if (!parameter.IsGeneral)
{
var factsq = Utilities.Objects.IntersectSorted(parameter.Data, group, null);
//var facts = factsq.ToList();
var facts = factsq;
if (!parameter.BindAsSequence && facts.Count > 1)
{
mffnspissue = true;
p_mffnspissue = parameter;
}
if (facts.Count > 0)
{
hasfacts = true;
}
parameter.AddTaxFacts(facts);
}
}
if (mffnspissue)
{
var factcounts = singlefactparameters.Select(i => new Utilities.KeyValue <int, LogicalModel.Validation.ValidationParameter>(i.TaxFacts.LastOrDefault().Count, i)).OrderByDescending(i => i.Key).ToList();
var distinctfactcounts = factcounts.Distinct().ToList();
if (singlefactparameters.Count == 1)
{
mffnspissue = false;
var p = singlefactparameters.FirstOrDefault();
var lasttaxfact = p.TaxFacts.LastOrDefault();
p.TaxFacts.Remove(lasttaxfact);
foreach (var fact in lasttaxfact.AsEnumerable())
{
p.AddTaxFacts(new List <int>()
{
fact
});
}
}
if ((distinctfactcounts.Count == 2 && distinctfactcounts[1].Key == 1))
{
mffnspissue = false;
var theparameter = factcounts[0].Value;
var parameterstocomplete = factcounts.Where(i => i.Key < factcounts[0].Key).Select(i => i.Value);
var facts = theparameter.TaxFacts.LastOrDefault();
theparameter.TaxFacts.Clear();
var run = 0;
foreach (var fact in facts.AsEnumerable())
{
theparameter.AddTaxFacts(new List <int>()
{
fact
});
foreach (var p in parameterstocomplete)
{
var firsttaxfact = p.TaxFacts.FirstOrDefault();
if (run > 0)
{
p.TaxFacts.Add(firsttaxfact);
}
}
run++;
}
}
if (distinctfactcounts.Count == 1 /*&& multifactparameters.Count==0*/)
{
mffnspissue = false;
foreach (var p in singlefactparameters)
{
var lasttaxfact = p.TaxFacts.LastOrDefault();
p.TaxFacts.Remove(lasttaxfact);
foreach (var fact in lasttaxfact.AsEnumerable())
{
p.AddTaxFacts(new List <int>()
{
fact
});
}
}
}
}
if (mffnspissue)
{
Utilities.Logger.WriteLine(String.Format("{0}: Multiple facts found for Non Sequenced parameter {1}", rule.ID, p_mffnspissue.Name));
}
ix++;
//Utilities.Logger.WriteToFile("End joining rule with parameters");
}
ValidationRuleHelper.SetParamerterTypes(Taxonomy, rule);
if (!hasfacts)
{
Utilities.Logger.WriteLine(String.Format("{0}: Rule has no facts!", rule.ID));
}
var parameterswithissues = singlefactparameters.Where(i => i.TaxFacts.Any(f => f.Count > 1)).ToList();
if (parameterswithissues.Count > 0)
{
Utilities.Logger.WriteLine(String.Format("{0}: Rule single factrule has multiple facts!", rule.ID));
}
}