public static string GetFactsOfRule(ValidationRule rule) { var sb = new StringBuilder(); foreach (var p in rule.Parameters) { sb.AppendLine(p.Name); sb.AppendLine(TaxonomyEngine.CurrentEngine.CurrentTaxonomy.GetFactStringKeys(p.TaxFacts)); sb.AppendLine("________________________________________________"); } return(sb.ToString()); }
public static void CheckConsistency(Taxonomy taxonomy, ValidationRule rule) { var sb = new StringBuilder(); var inconsistent = false; sb.AppendLine(rule.RawInfo); foreach (var p in rule.Parameters) { var tfcs = ""; var tfc = p.TaxFacts.Count; var tfall = p.TaxFacts.SelectMany(i => i).Count(); if (tfall > tfc && !p.BindAsSequence) { } tfcs = String.Format("{0}: {2}/{1}", p.Name, tfc, tfall); sb.AppendLine(p.ToString() + " TFCS: " + tfcs); foreach (var factgroup in p.TaxFacts) { if (!p.BindAsSequence && factgroup.Count > 1) { inconsistent = true; sb.AppendLine(taxonomy.GetFactStringKeys(p.TaxFacts)); break; } } } sb.AppendLine("___________"); if (rule.ID == "eba_v0204_m") { } if (inconsistent) { Logger.WriteLine(String.Format("Can't resolve rule {0}", rule.ID)); var validationfolder = TaxonomyEngine.LogFolder + "Validation\\"; var logfilename = Utilities.Strings.GetFileName(Logger.LogPath); var outputpath = validationfolder + "Issues_" + logfilename; Utilities.FS.AppendAllText(outputpath, sb.ToString()); } }
public static void SetParamerterTypes(Taxonomy taxonomy, ValidationRule rule) { foreach (var p in rule.Parameters) { if (String.IsNullOrEmpty(p.Concept)) { var firstfactgroup = p.TaxFacts.FirstOrDefault(i => i.Count > 0); int firstfactindex = firstfactgroup != null ? firstfactgroup[0] : -2; if (firstfactindex != -2) { var factkey = taxonomy.FactsManager.GetFactKey(firstfactindex); if (factkey.Length > 0 && taxonomy.CounterFactParts.ContainsKey(factkey[0])) { p.Concept = taxonomy.CounterFactParts[factkey[0]]; } } } if (p.Type == TypeEnum.Unknown) { p.Type = GetType(taxonomy, p.Concept); } } }
public LogicalModel.Validation.ValidationRule GetLogicalRule(Hierarchy <XbrlIdentifiable> hrule, XbrlTaxonomyDocument document) { this.Document = document; var tmp_rule = hrule.Copy(); //FixRule(tmp_rule); var logicalrule = new LogicalModel.Validation.ValidationRule(); var valueassertion = tmp_rule.Item as ValueAssertion; logicalrule.ID = valueassertion.ID; //Utilities.Logger.WriteLine("Getting rule for " + logicalrule.ID); logicalrule.LabelID = valueassertion.LabelID; logicalrule.OriginalExpression = valueassertion.Test.Replace("\r\n", " ").Replace("\r", " ").Replace("\n", " "); logicalrule.SetTaxonomy(this.Taxonomy); var sb = new StringBuilder(); sb.AppendLine(logicalrule.DisplayText); sb.AppendLine(valueassertion.Test); var rawval = document.FileName + "\r\n" + document.LocalPath + "\r\n" + logicalrule.DisplayText + "\r\n" + logicalrule.OriginalExpression + "\r\n" + hrule.ToHierarchyString(i => i.ToString()) + "\r\n"; logicalrule.RawInfo = rawval; Utilities.FS.AppendAllText(Taxonomy.TaxonomyValidationFolder + "Validations_XML.txt", rawval); if (logicalrule.ID.Contains("1058"))/*,"es_v354"*/ { } var factvariables = tmp_rule.Where(i => i.Item is FactVariable); foreach (var fv in factvariables) { tmp_rule.Remove(fv); } logicalrule.BaseQuery = GetQuery(tmp_rule); logicalrule.BaseQuery.GetString(Taxonomy); foreach (var fv in factvariables) { var factvariable = fv.Item as FactVariable; var name = factvariable.Name; var parameter = new LogicalModel.Validation.ValidationParameter(name, logicalrule.ID); logicalrule.Parameters.Add(parameter); parameter.BindAsSequence = factvariable.BindAsSequence; parameter.FallBackValue = factvariable.FallbackValue; parameter.BaseQuery = GetQuery(fv); parameter.Concept = parameter.BaseQuery.GetConcept(); if (String.IsNullOrEmpty(parameter.Concept)) { parameter.Concept = logicalrule.BaseQuery.GetConcept(); } if (parameter.BaseQuery.HasDictFilter("find:filingIndicator")) { parameter.IsGeneral = true; parameter.StringValue = "filingindicators"; } ValidationRuleHelper.SetParamerterTypes(Taxonomy, logicalrule); } var factparameterqueries = logicalrule.Parameters.Where(i => !i.IsGeneral).Select(i => i.BaseQuery).ToArray(); var commconparameterquery = FactBaseQuery.GetCommonQuery(factparameterqueries); if (commconparameterquery != null && commconparameterquery.HasFilters()) { FactBaseQuery.MergeQueries(logicalrule.BaseQuery, commconparameterquery); foreach (var pquery in factparameterqueries) { FactBaseQuery.RemoveQuery(pquery, commconparameterquery); } } var sbquery = new StringBuilder(); if (1 == 1) { sbquery.AppendLine("RuleQuery: "); sbquery.AppendLine(logicalrule.BaseQuery != null? logicalrule.BaseQuery.ToString():""); sbquery.AppendLine(); foreach (var p in logicalrule.Parameters) { sbquery.AppendLine(p + " Query: "); if (p.BaseQuery != null) { sbquery.AppendLine(p.BaseQuery != null ? p.BaseQuery.ToString() : ""); } } } foreach (var gp in Taxonomy.ValidationParameters) { if (valueassertion.Test.Contains(gp.Value.XPathName)) { if (gp.Key == "ReportingLevel") { var p_rl1 = new LogicalModel.Validation.ValidationParameter("ReportingLevel", logicalrule.ID); p_rl1.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("_con") ? "con" : "ind"; p_rl1.Type = LogicalModel.TypeEnum.String; p_rl1.IsGeneral = true; logicalrule.Parameters.Add(p_rl1); continue; } if (gp.Key == "AccountingStandard") { var p_rl2 = new LogicalModel.Validation.ValidationParameter("AccountingStandard", logicalrule.ID); p_rl2.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("GAAP") ? "GAAP" : "IFRS"; p_rl2.Type = LogicalModel.TypeEnum.String; p_rl2.IsGeneral = true; logicalrule.Parameters.Add(p_rl2); continue; } var p_rl = new LogicalModel.Validation.ValidationParameter(gp.Key, logicalrule.ID); p_rl.StringValue = "Unresolved"; p_rl.Type = LogicalModel.TypeEnum.String; p_rl.IsGeneral = true; logicalrule.Parameters.Add(p_rl); } } /* * if (valueassertion.Test.Contains("$ReportingLevel")) * { * var p_rl1 = new LogicalModel.Validation.ValidationParameter("ReportingLevel", logicalrule.ID); * p_rl1.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("_con") ? "con" : "ind"; * p_rl1.Type = LogicalModel.TypeEnum.String; * p_rl1.IsGeneral = true; * logicalrule.Parameters.Add(p_rl1); * } * if (valueassertion.Test.Contains("$AccountingStandard")) * { * var p_rl2 = new LogicalModel.Validation.ValidationParameter("AccountingStandard", logicalrule.ID); * p_rl2.StringValue = this.Taxonomy.EntryDocument.FileName.Contains("GAAP") ? "GAAP" : "IFRS"; * p_rl2.Type = LogicalModel.TypeEnum.String; * p_rl2.IsGeneral = true; * logicalrule.Parameters.Add(p_rl2); * } */ logicalrule.SetTaxonomy(this.Taxonomy); ValidationRuleHelper.ExecuteExplicitFiltering(this.Taxonomy, logicalrule); ValidationRuleHelper.ExecuteImplicitFiltering(this.Taxonomy, logicalrule); ValidationRuleHelper.ExecuteMatching(this.Taxonomy, logicalrule); ValidationRuleHelper.CheckConsistency(this.Taxonomy, logicalrule); //SetFacts(logicalrule); return(logicalrule); }
public static void GroupTaxFacts(Taxonomy taxonomy, Dictionary <int, int[]> factkeydict, ValidationRule rule, List <int> groupaspects) { if (groupaspects.Count == 0) { return; } var p1 = rule.Parameters.FirstOrDefault(i => !i.IsGeneral && !i.BindAsSequence); var factparameters = rule.Parameters.Where(i => !i.IsGeneral).ToList(); var nonsequencedfactparameters = factparameters.Where(i => !i.BindAsSequence).ToList(); p1 = nonsequencedfactparameters.OrderByDescending(i => GetFactCount(i)).FirstOrDefault(); var sb = new StringBuilder(); foreach (var p in factparameters) { var tfc = p.TaxFacts.Count; var tfall = p.TaxFacts.SelectMany(i => i).Count(); if (tfall > tfc && !p.BindAsSequence) { } sb.AppendLine(String.Format("{0}: {2}/{1}", p.Name, tfc, tfall)); } var groups = new List <Dictionary <string, List <int> > >(); var paramtaxfactdict = new Dictionary <string, List <List <int> > >(); if ((Object)p1 != null) { foreach (var fp in factparameters) { paramtaxfactdict.Add(fp.Name, new List <List <int> >()); } var otherfactparameters = rule.Parameters.Where(i => !i.IsGeneral && !i.Equals(p1)).ToList(); var groupix = 0; foreach (var factgroup in p1.TaxFacts) { var groupdictionary = new Dictionary <int[], Dictionary <string, List <int> > >(new IntArrayEqualityComparer()); foreach (var factid in factgroup.AsEnumerable()) { var groupkey = AddFactToGroup(factid, p1.Name, taxonomy, groupaspects, groupdictionary, factkeydict); foreach (var p in otherfactparameters) { var factids = p.TaxFacts[groupix]; //if (factids.Count > 1) //{ //} foreach (var p_factid in factids) { //AddFactToGroup(p_factid, p.Name, groupkey, groupdictionary, factkeydict); AddFactToGroup(p_factid, p.Name, taxonomy, groupaspects, groupdictionary, factkeydict); } if (factids.Count == 0) { AddEmptyFactToGroup(p.Name, groupkey, groupdictionary, factkeydict); } factids.Clear(); } } foreach (var group in groupdictionary) { foreach (var item in group.Value) { var il = new List <int>(); if (item.Key == "a" && paramtaxfactdict[item.Key].Count > 0) { } paramtaxfactdict[item.Key].Add(il); foreach (var fact in item.Value) { il.Add(fact); } } } //} groupix++; } foreach (var fp in factparameters) { var taxfacts = paramtaxfactdict[fp.Name]; fp.TaxFacts = taxfacts; } } }
public static void ExecuteExplicitFiltering(Taxonomy taxonomy, ValidationRule rule) { if (rule.ID.Contains("es_v308")) { } var tables = rule.Tables.Select(i => taxonomy.Tables.FirstOrDefault(t => t.ID == i)).ToList(); IntervalList tableintevallist = null; foreach (var table in tables) { tableintevallist = tableintevallist == null ? new IntervalList() : tableintevallist; tableintevallist = Utilities.Objects.MergeSorted(tableintevallist, table.FactindexList, null); //tableintevallist.Clear(); } var hastableinfo = tableintevallist != null ? tableintevallist.Count > 0 : false; IList <int> allfactsintevallist = new IntervalList(0, taxonomy.FactsManager.FactsOfPages.Count); if (hastableinfo) { allfactsintevallist = tableintevallist; } var ruletypeddimension = rule.BaseQuery.DictFilterIndexes.Where(i => taxonomy.IsTyped(i)); foreach (var parameter in rule.Parameters) { parameter.TaxFacts.Clear(); IList <int> sdata = tableintevallist; if (!parameter.IsGeneral) { if (parameter.BaseQuery.DictFilterIndexes.Count == 0) { sdata = allfactsintevallist; } parameter.Data = parameter.BaseQuery.ToIntervalList(taxonomy.FactsOfParts, sdata); parameter.TypedDimensions = parameter.BaseQuery.DictFilterIndexes.Where(i => taxonomy.IsTyped(i)).ToList(); parameter.TypedDimensions = parameter.TypedDimensions.Concat(ruletypeddimension).Distinct().ToList(); parameter.CoveredParts = parameter.BaseQuery.GetAspects(taxonomy); if (parameter.FallBackValue == "()" /*&& parameter.TypedDimensions.Count>0 */ && !parameter.BindAsSequence) { parameter.BindAsSequence = true; } } } bool hasfacts = false; var ix = 0; IList <int> data = tableintevallist; if (rule.BaseQuery.DictFilterIndexes.Count == 0) { data = allfactsintevallist; } rule.TypedDimensions = rule.BaseQuery.DictFilterIndexes.Where(i => taxonomy.IsTyped(i)).ToList(); rule.CoveredParts = rule.BaseQuery.GetAspects(taxonomy); var singlefactparameters = rule.Parameters.Where(i => !i.IsGeneral && !i.BindAsSequence).ToList(); var multifactparameters = rule.Parameters.Where(i => !i.IsGeneral && i.BindAsSequence).ToList(); //Utilities.Logger.WriteToFile(String.Format("EnumerateIntervals {0} on {1}", rule.BaseQuery, data)); foreach (var group in rule.BaseQuery.EnumerateIntervals(taxonomy.FactsOfParts, 0, data, false)) { //Utilities.Logger.WriteToFile("Joining rule with parameters..."); foreach (var parameter in rule.Parameters) { if (!parameter.IsGeneral) { IntervalList facts = (IntervalList)Utilities.Objects.IntersectSorted(parameter.Data, group, null); //var facts = factsq.ToList(); if (facts.Count > 0) { hasfacts = true; } else { } parameter.TaxFacts.Add(facts.ToList()); } } ix++; //Utilities.Logger.WriteToFile("End joining rule with parameters"); } ValidationRuleHelper.SetParamerterTypes(taxonomy, rule); if (!hasfacts) { Utilities.Logger.WriteLine(String.Format("{0}: Rule has no facts!", rule.ID)); } }
public static List <int> GetUncoveredDomains(Taxonomy taxonomy, Dictionary <int, int[]> factkeydict, ValidationRule rule) { var result = new List <int>(); var ruleparameterdictionary = rule.Parameters.ToDictionary(i => i.Name); var factparameterids = ruleparameterdictionary.Where(i => !i.Value.IsGeneral).Select(i => i.Key).ToList(); var mainaspectsofparameters = new List <List <int> >(); var factindexes = rule.Parameters.SelectMany(i => i.TaxFacts).SelectMany(i => i).OrderBy(i => i).ToList(); foreach (var factindex in factindexes) { var key = taxonomy.FactsManager.GetFactKey(factindex); if (!factkeydict.ContainsKey(factindex)) { factkeydict.Add(factindex, null); } factkeydict[factindex] = key; } foreach (var p in rule.Parameters) { var rulep = ruleparameterdictionary[p.Name]; var mainaspects = new HashSet <int>(); foreach (var factgroup in p.TaxFacts) { foreach (var factid in factgroup.AsEnumerable()) { var taxonomykey = factkeydict[factid]; foreach (var part in taxonomykey) { var mainaspectid = taxonomy.DimensionDomainsOfMembers.ContainsKey(part) ? taxonomy.DimensionDomainsOfMembers[part] : part; if (!mainaspects.Contains(mainaspectid)) { mainaspects.Add(mainaspectid); } } } } var aspectsofparameter = mainaspects.Except(rulep.CoveredParts).ToList(); aspectsofparameter = aspectsofparameter.Except(rule.CoveredParts).ToList(); mainaspectsofparameters.Add(aspectsofparameter); } List <int> commonmainaspects = null; foreach (var parameteraspects in mainaspectsofparameters) { if (commonmainaspects == null) { commonmainaspects = parameteraspects.ToList(); } else { commonmainaspects = commonmainaspects.Intersect(parameteraspects).ToList(); } } result = commonmainaspects; return(result); }
public void SetFacts(LogicalModel.Validation.ValidationRule rule) { if (rule.ID.Contains("0147")) { } var tables = rule.Tables.Select(i => Taxonomy.Tables.FirstOrDefault(t => t.ID == i)).ToList(); IList <int> tableintevallist = null; foreach (var table in tables) { tableintevallist = tableintevallist == null ? new IntervalList() : tableintevallist; tableintevallist = Utilities.Objects.MergeSorted(tableintevallist, table.FactindexList, null); } var hastableinfo = tableintevallist != null ? tableintevallist.Count > 0 : false; IList <int> allfactsintevallist = new IntervalList(0, Taxonomy.FactsManager.FactsOfPages.Count); if (hastableinfo) { allfactsintevallist = tableintevallist; } var ruletypeddimension = rule.BaseQuery.DictFilterIndexes.Where(i => Taxonomy.IsTyped(i)); foreach (var parameter in rule.Parameters) { parameter.TaxFacts.Clear(); IList <int> sdata = tableintevallist; if (!parameter.IsGeneral) { if (parameter.BaseQuery.DictFilterIndexes.Count == 0) { sdata = allfactsintevallist; } parameter.Data = parameter.BaseQuery.ToIntervalList(this.Taxonomy.FactsOfParts, sdata); parameter.TypedDimensions = parameter.BaseQuery.DictFilterIndexes.Where(i => Taxonomy.IsTyped(i)).ToList(); parameter.TypedDimensions = parameter.TypedDimensions.Concat(ruletypeddimension).Distinct().ToList(); parameter.CoveredParts = parameter.BaseQuery.GetAspects(this.Taxonomy); if (parameter.FallBackValue == "()" /*&& parameter.TypedDimensions.Count>0 */ && !parameter.BindAsSequence) { parameter.BindAsSequence = true; } } } bool hasfacts = false; var ix = 0; IList <int> data = tableintevallist; if (rule.BaseQuery.DictFilterIndexes.Count == 0) { // && rule.BaseQuery.Pools.Count == 0 //var interval = allfactsinteval; //var intervallist = new IntervalList(); //intervallist.AddInterval(interval); data = allfactsintevallist; } rule.TypedDimensions = rule.BaseQuery.DictFilterIndexes.Where(i => Taxonomy.IsTyped(i)).ToList(); rule.CoveredParts = rule.BaseQuery.GetAspects(this.Taxonomy); var singlefactparameters = rule.Parameters.Where(i => !i.IsGeneral && !i.BindAsSequence).ToList(); var multifactparameters = rule.Parameters.Where(i => !i.IsGeneral && i.BindAsSequence).ToList(); var mffnspissue = false; LogicalModel.Validation.ValidationParameter p_mffnspissue = null; //Utilities.Logger.WriteToFile(String.Format("EnumerateIntervals {0} on {1}", rule.BaseQuery, data)); foreach (var group in rule.BaseQuery.EnumerateIntervals(this.Taxonomy.FactsOfParts, 0, data, false)) { //Utilities.Logger.WriteToFile("Joining rule with parameters..."); foreach (var parameter in rule.Parameters) { if (!parameter.IsGeneral) { var factsq = Utilities.Objects.IntersectSorted(parameter.Data, group, null); //var facts = factsq.ToList(); var facts = factsq; if (!parameter.BindAsSequence && facts.Count > 1) { mffnspissue = true; p_mffnspissue = parameter; } if (facts.Count > 0) { hasfacts = true; } parameter.AddTaxFacts(facts); } } if (mffnspissue) { var factcounts = singlefactparameters.Select(i => new Utilities.KeyValue <int, LogicalModel.Validation.ValidationParameter>(i.TaxFacts.LastOrDefault().Count, i)).OrderByDescending(i => i.Key).ToList(); var distinctfactcounts = factcounts.Distinct().ToList(); if (singlefactparameters.Count == 1) { mffnspissue = false; var p = singlefactparameters.FirstOrDefault(); var lasttaxfact = p.TaxFacts.LastOrDefault(); p.TaxFacts.Remove(lasttaxfact); foreach (var fact in lasttaxfact.AsEnumerable()) { p.AddTaxFacts(new List <int>() { fact }); } } if ((distinctfactcounts.Count == 2 && distinctfactcounts[1].Key == 1)) { mffnspissue = false; var theparameter = factcounts[0].Value; var parameterstocomplete = factcounts.Where(i => i.Key < factcounts[0].Key).Select(i => i.Value); var facts = theparameter.TaxFacts.LastOrDefault(); theparameter.TaxFacts.Clear(); var run = 0; foreach (var fact in facts.AsEnumerable()) { theparameter.AddTaxFacts(new List <int>() { fact }); foreach (var p in parameterstocomplete) { var firsttaxfact = p.TaxFacts.FirstOrDefault(); if (run > 0) { p.TaxFacts.Add(firsttaxfact); } } run++; } } if (distinctfactcounts.Count == 1 /*&& multifactparameters.Count==0*/) { mffnspissue = false; foreach (var p in singlefactparameters) { var lasttaxfact = p.TaxFacts.LastOrDefault(); p.TaxFacts.Remove(lasttaxfact); foreach (var fact in lasttaxfact.AsEnumerable()) { p.AddTaxFacts(new List <int>() { fact }); } } } } if (mffnspissue) { Utilities.Logger.WriteLine(String.Format("{0}: Multiple facts found for Non Sequenced parameter {1}", rule.ID, p_mffnspissue.Name)); } ix++; //Utilities.Logger.WriteToFile("End joining rule with parameters"); } ValidationRuleHelper.SetParamerterTypes(Taxonomy, rule); if (!hasfacts) { Utilities.Logger.WriteLine(String.Format("{0}: Rule has no facts!", rule.ID)); } var parameterswithissues = singlefactparameters.Where(i => i.TaxFacts.Any(f => f.Count > 1)).ToList(); if (parameterswithissues.Count > 0) { Utilities.Logger.WriteLine(String.Format("{0}: Rule single factrule has multiple facts!", rule.ID)); } }