/// <summary>
/// <para>Performs the "building of reports" based on the <paramref name="instancePath"/> and <paramref name="taxonomy"/> provided.</para>
/// <para>As a result, <paramref name="filingSummary"/>.<see>MyReports</see> (<see cref="FilingSummary"/>) will be populated with instances of <see cref="ReportHeader"/>,</para>
/// <para>and <paramref name="filingSummary"/>.<see>MyReports</see> will be XML serialized to <paramref name="filingSummaryPath"/></para>
/// <para>If an error occurs, <paramref name="error" /> will be populated.</para>
/// </summary>
/// <param name="instancePath">The path to the instance document.</param>
/// <param name="taxonomy">The loaded and parsed taxonomy object.</param>
/// <param name="filingSummaryPath">The path where the generated <see cref="FilingSummary"/> object should be saved.</param>
/// <param name="reportDirectory">The path where the generated content is saved.</param>
/// <param name="filingSummary">The <see cref="FilingSummary"/> object to populate.</param>
/// <param name="error">The error message for any critical errors which might occur.</param>
/// <returns>True on success or false for fail.</returns>
public bool BuildReports( string instancePath, Taxonomy taxonomy,
string filingSummaryPath, string reportDirectory,
out FilingSummary filingSummary, out string error )
{
error = string.Empty;
filingSummary = null;
DateTime dtStart = DateTime.Now;
try
{
this.currentFilingSummary = new FilingSummary();
this.currentInstancePath = instancePath;
this.currentTaxonomy = taxonomy;
this.currentReportDirectory = reportDirectory;
if( !this.ValidateSettings( out error ) )
return false;
//create the reports directory so that defnref can be generated
if( !Directory.Exists( this.currentReportDirectory ) )
Directory.CreateDirectory( this.currentReportDirectory );
if( !string.IsNullOrEmpty( this.CurrencyMappingFile ) )
this.LoadCurrencies();
//set up this symbol for reuse throughout
InstanceUtils.USDCurrencySymbol = InstanceUtils.GetCurrencySymbolFromCode( InstanceUtils.USDCurrencyCode );
if( string.IsNullOrEmpty( this.preferredLanguage ) )
this.preferredLanguage = this.GetPreferredLanguage();
bool isIMBased = this.CheckIsIMBased();
//DateTime startInstance = DateTime.Now;
ArrayList errors = null;
if( !InstanceUtils.TryLoadInstanceDocument( this.currentInstancePath, out this.currentInstance, out errors ) )
{
string[] arrErrors = new string[ errors.Count ];
for( int i = 0; i < errors.Count; i++ )
{
arrErrors[ i ] = ( (ParserMessage)errors[ i ] ).Message;
}
string instanceDocErrors = string.Join( "\r\n ", arrErrors );
Regex splitPoint = new Regex( @"\S\s+at" );
arrErrors = splitPoint.Split( instanceDocErrors );
instanceDocErrors = arrErrors[ 0 ];
error = "Unable to load the instance document:\r\n " + instanceDocErrors;
return false;
}
ArrayList taxonomies = new ArrayList();
taxonomies.Add( this.currentTaxonomy );
this.currentInstance.FixPrefixInInstanceDocument( taxonomies );
//this.currentFilingSummary.InstanceLoadTime = DateTime.Now - startInstance;
//this.currentFilingSummary.FactCount = this.currentInstance.markups.Count;
this.PopulateMarkupDictionaries();
this.BuildFilingSummary();
this.FireBuildReportsProcessing();
this.currentFilingSummary.MyReports.Clear();
ArrayList topNodes = currentTaxonomy.GetNodesByPresentation( false, this.ExcludedReports );
foreach( Node topNode in topNodes )
{
InstanceReport report = null;
try
{
if( this.BuildReport( topNode, out report ) )
{
if( report.IsEmbedReport || report.HasEmbeddedReports )
{
this.roleAxes[ report.RoleURI ] = report.AxisByPresentation;
this.roleAxisDefaults[ report.RoleURI ] = report.AxisMemberDefaults;
this.roleAxisMembers[ report.RoleURI ] = report.AxisMembersByPresentation;
}
ReportHeader header = this.currentFilingSummary.AddReport( report );
this.ApplyRulesToReport( report );
report.OnRuleProcessing -= this.OnRuleProcessing;
report.OnRuleProcessing += this.OnRuleProcessing;
report.OnRuleProcessed -= this.OnRuleProcessed;
report.OnRuleProcessed += this.OnRuleProcessed;
report.UnitDictionary = this.unitDictionary;
string fullPath = Path.Combine( this.currentReportDirectory, header.XmlFileName );
report.BuildXMLDocument( fullPath, true, isIMBased, this.currentFilingSummary );
report.UnitDictionary = null;
report.OnRuleProcessing -= this.OnRuleProcessing;
report.OnRuleProcessed -= this.OnRuleProcessed;
if( header.HasEmbeddedReports )
this.reportsWithEmbeds.Add( header );
}
}
finally
{
if( report != null )
report.Dispose();
}
}
//Build Missing Elements (Uncategorized) Report before we flush the markups
InstanceReport missingReport;
if( this.BuildMissingDataReport( out missingReport ) )
{
ReportHeader uncatHeader = this.currentFilingSummary.AddReport( missingReport );
uncatHeader.XmlFileName = InstanceUtils.TOP_LEVEL_REPORT_INDICATOR+ _missingReportIndex +".xml";
}
//Free up some resources
this.currentInstance = null;
if( this.internalReports.Count == 0 )
{
this.currentTaxonomy.Close();
this.currentTaxonomy = null;
}
//clear the dictionaries after checking internalReports
this.ClearMarkupDictionaries();
#region Clean up Columns
//if the company has filed earning release, do not touch the reports.
//Based on request from SEC, do not remove any reports if the base taxonomy is the new GAAP taxonomy (2208)
if( !this.HasEarningRelease() )
{
//DO NOT REMOVE - These are used in rule processing
isGAAP2005 = this.TaxonomyIsGAAP2005();
isNewGAAP = ( !isGAAP2005 );
ProcessFlowThroughColumnsReports();
}
#endregion
#region Build All Reports
//Build book
ReportHeader r1 = new ReportHeader();
r1.LongName = _allReports;
r1.ShortName = _allReports;
r1.ReportType = ReportHeaderType.Book;
r1.IsDefault = this.currentFilingSummary.MyReports.Count == 0;
this.currentFilingSummary.AddReports( r1 );
#endregion
#region Process Embeeded Reports
bool isRuleEnabled = this.FireRuleProcessing( RulesEngineUtils.EMBED_REPORTS_RULE );
if( isRuleEnabled && this.reportsWithEmbeds.Count > 0 )
{
List<ReportHeader> embedReports = this.ProcessEmbeddedReports();
if( embedReports.Count > 0 )
{
foreach( ReportHeader embedReport in embedReports )
{
if( this.currentFilingSummary.MyReports.Contains( embedReport ) )
{
this.currentFilingSummary.MyReports.Remove( embedReport );
string reportNameToDelete = Path.Combine( this.currentReportDirectory, embedReport.XmlFileName );
if( File.Exists( reportNameToDelete ) )
File.Delete( reportNameToDelete );
}
}
}
this.FireRuleProcessed( RulesEngineUtils.EMBED_REPORTS_RULE );
}
#endregion
#region Generate Excel Workbook
isRuleEnabled = this.FireRuleProcessing( RulesEngineUtils.GENERATE_EXCEL_RULE );
if( isRuleEnabled )
{
string excelReportName = "Financial_Report";
bool processed = ExcelUtility.GenerateExcelWorkbook( this.currentFilingSummary, this.currentReportDirectory, excelReportName );
this.FireRuleProcessed( RulesEngineUtils.GENERATE_EXCEL_RULE );
if( !processed )
{
error = "Failed to generate Excel Workbook for report: " + this.currentReportDirectory + " " + excelReportName;
return false;
}
}
#endregion
#region Generate Output Formats
if( ( this.ReportFormat & ReportFormat.Html ) == ReportFormat.Html )
{
if( !this.GenerateHtmlFiles() )
{
//The error was logged to the tracer
}
}
if( ( this.ReportFormat & ReportFormat.Xml ) == ReportFormat.Xml )
{
//this format already exists
}
else
{
// this.currentFilingSummary.TraceInformation( "Information: Report Formart does not include 'Xml'. Removing references and files." );
foreach( ReportHeader rh in this.currentFilingSummary.MyReports )
{
if( !string.IsNullOrEmpty( rh.HtmlFileName ) )
{
string deleteFile = Path.Combine( this.currentReportDirectory, rh.XmlFileName );
if( File.Exists( deleteFile ) )
File.Delete( deleteFile );
rh.XmlFileName = null;
}
}
}
#endregion
#region Check and fix default report
//Recheck the default report just in case it was deleted
bool foundDefault = false;
foreach( ReportHeader header in this.currentFilingSummary.MyReports )
{
if( header.IsDefault )
{
foundDefault = true;
break;
}
}
if( !foundDefault )
{
foreach( ReportHeader header in this.currentFilingSummary.MyReports )
{
if( header.IsBalanceSheet() || header.ReportType == ReportHeaderType.Book )
{
header.IsDefault = true;
break;
}
}
}
#endregion
return true;
}
catch( Exception ex )
{
error = string.Format( "Exception thrown in BuildReports: {0}", ex.Message );
return false;
}
finally
{
this.FireBuildReportsProcessed();
this.currentFilingSummary.ProcessingTime = DateTime.Now - dtStart;
this.currentFilingSummary.SaveAsXml( filingSummaryPath );
filingSummary = this.currentFilingSummary;
this.currentFilingSummary = null;
if( this.currentTaxonomy != null )
this.currentTaxonomy.Close();
this.currentTaxonomy = null;
}
}
public void Test_us_gaap_ci_stm_dis_all_2007_12_31()
{
string fileName = TestCommon.FolderRoot + @"USFRTF-2007-11-21-prerelease\ind\ci" + System.IO.Path.DirectorySeparatorChar + "us-gaap-ci-stm-dis-all-2007-12-31.xsd";
Taxonomy s = new Taxonomy();
int errors = 0;
DateTime startLoad = DateTime.Now;
DateTime startLoadParse = DateTime.Now;
if (s.Load(fileName, out errors) != true)
{
Assert.Fail((string)s.ErrorList[0]);
}
DateTime endLoad = DateTime.Now;
DateTime startParse = DateTime.Now;
s.Parse(out errors);
DateTime endParse = DateTime.Now;
DateTime endLoadParse = DateTime.Now;
s.CurrentLanguage = "en";
s.CurrentLabelRole = "preferredLabel";
long FileSize = new long();
DateTime startSerialize = DateTime.Now;
Taxonomy.Serialize(s, @"C:\Aucent\TaxonomySerialized.bin", out FileSize);
DateTime endSerialize = DateTime.Now;
//Taxonomy sNew = new Taxonomy();
//DateTime startDeserialize = DateTime.Now;
//Taxonomy.Deserialize(@"C:\Aucent\TaxonomySerialized.bin", out sNew);
//DateTime endDeserialize = DateTime.Now;
DateTime startGetNodes = DateTime.Now;
ArrayList nodes = s.GetNodesByPresentation(true);
DateTime endGetNodes = DateTime.Now;
//ArrayList nodesNew = sNew.GetNodesByPresentation(true);
//int count = 0;
//int dimensionNodeCount = 0;
//foreach (Node n in nodes)
//{
// StringBuilder sb = DisplayNode(n, 0, ref count, ref dimensionNodeCount);
// Console.WriteLine(sb.ToString());
//}
//int countNew = 0;
//int dimensionNodeCountNew = 0;
//foreach (Node n in nodesNew)
//{
// StringBuilder sb = DisplayNode(n, 0, ref countNew, ref dimensionNodeCountNew);
// Console.WriteLine(sb.ToString());
//}
Console.WriteLine("Time to Load: {0}", endLoad - startLoad);
Console.WriteLine("Time to Parse: {0}", endParse - startParse);
Console.WriteLine("Time to Load/Parse: {0}", endLoadParse - startLoadParse);
Console.WriteLine("Time to Serialize: {0}", endSerialize - startSerialize);
//Console.WriteLine("Time to Deserialize: {0}", endDeserialize - startDeserialize);
Console.WriteLine("Time to GetNodes: {0}", endGetNodes - startGetNodes);
Console.WriteLine("Serialized Filesize: {0}", FileSize);
//Console.WriteLine("Count of elements in original presentation = {0}", count);
//Console.WriteLine("Count of elements in deserialized presentation = {0}", countNew);
}
public void TestLoadTaxonomyFromInstanceDocument_SEC_4Tax_Local_HREF()
{
string fileName = @"S:\TestSchemas\BaseHRef\Instance\aabxx-20060228.xml";
string baseHRef = @"S:\TestSchemas\BaseHRef\Instance\";
Instance ins = new Instance();
ArrayList errors;
ins.TryLoadInstanceDoc( fileName, out errors );
Taxonomy tax = new Taxonomy();
Assert.IsTrue(
tax.LoadTaxonomyFromInstanceDocument( Path.GetTempPath(), ins, false, baseHRef ),
"Failed to load taxonomy" );
ArrayList nodes = tax.GetNodesByPresentation();
Assert.IsTrue( nodes.Count >= 16, "should have presentation nodes" );
}
public void Dow_ParseUsingCloneOfBaseTaxonomy()
{
DateTime start = DateTime.Now;
DOW_LoadAndParse();
DateTime end = DateTime.Now;
Console.WriteLine("TIME TAKEN TO LOAD WITHOUT CACHE = {0}", end - start);
string US_GAAP_FILE = TestCommon.FolderRoot + @"XBRL 2.1 Updated" +System.IO.Path.DirectorySeparatorChar +"2005-02-28" +System.IO.Path.DirectorySeparatorChar +"us-gaap-ci-2005-02-28.xsd";
Taxonomy bt = new Taxonomy();
bt.Load(US_GAAP_FILE);
int errors = 0;
bt.Parse(out errors);
bt.CurrentLabelRole = "Label";
bt.CurrentLanguage = "en";
ArrayList nodeList = bt.GetNodesByPresentation();
MemoryStream msWrite = new MemoryStream();
BinaryFormatter formatterWrite = new BinaryFormatter();
formatterWrite.AssemblyFormat = System.Runtime.Serialization.Formatters.FormatterAssemblyStyle.Simple;
formatterWrite.Serialize(msWrite, bt);
byte[] bufOrig = msWrite.GetBuffer();
ArrayList flatListOrig = new ArrayList();
TestTaxonomy_US_GAAP_IM.GetFlatListOfPresentationNodes(nodeList, ref flatListOrig);
Hashtable taxIdInfos = new Hashtable();
for (int i = 0; i < flatListOrig.Count; i++)
{
Node orig = flatListOrig[i] as Node;
if( orig.MyElement != null )
{
taxIdInfos[orig.Id] = orig.MyElement.TaxonomyInfoId;
}
}
taxonomyCache[Path.GetFileName(US_GAAP_FILE)] = bt;
Taxonomy.TaxonomyCacheManager = this;
start = DateTime.Now;
DOW_LoadAndParse();
end = DateTime.Now;
Console.WriteLine("TIME TAKEN TO LOAD WITH CACHE = {0}", end - start);
ArrayList nodeListAfter = bt.GetNodesByPresentation();
ArrayList flatListAfter = new ArrayList();
TestTaxonomy_US_GAAP_IM.GetFlatListOfPresentationNodes(nodeListAfter, ref flatListAfter);
Assert.AreEqual(flatListOrig.Count, flatListAfter.Count, "should not change");
for (int i = 0; i < flatListAfter.Count; i++)
{
Node orig = flatListOrig[i] as Node;
Node after = flatListAfter[i] as Node;
Assert.AreEqual(orig.Label, after.Label, "Label should match");
Assert.AreEqual(orig.Id, after.Id);
if (after.MyElement != null)
{
Assert.AreEqual(taxIdInfos[after.Id], after.MyElement.TaxonomyInfoId);
}
}
Taxonomy.TaxonomyCacheManager = null;
msWrite = new MemoryStream();
formatterWrite = new BinaryFormatter();
formatterWrite.AssemblyFormat = System.Runtime.Serialization.Formatters.FormatterAssemblyStyle.Simple;
formatterWrite.Serialize(msWrite, bt);
byte[] bufAfter = msWrite.GetBuffer();
Assert.AreEqual(bufOrig.Length, bufAfter.Length, "Cache object has changed");
MemoryStream msRead = new MemoryStream(bufOrig);
BinaryFormatter formatterRead = new BinaryFormatter();
formatterRead.AssemblyFormat = System.Runtime.Serialization.Formatters.FormatterAssemblyStyle.Simple;
Taxonomy origbt = formatterRead.Deserialize(msRead) as Taxonomy;
Assert.AreEqual(origbt.AllElements.Count, bt.AllElements.Count);
//for( int i = 0 ;i < bufAfter.Length ; i++ )
//{
// if (bufAfter[i] != bufOrig[i])
// {
// Console.WriteLine("Position that is not correct = {0} out of total = {1}" ,i, bufAfter.Length);
// }
// Assert.AreEqual(bufAfter[i], bufOrig[i], "binary data is different");
//}
}
public void TestLoadTaxonomyFromInstanceDocument_SEC_4Tax()
{
string fileName = @"http://www.sec.gov/Archives/edgar/data/908823/000119312506095211/aabxx-20060228.xml";
string baseHRef = @"http://www.sec.gov/Archives/edgar/data/908823/000119312506095211/";
Instance ins = new Instance();
ArrayList errors;
ins.TryLoadInstanceDoc( fileName, out errors );
Taxonomy tax = new Taxonomy();
Assert.IsTrue(
tax.LoadTaxonomyFromInstanceDocument( Path.GetTempPath(), ins, false, baseHRef ),
"Failed to load taxonomy" );
ArrayList nodes = tax.GetNodesByPresentation();
Assert.IsTrue( nodes.Count >= 16, "should have presentation nodes" );
}
public void TestLoadTaxonomyFromInstanceDocument_SEC_2Tax()
{
string fileName = @"http://www.sec.gov/Archives/edgar/data/89800/000095015208008578/shw-20080930.xml";
string baseHRef = @"http://www.sec.gov/Archives/edgar/data/89800/000095015208008578";
Instance ins = new Instance();
ArrayList errors;
ins.TryLoadInstanceDoc( fileName, out errors );
Taxonomy tax = new Taxonomy();
Assert.IsTrue(
tax.LoadTaxonomyFromInstanceDocument( Path.GetTempPath(), ins, false, baseHRef ),
"Failed to load taxonomy" );
ArrayList nodes = tax.GetNodesByPresentation();
Assert.IsTrue( nodes.Count >= 5, "should have presentation nodes" );
}
public void TestLoadTaxonomyFromInstanceDocument()
{
string fileName = @"S:\TestSchemas\Instance With TaxonomyInfo.xml";
Instance ins = new Instance();
ArrayList errors;
ins.TryLoadInstanceDoc( fileName, out errors );
Taxonomy tax = new Taxonomy();
Assert.IsTrue(
tax.LoadTaxonomyFromInstanceDocument( @"S:\TestSchemas", ins, false ),
"Failed to load taxonomy" );
ArrayList nodes = tax.GetNodesByPresentation();
Assert.IsTrue( nodes.Count > 10, "should have presentation nodes" );
}
public void TestGetNodeByPath()
{
int numErrors;
Taxonomy tax = new Taxonomy();
tax.Load(@"http://www.xbrl.org/us/fr/gaap/ci/2005-02-28/us-gaap-ci-2005-02-28.xsd");
tax.Parse(out numErrors);
tax.currentLabelRole = "label";
tax.currentLanguage = "en";
ArrayList nodes = tax.GetNodesByPresentation(true);
Node n = nodes[2] as Node; //Income Statement root node
n = n.children[0] as Node; //Income & Expense abstarct node
n = n.children[1] as Node; //Cost of Goods and Services Sold
n = n.children[1] as Node; // Cost of Services Sold
n = n.children[5] as Node; // Cost of Services - Total
string path = n.GetPresentationPath();
Node testNode = tax.GetNodeFromPresentation(path, true);
Assert.IsNotNull(testNode, "Failed to get node from presentation using path");
Assert.AreEqual(n, testNode, "Incorrect node returned by GetNodeFromPresentation");
}
private void MutualFundTaxonomyLoad()
{
string fileName = @"Q:\Taxonomies\Fidelity.xsd";
DateTime start = DateTime.Now;
Taxonomy tax = new Taxonomy();
tax.Load(fileName);
int error;
Assert.IsTrue(tax.Parse(out error));
tax.GetNodesByPresentation(true);
List<DimensionNode> dNodes = new List<DimensionNode>();
Assert.IsTrue(tax.TryGetAllDimensionNodesForDisplay(null, null, true, out dNodes));
DateTime end = DateTime.Now;
Console.WriteLine("Time taken = {0}", end - start);
Element ele = tax.allElements["rr_MaximumAccountFeeOverAssets"] as Element;
object notUsed = null;
string errorStr;
Assert.IsTrue( ele.TryValidateElement("0.62%", ref notUsed , out errorStr));
}
