/// <summary>
/// Builds a dictionary of all elements for 1 report, specified by <paramref name="rh"/>.
/// </summary>
/// <param name="rh">The <see cref="ReportHeader"/> whose elements should be loaded.</param>
/// <returns></returns>
private Dictionary <string, int> BuildInUseElementsForCurrentReport(ReportHeader rh)
{
if (rh.ReportType == ReportHeaderType.Sheet || rh.ReportType == ReportHeaderType.Notes)
{
if (!this.uniqueReportElements.ContainsKey(rh.XmlFileName) ||
this.uniqueReportElements == null ||
this.uniqueReportElements.Count == 0)
{
InstanceReport report = InstanceReport.LoadXml(Path.Combine(this.currentReportDirectory, rh.XmlFileName));
string[] uniqueElements = report.GetUniqueInUseElements();
this.uniqueReportElements[rh.XmlFileName] = new List <string>(uniqueElements);
}
}
Dictionary <string, int> reportUniqueElements = new Dictionary <string, int>();
if (this.uniqueReportElements.ContainsKey(rh.XmlFileName) && this.uniqueReportElements.Count > 0)
{
foreach (string key in this.uniqueReportElements[rh.XmlFileName])
{
reportUniqueElements[key] = 1;
}
}
return(reportUniqueElements);
}
public void ReplaceWith(ReportHeader arg)
{
this.IsDefault = arg.IsDefault;
this.LongName = arg.LongName;
this.ReportType = arg.ReportType;
this.ShortName = arg.ShortName;
this.XmlFileName = arg.XmlFileName;
}
public void CleanupFlowThroughColumns()
{
for (int rhIndex = 0; rhIndex < currentFilingSummary.MyReports.Count; rhIndex++)
{
ReportHeader rh = currentFilingSummary.MyReports[rhIndex] as ReportHeader;
//Equity reports that do not have segments still need to have flow through columns
//cleaned up because they were not processed with the regular equity processing logic
bool isEquityStatement = false;
if (ReportUtils.IsStatementOfEquityCombined(rh.LongName))
{
isEquityStatement = true;
}
if (isEquityStatement)
{
string instanceFile = Path.Combine(this.currentReportDirectory, rh.XmlFileName);
InstanceReport ir = InstanceReport.LoadXml(instanceFile);
if (ir.IsEquityReport)
{
continue;
}
}
if (rh.ReportType == ReportHeaderType.Sheet || rh.ReportType == ReportHeaderType.Notes)
{
if (cleanupStatementsOnly)
{
if (ReportUtils.IsStatement(rh.LongName) == true)
{
this.currentFilingSummary.TraceInformation("Process Flow-Through: " + rh.LongName);
InstanceReport.ElementSegmentCombinations allInUseElementsSegments = this.BuildInUseElementSegmentCombinationsForAllReports(rh);
this.CleanupColumns(rh, allInUseElementsSegments);
this.uniqueReportElementSegmentCombos.Remove(rh.XmlFileName);
}
}
else
{
this.currentFilingSummary.TraceInformation("Process Flow-Through: " + rh.LongName);
InstanceReport.ElementSegmentCombinations allInUseElementsSegments = this.BuildInUseElementSegmentCombinationsForAllReports(rh);
this.CleanupColumns(rh, allInUseElementsSegments);
this.uniqueReportElementSegmentCombos.Remove(rh.XmlFileName);
}
}
}
this.uniqueReportElements = new Dictionary <string, List <string> >();
this.uniqueReportSegments = new Dictionary <string, List <string> >();
this.uniqueReportElementSegmentCombos = new Dictionary <string, InstanceReport.ElementSegmentCombinations>();
}
public ReportHeader AddReport(InstanceReport report)
{
ReportHeader header = new ReportHeader();
header.HasEmbeddedReports = report.HasEmbeddedReports;
header.LongName = report.ReportLongName;
header.ReportType = report.ReportType;
header.Role = report.RoleURI;
header.ShortName = report.ReportName;
header.TopLevelNode = report.TopLevelNode;
int reportHeaderIndex = this.MyReports.Count;
do
{
reportHeaderIndex++;
header.XmlFileName = string.Format("{0}{1}.xml", InstanceUtils.TOP_LEVEL_REPORT_INDICATOR, reportHeaderIndex);
}while(File.Exists(header.XmlFileName));
this.MyReports.Add(header);
return(header);
}
/// <summary>
/// Builds a nested dictionary of all elements for all reports.
/// </summary>
/// <param name="rhCurrent">The current <see cref="ReportHeader"/> - its report will be skipped from cataloging.</param>
/// <returns></returns>
private Dictionary <string, int> BuildInUseElementsForAllReports(ReportHeader rhCurrent)
{
Dictionary <string, int> allUniqueElements = new Dictionary <string, int>();
foreach (ReportHeader rh in this.currentFilingSummary.MyReports)
{
if (rh.XmlFileName == rhCurrent.XmlFileName)
{
continue;
}
Dictionary <string, int> reportUniqueElements = this.BuildInUseElementsForCurrentReport(rh);
if (reportUniqueElements.Count > 0)
{
foreach (string key in reportUniqueElements.Keys)
{
allUniqueElements[key] = 1;
}
}
}
return(allUniqueElements);
}
public void ReplaceWith( ReportHeader arg )
{
this.IsDefault = arg.IsDefault;
this.LongName = arg.LongName;
this.ReportType = arg.ReportType;
this.ShortName = arg.ShortName;
this.XmlFileName = arg.XmlFileName;
}
public void Test_29039()
{
Test_Abstract.OutputFormat = ( Test_Abstract.OutputFormat & ReportFormat.Xml ) | ReportFormat.Html;
Test_Abstract.HtmlFormat = HtmlReportFormat.Complete;
const string accessionNumber = "29039";
string reportsFolder = PathCombine( this.baseDir, accessionNumber, "Reports" );
CleanAndPrepareFolder( reportsFolder );
string resultsFolder = PathCombine( this.baseDir, accessionNumber, "Results" );
FilingSummary fs = new FilingSummary();
foreach( string from in Directory.GetFiles( resultsFolder, "R1_*.xml" ) )
{
string file = Path.GetFileName( from );
string to = PathCombine( reportsFolder, file );
FileUtilities.Copy( from, to );
ReportHeader header = new ReportHeader( file, file );
fs.MyReports.Add( header );
}
XRB.ReportBuilder rb = new XRB.ReportBuilder();
rb.ReportFormat = ReportFormat.Html;
rb.HtmlReportFormat = HtmlReportFormat.Complete;
rb.GetType().GetField( "currentFilingSummary", BindingFlags.NonPublic | BindingFlags.Instance ).SetValue( rb, fs );
rb.GetType().GetField( "currentReportDirectory", BindingFlags.NonPublic | BindingFlags.Instance ).SetValue( rb, reportsFolder );
rb.GetType().GetMethod( "GenerateHtmlFiles", BindingFlags.NonPublic | BindingFlags.Instance ).Invoke( rb, null );
foreach( string file in Directory.GetFiles( reportsFolder, "R1_*.htm" ) )
{
string html = File.ReadAllText( file );
int start = html.IndexOf( "<body" );
int end = html.IndexOf( "</body" );
end = html.IndexOf( '>', end ) + 1;
html = html.Substring( start, end - start );
html = html.Replace( "<br>", "<br />" );
XmlDocument xDoc = new XmlDocument();
xDoc.LoadXml( html );
XmlNodeList headerRows = xDoc.SelectNodes( "/body/table/tr[ th ]" );
Assert.AreEqual( 2, headerRows.Count, "The transform must generate 2 headers rows for the right display." );
}
}
public void Compare(bool SkipZip, bool baseHTML, out int totalRFiles, out int failedRFiles)
{
//Load list of files:
string[] files;
string error = string.Empty;
FileInfo[] baseRFiles = null;
FileInfo[] newRFiles = null;
bool reportFailed = false;
output op = new output(Output);
totalRFiles = 0;
failedRFiles = 0;
int totalFilings = 0;
int failedFilings = 0;
baseList = new List <InstanceReport>();
newList = new List <InstanceReport>();
//Allows the user to skip this proccess if they're simply rerunning the comparison.
if (!SkipZip)
{
UnzipMove(baseDirs, out files, out error);
UnzipMove(newDirs, out files, out error);
}
baseDirs = new DirectoryInfo(baseDirs.Parent.FullName + "\\Comparison");
newDirs = new DirectoryInfo(newDirs.Parent.FullName + "\\Comparison");
DateTime renderStart = new DateTime();
DateTime renderEnd = new DateTime();
int renderCount = 0;
TimeSpan totalRender = GetRenderTime(newDirs, out renderStart, out renderEnd, out renderCount);
//List of unzipped instance doc directories:
List <DirectoryInfo> allB = new List <DirectoryInfo>(baseDirs.GetDirectories());
List <DirectoryInfo> allN = new List <DirectoryInfo>(newDirs.GetDirectories());
allB.Sort(new CompareFileInfo());
allN.Sort(new CompareFileInfo());
//Sync the resources:
Trace.TraceInformation("Information: Synchronizing resources. . .");
XBRLReportBuilder.Utilities.RulesEngineUtils.SynchronizeResources();
int minCount = Math.Min(allB.Count, allN.Count);
DateTime compareStart = DateTime.Now;
try
{
Trace.TraceInformation("Information: Starting the Comparison loop.");
for (int i = 0; i <= minCount - 1; i++)
{
reportFailed = false;
//Proccess current directory:
DirectoryInfo baseDirectory = allB.Find(ab => ab.Name == allN[i].Name);
DirectoryInfo newDirectory = allN[i];
if (baseDirectory != null)
{
//get the Filing summary for the current filing.
List <ReportHeader> baseHeaders = returnHeaders(baseDirectory);
List <ReportHeader> newHeaders = returnHeaders(newDirectory);
Trace.TraceInformation("Comparing report {0}:\t\t{1} of {2}", baseDirectory.Name, (i + 1).ToString(), minCount);
baseRFiles = gFiles(baseDirectory, "R*.xml");
newRFiles = gFiles(newDirectory, "R*.xml");
reportName = baseDirectory.Name;
baseList.Clear();
newList.Clear();
try
{
//Trace.TraceInformation("Loading Files into memory.");
//Deserialize, load and sort the r files:
foreach (FileInfo file in baseRFiles)
{
LoadFiles(file, ref baseList);
}
foreach (FileInfo file in newRFiles)
{
LoadFiles(file, ref newList);
}
}
catch (Exception ex)
{
Trace.TraceInformation("The following error occured while trying to load the R files for {0}: \n{1}", baseDirectory.Name, ex.Message);
RLogger.Error(string.Format("The following error occured while trying to load the R files for {0}: \n{1}", baseDirectory.Name, ex.Message));
}
//baseList and newList now has the list of the R files:
for (int j = 0; j <= baseList.Count - 1; j++)
{
//Trace.TraceInformation("Starting Instance Report Comparison Inner Loop.");
//Compare the actual R Files:
swCompare.Reset();
swCompare.Start();
//find the exact R file.
InstanceReport currentBase = baseList[j];
string baseLongName = WHITE_SPACE.Replace(currentBase.ReportLongName.Trim(), " ");
//string baseShortName = WHITE_SPACE.Replace( currentBase.ReportName.Trim(), " " );
InstanceReport currentNew = newList.Find(
nl =>
WHITE_SPACE.Replace(nl.ReportLongName.Trim(), " ") == baseLongName);
//try to match by report long name if it's there. If not, try by report name
if (currentNew != null)
{
string newLongName = WHITE_SPACE.Replace(currentNew.ReportLongName.Trim(), " ");
string newShortName = WHITE_SPACE.Replace(currentNew.ReportName.Trim(), " ");
if (!string.IsNullOrEmpty(newLongName))
{
workingHeader = baseHeaders.Find(
bh =>
WHITE_SPACE.Replace(bh.LongName.Trim(), " ") == newLongName);
}
if (workingHeader == null)
{
workingHeader = baseHeaders.Find(
bh =>
WHITE_SPACE.Replace(bh.ShortName.Trim(), " ") == newShortName);
}
if (workingHeader != null && baseHTML)
{
//build base html files. Skip if this already exists.
string xmlFilename = allB[i].FullName + "\\" + workingHeader.XmlFileName;
string htmlFilename = xmlFilename + ".html";
if (!File.Exists(htmlFilename))
{
this.HTML(currentBase, xmlFilename, htmlFilename);
}
}
if (currentNew != null && workingHeader != null)
{
//html.RFile = workingHeader.XmlFileName;
string newPath = Path.Combine(newDirectory.FullName, workingHeader.XmlFileName);
this.rFileLoc1 = Path.Combine(newDirectory.FullName, Path.GetFileNameWithoutExtension(newPath) + ".html");
string basePath = Path.Combine(baseDirectory.FullName, workingHeader.XmlFileName);
this.rFileLoc2 = Path.Combine(baseDirectory.FullName, Path.GetFileNameWithoutExtension(basePath) + ".html");
if (!this.ReportCompare(currentBase, currentNew, basePath, newPath, compareStart, out error))
{
failedRFiles++;
reportFailed = true;
}
}
else
{
RLogger.Error("New report does not contain a matching report name.");
htmlStarter("New report does not contain a matching report name<br>New Report does not contain a matching report name.",
compareStart, out error);
failedRFiles++;
reportFailed = true;
}
totalRFiles++;
TimeSpan seconds = TimeSpan.FromMilliseconds(swCompare.ElapsedMilliseconds);
swCompare.Stop();
}
else //if (currentNew == null)
{
RLogger.Info("The 'Current' report was null. Setting appropriate variables and closing the current R report.");
//Set the variables, write the log entry and save the HTML
workingHeader = baseHeaders.Find(bh => bh.LongName == baseList[j].ReportLongName);
rFileLoc1 = allN[i].FullName + "\\" +
workingHeader.XmlFileName.Remove(workingHeader.XmlFileName.Length - 4) + ".html";
rFileLoc2 = baseDirectory.FullName + "\\" +
workingHeader.XmlFileName.Remove(workingHeader.XmlFileName.Length - 4) + ".html";
htmlStarter("Could not match report name for this report<br>The 'new' report name could not be matched", compareStart, out error);
failedRFiles++;
reportFailed = true;
}
}
//finish up the last report:
if (reportFailed)
{
failedFilings++;
}
totalFilings++;
if (html.reportStarted)
{
html.EndReport();
}
}
else
{
RLogger.Error("Could not find matching directory.");
Trace.TraceWarning("Warning: Could not find matching directory.");
}
}
}
catch (Exception ex)
{
Trace.TraceInformation("Error: An error occured trying to compare the following report {0}: \n{1}", reportName, ex.Message);
RLogger.Error(string.Format("An error occured trying to compare the following report {0}: \n{1}", reportName, ex.Message));
}
DateTime compareEnd = DateTime.Now;
TimeSpan timeToCompare = compareEnd.Subtract(compareStart);
//get render time:
if (html.reportStarted)
{
html.EndReport();
}
html.BuildSummary(compareStart, compareEnd, minCount, failedFilings, totalRFiles, failedRFiles, timeToCompare, renderStart,
renderEnd, totalRender, renderCount);
//string path = string.Format("{0}\\Logs\\AutoTesterLog-{1}.html", newDirs.Root.FullName, DateTime.Now.ToString("yyyy-MM-dd_hhmmss"));
DirectoryInfo logPath = new DirectoryInfo(System.Configuration.ConfigurationManager.AppSettings["logPath"]);
if (logPath.Exists)
{
string path = string.Format("{0}\\AutoTesterLog-{1}.html", logPath.FullName, DateTime.Now.ToString("yyyy-MM-dd_hhmmss"));
HTML(html.html.ToString(), path);
}
else
{
Trace.TraceError("Error: Unable to write the HTML log file. Check to make sure the specified log directory exists.");
RLogger.Error("Unable to write the HTML log file. Check to make sure the specified log directory exists.");
}
}
/// <summary>
/// <para>Performs the "building of reports" based on the <paramref name="instancePath"/> and <paramref name="taxonomy"/> provided.</para>
/// <para>As a result, <paramref name="filingSummary"/>.<see>MyReports</see> (<see cref="FilingSummary"/>) will be populated with instances of <see cref="ReportHeader"/>,</para>
/// <para>and <paramref name="filingSummary"/>.<see>MyReports</see> will be XML serialized to <paramref name="filingSummaryPath"/></para>
/// <para>If an error occurs, <paramref name="error" /> will be populated.</para>
/// </summary>
/// <param name="instancePath">The path to the instance document.</param>
/// <param name="taxonomy">The loaded and parsed taxonomy object.</param>
/// <param name="filingSummaryPath">The path where the generated <see cref="FilingSummary"/> object should be saved.</param>
/// <param name="reportDirectory">The path where the generated content is saved.</param>
/// <param name="filingSummary">The <see cref="FilingSummary"/> object to populate.</param>
/// <param name="error">The error message for any critical errors which might occur.</param>
/// <returns>True on success or false for fail.</returns>
public bool BuildReports( string instancePath, Taxonomy taxonomy,
string filingSummaryPath, string reportDirectory,
out FilingSummary filingSummary, out string error )
{
error = string.Empty;
filingSummary = null;
DateTime dtStart = DateTime.Now;
try
{
this.currentFilingSummary = new FilingSummary();
this.currentInstancePath = instancePath;
this.currentTaxonomy = taxonomy;
this.currentReportDirectory = reportDirectory;
if( !this.ValidateSettings( out error ) )
return false;
//create the reports directory so that defnref can be generated
if( !Directory.Exists( this.currentReportDirectory ) )
Directory.CreateDirectory( this.currentReportDirectory );
if( !string.IsNullOrEmpty( this.CurrencyMappingFile ) )
this.LoadCurrencies();
//set up this symbol for reuse throughout
InstanceUtils.USDCurrencySymbol = InstanceUtils.GetCurrencySymbolFromCode( InstanceUtils.USDCurrencyCode );
if( string.IsNullOrEmpty( this.preferredLanguage ) )
this.preferredLanguage = this.GetPreferredLanguage();
bool isIMBased = this.CheckIsIMBased();
//DateTime startInstance = DateTime.Now;
ArrayList errors = null;
if( !InstanceUtils.TryLoadInstanceDocument( this.currentInstancePath, out this.currentInstance, out errors ) )
{
string[] arrErrors = new string[ errors.Count ];
for( int i = 0; i < errors.Count; i++ )
{
arrErrors[ i ] = ( (ParserMessage)errors[ i ] ).Message;
}
string instanceDocErrors = string.Join( "\r\n ", arrErrors );
Regex splitPoint = new Regex( @"\S\s+at" );
arrErrors = splitPoint.Split( instanceDocErrors );
instanceDocErrors = arrErrors[ 0 ];
error = "Unable to load the instance document:\r\n " + instanceDocErrors;
return false;
}
ArrayList taxonomies = new ArrayList();
taxonomies.Add( this.currentTaxonomy );
this.currentInstance.FixPrefixInInstanceDocument( taxonomies );
//this.currentFilingSummary.InstanceLoadTime = DateTime.Now - startInstance;
//this.currentFilingSummary.FactCount = this.currentInstance.markups.Count;
this.PopulateMarkupDictionaries();
this.BuildFilingSummary();
this.FireBuildReportsProcessing();
this.currentFilingSummary.MyReports.Clear();
ArrayList topNodes = currentTaxonomy.GetNodesByPresentation( false, this.ExcludedReports );
foreach( Node topNode in topNodes )
{
InstanceReport report = null;
try
{
if( this.BuildReport( topNode, out report ) )
{
if( report.IsEmbedReport || report.HasEmbeddedReports )
{
this.roleAxes[ report.RoleURI ] = report.AxisByPresentation;
this.roleAxisDefaults[ report.RoleURI ] = report.AxisMemberDefaults;
this.roleAxisMembers[ report.RoleURI ] = report.AxisMembersByPresentation;
}
ReportHeader header = this.currentFilingSummary.AddReport( report );
this.ApplyRulesToReport( report );
report.OnRuleProcessing -= this.OnRuleProcessing;
report.OnRuleProcessing += this.OnRuleProcessing;
report.OnRuleProcessed -= this.OnRuleProcessed;
report.OnRuleProcessed += this.OnRuleProcessed;
report.UnitDictionary = this.unitDictionary;
string fullPath = Path.Combine( this.currentReportDirectory, header.XmlFileName );
report.BuildXMLDocument( fullPath, true, isIMBased, this.currentFilingSummary );
report.UnitDictionary = null;
report.OnRuleProcessing -= this.OnRuleProcessing;
report.OnRuleProcessed -= this.OnRuleProcessed;
if( header.HasEmbeddedReports )
this.reportsWithEmbeds.Add( header );
}
}
finally
{
if( report != null )
report.Dispose();
}
}
//Build Missing Elements (Uncategorized) Report before we flush the markups
InstanceReport missingReport;
if( this.BuildMissingDataReport( out missingReport ) )
{
ReportHeader uncatHeader = this.currentFilingSummary.AddReport( missingReport );
uncatHeader.XmlFileName = InstanceUtils.TOP_LEVEL_REPORT_INDICATOR+ _missingReportIndex +".xml";
}
//Free up some resources
this.currentInstance = null;
if( this.internalReports.Count == 0 )
{
this.currentTaxonomy.Close();
this.currentTaxonomy = null;
}
//clear the dictionaries after checking internalReports
this.ClearMarkupDictionaries();
#region Clean up Columns
//if the company has filed earning release, do not touch the reports.
//Based on request from SEC, do not remove any reports if the base taxonomy is the new GAAP taxonomy (2208)
if( !this.HasEarningRelease() )
{
//DO NOT REMOVE - These are used in rule processing
isGAAP2005 = this.TaxonomyIsGAAP2005();
isNewGAAP = ( !isGAAP2005 );
ProcessFlowThroughColumnsReports();
}
#endregion
#region Build All Reports
//Build book
ReportHeader r1 = new ReportHeader();
r1.LongName = _allReports;
r1.ShortName = _allReports;
r1.ReportType = ReportHeaderType.Book;
r1.IsDefault = this.currentFilingSummary.MyReports.Count == 0;
this.currentFilingSummary.AddReports( r1 );
#endregion
#region Process Embeeded Reports
bool isRuleEnabled = this.FireRuleProcessing( RulesEngineUtils.EMBED_REPORTS_RULE );
if( isRuleEnabled && this.reportsWithEmbeds.Count > 0 )
{
List<ReportHeader> embedReports = this.ProcessEmbeddedReports();
if( embedReports.Count > 0 )
{
foreach( ReportHeader embedReport in embedReports )
{
if( this.currentFilingSummary.MyReports.Contains( embedReport ) )
{
this.currentFilingSummary.MyReports.Remove( embedReport );
string reportNameToDelete = Path.Combine( this.currentReportDirectory, embedReport.XmlFileName );
if( File.Exists( reportNameToDelete ) )
File.Delete( reportNameToDelete );
}
}
}
this.FireRuleProcessed( RulesEngineUtils.EMBED_REPORTS_RULE );
}
#endregion
#region Generate Excel Workbook
isRuleEnabled = this.FireRuleProcessing( RulesEngineUtils.GENERATE_EXCEL_RULE );
if( isRuleEnabled )
{
string excelReportName = "Financial_Report";
bool processed = ExcelUtility.GenerateExcelWorkbook( this.currentFilingSummary, this.currentReportDirectory, excelReportName );
this.FireRuleProcessed( RulesEngineUtils.GENERATE_EXCEL_RULE );
if( !processed )
{
error = "Failed to generate Excel Workbook for report: " + this.currentReportDirectory + " " + excelReportName;
return false;
}
}
#endregion
#region Generate Output Formats
if( ( this.ReportFormat & ReportFormat.Html ) == ReportFormat.Html )
{
if( !this.GenerateHtmlFiles() )
{
//The error was logged to the tracer
}
}
if( ( this.ReportFormat & ReportFormat.Xml ) == ReportFormat.Xml )
{
//this format already exists
}
else
{
// this.currentFilingSummary.TraceInformation( "Information: Report Formart does not include 'Xml'. Removing references and files." );
foreach( ReportHeader rh in this.currentFilingSummary.MyReports )
{
if( !string.IsNullOrEmpty( rh.HtmlFileName ) )
{
string deleteFile = Path.Combine( this.currentReportDirectory, rh.XmlFileName );
if( File.Exists( deleteFile ) )
File.Delete( deleteFile );
rh.XmlFileName = null;
}
}
}
#endregion
#region Check and fix default report
//Recheck the default report just in case it was deleted
bool foundDefault = false;
foreach( ReportHeader header in this.currentFilingSummary.MyReports )
{
if( header.IsDefault )
{
foundDefault = true;
break;
}
}
if( !foundDefault )
{
foreach( ReportHeader header in this.currentFilingSummary.MyReports )
{
if( header.IsBalanceSheet() || header.ReportType == ReportHeaderType.Book )
{
header.IsDefault = true;
break;
}
}
}
#endregion
return true;
}
catch( Exception ex )
{
error = string.Format( "Exception thrown in BuildReports: {0}", ex.Message );
return false;
}
finally
{
this.FireBuildReportsProcessed();
this.currentFilingSummary.ProcessingTime = DateTime.Now - dtStart;
this.currentFilingSummary.SaveAsXml( filingSummaryPath );
filingSummary = this.currentFilingSummary;
this.currentFilingSummary = null;
if( this.currentTaxonomy != null )
this.currentTaxonomy.Close();
this.currentTaxonomy = null;
}
}
/// <summary>
/// Builds a nested dictionary of all elements and which Segments they are combined with for 1 report, specified by <paramref name="rh"/>.
/// </summary>
/// <param name="rh">The <see cref="ReportHeader"/> whose element/segment combinations should be loaded.</param>
/// <returns></returns>
private InstanceReport.ElementSegmentCombinations BuildInUseElementSegmentCombinationsForCurrentReport(ReportHeader rh)
{
if (rh.ReportType == ReportHeaderType.Sheet || rh.ReportType == ReportHeaderType.Notes)
{
if (!this.uniqueReportElementSegmentCombos.ContainsKey(rh.XmlFileName) ||
this.uniqueReportElementSegmentCombos == null ||
this.uniqueReportElementSegmentCombos.Count == 0)
{
InstanceReport report = InstanceReport.LoadXml(Path.Combine(this.currentReportDirectory, rh.XmlFileName));
InstanceReport.ElementSegmentCombinations elementSegmentCombinations = report.GetUniqueInUseElementSegmentCombinations();
this.uniqueReportElementSegmentCombos[rh.XmlFileName] = elementSegmentCombinations;
}
}
if (this.uniqueReportElementSegmentCombos.ContainsKey(rh.XmlFileName))
{
return(uniqueReportElementSegmentCombos[rh.XmlFileName]);
}
return(null);
}
/// <summary>
/// Builds a nested dictionary of all elements and which Segments they are combined with for all reports.
/// </summary>
/// <param name="rhCurrent">The current <see cref="ReportHeader"/> - its report will be skipped from cataloging.</param>
/// <returns></returns>
private InstanceReport.ElementSegmentCombinations BuildInUseElementSegmentCombinationsForAllReports(ReportHeader rhCurrent)
{
InstanceReport.ElementSegmentCombinations allReportCombinations = new InstanceReport.ElementSegmentCombinations();
foreach (ReportHeader rh in this.currentFilingSummary.MyReports)
{
if (rh.XmlFileName == rhCurrent.XmlFileName)
{
continue;
}
InstanceReport.ElementSegmentCombinations reportCombinations = this.BuildInUseElementSegmentCombinationsForCurrentReport(rh);
if (reportCombinations != null)
{
allReportCombinations.Merge(reportCombinations);
}
}
return(allReportCombinations);
}
/// <summary>
/// Go thorugh each column to determine if the columns should be removed --
/// if all elements (that have data) in the columns can be found in other reports then consider this a flow thru column and delete it.
/// For "disclosures", if one element or one dimension member s unique, do remove ANY columns from the disclosure
/// </summary>
/// <param name="rh"></param>
/// <param name="inUseElements"></param>
/// <param name="inUseSegments"></param>
private void CleanupColumns(ReportHeader rh, Dictionary <string, int> inUseElements, Dictionary <string, int> inUseSegments)
{
string filePath = Path.Combine(this.currentReportDirectory, rh.XmlFileName);
InstanceReport report = InstanceReport.LoadXml(filePath);
if (report == null)
{
return;
}
List <InstanceReportColumn> columnsToRemove = new List <InstanceReportColumn>();
List <InstanceReportRow> uniqueRows = report.Rows.FindAll(
row =>
{
if (string.IsNullOrEmpty(row.ElementName))
{
return(false);
}
if (row.IsEmpty())
{
return(false);
}
if (inUseElements.ContainsKey(row.ElementName))
{
return(false);
}
return(true);
});
bool hasAnyUniqeRows = uniqueRows.Count > 0;
for (int c = 0; c < report.Columns.Count; c++)
{
InstanceReportColumn col = report.Columns[c];
bool columnHasUniqueSegments = false;
bool columnHasUniqueCells = ReportUtils.Exists(uniqueRows, row => row.Cells[c].HasData);
if (!columnHasUniqueCells) //this column might need to be removed
{
Dictionary <string, int> colSegments = report.GetUniqueInUseSegments(col);
foreach (string key in colSegments.Keys)
{
if (!inUseSegments.ContainsKey(key))
{
columnHasUniqueSegments = true;
break;
}
}
}
if (!(columnHasUniqueCells || columnHasUniqueSegments))
{
columnsToRemove.Add(col);
}
}
if (columnsToRemove.Count > 0 && columnsToRemove.Count < report.Columns.Count)
{
foreach (InstanceReportColumn col in columnsToRemove)
{
col.RemoveSelf(report);
}
if (report.Columns.Count > 1)
{
report.PromoteSharedColumnLabelsAfterRemoveFlowThroughColumns();
}
report.BuildXMLDocument(filePath, false, false, this.currentFilingSummary);
}
}
/// <summary>
/// Go thorugh each column to determine if the columns should be removed --
/// if all elements (that have data) in the columns can be found in other reports then consider this a flow thru column and delete it.
/// For "disclosures", if one element or one dimension member s unique, do remove ANY columns from the disclosure
/// </summary>
/// <param name="rh"></param>
/// <param name="allInUseElementsSegments"></param>
private void CleanupColumns(ReportHeader rh, InstanceReport.ElementSegmentCombinations allInUseElementsSegments)
{
string filePath = Path.Combine(this.currentReportDirectory, rh.XmlFileName);
InstanceReport report = InstanceReport.LoadXml(filePath);
if (report == null)
{
return;
}
InstanceReportColumn lastSegmentColumn = null;
bool hasSegmentsOnRows = report.HasSegmentsOnRows();
List <InstanceReportColumn> columnsToRetain = new List <InstanceReportColumn>();
foreach (InstanceReportRow row in report.Rows)
{
if (hasSegmentsOnRows)
{
if (row.OriginalInstanceReportColumn != null)
{
lastSegmentColumn = row.OriginalInstanceReportColumn;
}
}
//we can't test this row
if (string.IsNullOrEmpty(row.ElementName))
{
continue;
}
//we can't test this row
if (row.IsEmpty())
{
continue;
}
for (int c = 0; c < row.Cells.Count; c++)
{
//skip this cell if we already have this column
InstanceReportColumn currentColumn = report.Columns[c];
if (columnsToRetain.Contains(currentColumn))
{
continue;
}
//we only want columns for cells which have data
Cell cell = row.Cells[c];
if (!cell.HasData)
{
continue;
}
//if the element doesn't even exist, we're set - add the column
InstanceReportColumn segmentColumn = hasSegmentsOnRows ? lastSegmentColumn : currentColumn;
if (!allInUseElementsSegments.ContainsElement(row.ElementName))
{
columnsToRetain.Add(currentColumn);
}
else if (segmentColumn != null)
{
if (segmentColumn.Segments != null && segmentColumn.Segments.Count > 0)
{
//if the element exists, but maybe its combination with the segments does not
StringBuilder sb = new StringBuilder();
foreach (string segmentKey in segmentColumn.GetUniqueInUseSegments().Keys)
{
sb.AppendLine(segmentKey);
}
if (!allInUseElementsSegments.ContainsElementAndSegments(row.ElementName, sb.ToString()))
{
columnsToRetain.Add(currentColumn);
}
}
}
}
if (columnsToRetain.Count == report.Columns.Count)
{
return;
}
}
if (columnsToRetain.Count == 0 || columnsToRetain.Count == report.Columns.Count)
{
return;
}
List <InstanceReportColumn> columnsToRemove = report.Columns.FindAll(col => !columnsToRetain.Contains(col));
foreach (InstanceReportColumn col in columnsToRemove)
{
this.currentFilingSummary.TraceInformation("\tProcess Flow-Through: Removing column '" + col.Label + "'");
col.RemoveSelf(report);
}
if (report.Columns.Count > 1)
{
report.PromoteSharedColumnLabelsAfterRemoveFlowThroughColumns();
}
report.BuildXMLDocument(filePath, false, false, this.currentFilingSummary);
}
/// <summary>
/// For each report, check to see if at least ONE element do not exist in all other reports. If not, delete the report.
/// For each column, check to see if at least ONE element do not exist in all other reports. If not, delete the column.
/// </summary>
public void CleanupFlowThroughReports()
{
ArrayList reportsToRemove = new ArrayList();
foreach (ReportHeader rh in this.currentFilingSummary.MyReports)
{
//EH: Based on Christy's request, if the report is Stockholder's Equity, do not remove columns
//DuPont and Oceanic filings will not look correct if we remove the columns. All elements for certain periods are in both
//Balance Sheet and Stockholder's Equity
if ((rh.ReportType == ReportHeaderType.Sheet || rh.ReportType == ReportHeaderType.Notes))
{
if (!rh.IsStatementOfStockholdersEquity())
{
Dictionary <string, int> htElementsCurrentReport = this.BuildInUseElementsForCurrentReport(rh);
Dictionary <string, int> htElementsOtherReports = this.BuildInUseElementsForAllReports(rh);
//Dictionary<string, int> uniqueSegments = this.BuildInUseSegmentsForAllReports( rh );
int elementFoundInOtherReports = 0;
foreach (string currentElementName in htElementsCurrentReport.Keys)
{
if (htElementsOtherReports.ContainsKey(currentElementName))
{
elementFoundInOtherReports += 1;
}
}
if (elementFoundInOtherReports == htElementsCurrentReport.Count) //all elements in other report
{
//Remove the report
if (!ReportUtils.IsHighlights(rh.ShortName) && !ReportUtils.IsBalanceSheet(rh.ShortName) && !ReportUtils.IsIncomeStatement(rh.ShortName))
{
reportsToRemove.Add(rh);
}
}
else
{
InstanceReport.ElementSegmentCombinations allInUseElementsSegments = this.BuildInUseElementSegmentCombinationsForAllReports(rh);
this.CleanupColumns(rh, allInUseElementsSegments);
this.uniqueReportElementSegmentCombos.Remove(rh.XmlFileName);
//CleanupColumns( rh, htElementsOtherReports, uniqueSegments );
}
}
}
}
if (reportsToRemove.Count > 0)
{
foreach (ReportHeader rhToRemove in reportsToRemove)
{
//delete from MyFilingSummary
for (int rIndex = currentFilingSummary.MyReports.Count - 1; rIndex >= 0; rIndex--)
{
ReportHeader rh1 = currentFilingSummary.MyReports[rIndex] as ReportHeader;
if (rh1.XmlFileName == rhToRemove.XmlFileName)
{
currentFilingSummary.MyReports.RemoveAt(rIndex);
break;
}
}
string fileName = currentReportDirectory + Path.DirectorySeparatorChar + rhToRemove.XmlFileName;
this.currentFilingSummary.TraceWarning("Process Flow-Through Report: removing '" + rhToRemove.LongName + "'");
File.Delete(fileName);
}
}
}
public void AddReports(ReportHeader r)
{
this.reports.Add(r);
}