public void Test_30553_snda_20101231()
{
const string accessionNumber = "30553_snda-20101231";
const string instanceName = "snda-20101231";
const string taxonomyName = "snda-20101231";
FilingSummary fs = this.BuildAndVerifyRecursive(accessionNumber, instanceName, taxonomyName);
Assert.AreEqual(6, fs.Logs.Count, "This filing should not generate any logs regarding embed commands and/or tildas (~).");
}
//[Description( "SEC 130" )]
public void Test_29034_ng001decimals3()
{
const string accessionNumber = "29034_ng001decimals3";
const string instanceName = "xyzCorp-20111231";
const string taxonomyName = "xyzCorp-20111231";
FilingSummary fs = this.BuildAndVerifyRecursive(accessionNumber, instanceName, taxonomyName);
LogItem item = fs.Logs.Find(log => string.Equals(log.Item, "Element xyzCorp_Afact had a mix of decimals attribute values: -9 -6 -3."));
Assert.AreNotEqual(null, item, "The previous log message should be in the FilingSummary");
}
//[Description( "SEC 131" )]
public void Test_29030_SEC0131()
{
const string accessionNumber = "29030_SEC0131";
const string instanceName = "mon-20101231";
const string taxonomyName = "mon-20101231";
const string error1 = "Multiple embeddings in Element \"Investment Holdings Details Text Block\" with UnitID \"No UnitID Specified\" and contextID \"no-dim-D\". Only the first embed command will be used.";
FilingSummary logCheckSummary = this.BuildAndVerifyRecursive(accessionNumber, instanceName, taxonomyName);
LogItem item = logCheckSummary.Logs.Find(log => log.Item.Contains(error1));
Assert.AreNotEqual(null, item, "An expected warning was not found in the filing summary. The warning not found is:" + error1);
}
public bool BuildReports(FilingInfo filing)
{
string instancePath = GetInstanceDocPath(filing);
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
myEventLog.WriteEntry("Can not find instance document for filing: " + filing.AccessionNumber);
return(false);
}
string taxonomyPath = GetTaxonomyPath(filing);
if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
myEventLog.WriteEntry("Can not find taxonomy file for filing: " + filing.AccessionNumber);
return(false);
}
ReportBuilder rb = new ReportBuilder();
string reportPath = string.Format("{0}{1}{2}", reportsFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
string filingFile = string.Format("{0}{1}{2}", reportPath, Path.DirectorySeparatorChar, FilingSummary._FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(reportPath))
{
Directory.Delete(reportPath, true);
}
Directory.CreateDirectory(reportPath);
//TODO: get this info from the manifest once we have it
//rb.PeriodEnding = filing.period_ending;
//rb.FilingDate = filing.filing_date;
//rb.TickerSymbol = filing.ticker_symbol;
//rb.CompanyName = filing.company_name;
//rb.AccessionNumber = filing.accession_number;
//rb.FiscalYearEnd = filing.fiscal_year_end;
string error = null;
FilingSummary summary = null;
if (!rb.BuildReports(instancePath, taxonomyPath, filingFile, reportPath, filing.FormType, out summary, out error))
{
myEventLog.WriteEntry("build reports failed! " + error, EventLogEntryType.Error);
return(false);
}
//Increment the number of filings that were successfully process. This needs to be done
//using Interlocked because other worker threads could be accessing the property as well
Interlocked.Increment(ref completedFilings);
return(true);
}
//[Description( "SEC 140" )]
public void Test_29039()
{
Test_Abstract.OutputFormat = (Test_Abstract.OutputFormat & ReportFormat.Xml) | ReportFormat.Html;
Test_Abstract.HtmlFormat = HtmlReportFormat.Complete;
const string accessionNumber = "29039";
string reportsFolder = PathCombine(this.baseDir, accessionNumber, "Reports");
CleanAndPrepareFolder(reportsFolder);
string resultsFolder = PathCombine(this.baseDir, accessionNumber, "Results");
FilingSummary fs = new FilingSummary();
foreach (string from in Directory.GetFiles(resultsFolder, "R1_*.xml"))
{
string file = Path.GetFileName(from);
string to = PathCombine(reportsFolder, file);
FileUtilities.Copy(from, to);
ReportHeader header = new ReportHeader(file, file);
fs.MyReports.Add(header);
}
XRB.ReportBuilder rb = new XRB.ReportBuilder();
rb.ReportFormat = ReportFormat.Html;
rb.HtmlReportFormat = HtmlReportFormat.Complete;
rb.GetType().GetField("currentFilingSummary", BindingFlags.NonPublic | BindingFlags.Instance).SetValue(rb, fs);
rb.GetType().GetField("currentReportDirectory", BindingFlags.NonPublic | BindingFlags.Instance).SetValue(rb, reportsFolder);
rb.GetType().GetMethod("GenerateHtmlFiles", BindingFlags.NonPublic | BindingFlags.Instance).Invoke(rb, null);
foreach (string file in Directory.GetFiles(reportsFolder, "R1_*.htm"))
{
string html = File.ReadAllText(file);
int start = html.IndexOf("<body");
int end = html.IndexOf("</body");
end = html.IndexOf('>', end) + 1;
html = html.Substring(start, end - start);
html = html.Replace("<br>", "<br />");
XmlDocument xDoc = new XmlDocument();
xDoc.LoadXml(html);
XmlNodeList headerRows = xDoc.SelectNodes("/body/table/tr[ th ]");
Assert.AreEqual(2, headerRows.Count, "The transform must generate 2 headers rows for the right display.");
}
}
/// <summary>
/// The method name refers to RR, but it is for all Instance Documents
/// </summary>
/// <param name="reportDirectory">The folder containing all reports (R1.xml, etc).
/// This is also the destination folder.</param>
/// <param name="excelReportName">The destination file name, without path or extention</param>
/// <param name="xsltPath">The path to InstanceReport_XmlWorkbook.xslt</param>
/// <param name="cssPath">The path to report.css</param>
/// <returns></returns>
public static bool GenerateExcelWorkbook( FilingSummary fs, string reportDirectory, string excelReportName )
{
if( !Directory.Exists( reportDirectory ) )
return false;
try
{
SingleFileWorkbook sfwb = GenerateExcelWorkbook( fs, reportDirectory );
string saveAt = Path.Combine( reportDirectory, excelReportName +".xls" );
sfwb.SaveAs( saveAt );
return true;
}
catch { }
return false;
}
protected FilingSummary BuildFiling(string accessionNumberIn, string instanceNameIn, string taxonomyNameIn)
{
if (!ConcurrentTestsRunning)
{
Type myType = MethodInfo.GetCurrentMethod().DeclaringType;
MethodBase method = null;
StackTrace trace = new StackTrace();
StackFrame[] frames = trace.GetFrames();
for (int f = 0; f < frames.Length; f++)
{
method = frames[f].GetMethod();
if (method.DeclaringType != myType)
{
break;
}
}
if (ConcurrentTestsResults.ContainsKey(method.Name))
{
//RB_Tests.IsConcurrentTestResult = true;
if (ConcurrentTestsResults[method.Name] != null)
{
throw ConcurrentTestsResults[method.Name];
}
else
{
return(null);
}
}
}
string error;
FilingSummary filingSummary = null;
string folderPath = Path.Combine(baseDir, accessionNumberIn);
bool buildSuccess = BuildReports(folderPath, instanceNameIn, taxonomyNameIn, out filingSummary, out error);
if (!buildSuccess)
{
throw new Exception("Failed to generate reports: " + error);
}
this.myFilingSummary = filingSummary;
return(filingSummary);
}
protected FilingSummary BuildAndVerifyRecursive(string folder, string instanceName, string taxonomyName)
{
Console.WriteLine("=================================================");
Console.WriteLine(" BUILD AND VERIFY " + folder + "...");
Console.WriteLine("=================================================");
string baselineResultFolder = PathCombine(this.baseDir, folder, "Results");
this.resultsPath = baselineResultFolder;
if (!BuildAndVerifyWithoutResults && !Directory.Exists(this.resultsPath))
{
throw new Exception("There are no results to verify againsts.");
}
string generatedResultFolder = PathCombine(this.baseDir, folder, "Reports");
this.reportsPath = generatedResultFolder;
string failedResultsFolder = Path.GetDirectoryName(generatedResultFolder);
failedResultsFolder = Path.Combine(failedResultsFolder, "Failures");
CleanAndPrepareFolder(failedResultsFolder);
FilingSummary fs = BuildFiling(folder, instanceName, taxonomyName);
using (ErrorWriter writer = new ErrorWriter())
{
ExtendedDirectoryInfo baselineResultInfo = new ExtendedDirectoryInfo(baselineResultFolder);
ExtendedDirectoryInfo generatedResultInfo = new ExtendedDirectoryInfo(generatedResultFolder);
VerifyIndividualReports(writer, baselineResultInfo, generatedResultInfo, failedResultsFolder);
VerifyCharts(writer, baselineResultInfo, generatedResultInfo);
if (writer.HasErrors)
{
writer.Flush();
writer.SaveResults(failedResultsFolder);
Console.Write(writer.GetStringBuilder().ToString().Trim());
throw new Exception(writer.GetStringBuilder().ToString().Trim());
}
}
return(fs);
}
public bool BuildReports(string instanceFolder, string instanceName, string taxonomyName, out FilingSummary filingSummary, out string error)
{
Console.WriteLine("=================================================");
Console.WriteLine(" BUILD " + instanceFolder + "...");
Console.WriteLine("=================================================");
this.processingTime = TimeSpan.Zero;
DateTime startTime = DateTime.Now;
string instance = PathCombine(instanceFolder, instanceName + ".xml");
string taxonomy = PathCombine(instanceFolder, taxonomyName + ".xsd");
string reportDir = PathCombine(instanceFolder, "Reports");
string filingSummaryPath = PathCombine(reportDir, "FilingSummary.xml");
CleanAndPrepareFolder(reportDir);
XRB.ReportBuilder myReportBuilder = new XRB.ReportBuilder();
myReportBuilder.ReportFormat = OutputFormat;
myReportBuilder.HtmlReportFormat = HtmlFormat;
myReportBuilder.RemoteFileCachePolicy = CachePolicy;
myReportBuilder.OnBuildReportsProcessing += this.FireBuildReportsProcessing;
myReportBuilder.OnBuildReportsProcessed += this.FireBuildReportsProcessed;
if (string.IsNullOrEmpty(ResourcePath))
{
RulesEngineUtils.SetBaseResourcePath(null);
}
else
{
RulesEngineUtils.SetBaseResourcePath(ResourcePath);
}
bool status = myReportBuilder.BuildReports(instance, taxonomy, filingSummaryPath, reportDir, out filingSummary, out error);
this.myFilingSummary = status ? filingSummary : null;
myReportBuilder.OnBuildReportsProcessing -= this.FireBuildReportsProcessing;
myReportBuilder.OnBuildReportsProcessed -= this.FireBuildReportsProcessed;
DateTime endTime = DateTime.Now;
this.processingTime = (endTime - startTime);
return(status);
}
/// <summary>
/// validates that the report files where generated correctly, but does not validate the contents.
/// That functionality is unit tested in the ReportBuilder project
/// </summary>
/// <param name="filingSummary"></param>
/// <param name="folderPath"></param>
private void ValidateReports(string accensionNumber)
{
string generatedReportsPath = string.Format(@"{0}{1}{2}", this.reportsFolder, Path.DirectorySeparatorChar, accensionNumber);
string generatedSummaryPath = string.Format(@"{0}{1}Filingsummary.xml", generatedReportsPath, Path.DirectorySeparatorChar);
Assert.IsTrue(File.Exists(generatedSummaryPath), "Filing Summary file was not generated");
object objSummary;
string errorMsg;
Assert.IsTrue(TryXmlDeserializeObjectFromFile(generatedSummaryPath, typeof(FilingSummary), out objSummary, out errorMsg), "Can not validate FilingSummary, unable to deserialize the \"Expected\" FilingSummary from disk: " + errorMsg);
FilingSummary filingSummary = objSummary as FilingSummary;
Assert.IsNotNull(filingSummary, "Can not validate FilingSummary, unable to deserialize the FilingSummary from file.");
foreach (ReportHeader rh in filingSummary.MyReports)
{
string generatedReportFile = string.Format("{0}{1}{2}", generatedReportsPath, Path.DirectorySeparatorChar, rh.XmlFileName);
if (!string.IsNullOrEmpty(rh.XmlFileName))
{
Assert.IsTrue(File.Exists(generatedReportFile), "Report file was not generated: " + rh.XmlFileName);
}
}
}
//[Description( "SEC 131" )]
public void Test_29030_multiple()
{
const string accessionNumber = "29030_multiple";
const string instanceName = "eggplant-20090701";
const string taxonomyName = "eggplant-20090701";
const string warning1 = "Multiple embeddings in Element \"Risk/Return Detail [Table]\" with UnitID \"No UnitID Specified\" and contextID \"_\". Only the first embed command will be used.";
const string warning2 = "Multiple embeddings in Element \"Shareholder Fees [Table]\" with UnitID \"No UnitID Specified\" and contextID \"Investors_S123456789\". Only the first embed command will be used.";
const string warning3 = "Multiple embeddings in Element \"Annual Fund Operating Expenses [Table]\" with UnitID \"No UnitID Specified\" and contextID \"Investors_S123456789\". Only the first embed command will be used.";
const string warning4 = "Multiple embeddings in Element \"Expense Example with Redemption [Table]\" with UnitID \"No UnitID Specified\" and contextID \"Investors_S123456789\". Only the first embed command will be used.";
FilingSummary logCheckSummary = this.BuildAndVerifyRecursive(accessionNumber, instanceName, taxonomyName);
bool logCheckExists = true;
logCheckExists &= logCheckSummary.Logs.Exists(iLog => iLog.Item.Contains(warning1));
logCheckExists &= logCheckSummary.Logs.Exists(iLog => iLog.Item.Contains(warning2));
logCheckExists &= logCheckSummary.Logs.Exists(iLog => iLog.Item.Contains(warning3));
logCheckExists &= logCheckSummary.Logs.Exists(iLog => iLog.Item.Contains(warning4));
if (!logCheckExists)
{
throw new Exception("This filing has four embed commands that should throw warnings. The warnings were not found in the filing summary.");
}
}
/// <summary>
///
/// </summary>
/// <param name="reportDirectory">The folder containing all reports (R1.xml, etc)</param>
/// <param name="xsltPath">The path to InstanceReport_XmlWorkbook.xslt</param>
/// <param name="cssPath">The path to report.css</param>
/// <returns></returns>
public static SingleFileWorkbook GenerateExcelWorkbook( FilingSummary myFilingSummary, string reportDirectory )
{
string cssPath = RulesEngineUtils.GetResourcePath( RulesEngineUtils.ReportBuilderFolder.Resources, RulesEngineUtils.StylesheetFile );
string xsltPath = RulesEngineUtils.GetResourcePath( RulesEngineUtils.ReportBuilderFolder.Resources, RulesEngineUtils.TransformWorkbookFile );
XslCompiledTransform xslt = new XslCompiledTransform();
xslt.Load( xsltPath );
SingleFileWorkbook sfwb = new SingleFileWorkbook();
Dictionary<string, int> sheetNames = new Dictionary<string, int>();
//Process the R files. Each R File will be transformed into a worksheet in the workbook
int idx = 0;
foreach( ReportHeader reportHeader in myFilingSummary.MyReports )
{
if( reportHeader.ReportType == ReportHeaderType.Book )
continue;
string reportPath = Path.Combine( reportDirectory, reportHeader.XmlFileName );
if( !File.Exists( reportPath ) )
{
Trace.TraceError(
"Error: An error occurred while creating the Excel Workbook.\n" +
"Sheet '" + reportHeader.ShortName + "' could not be loaded.\n" +
"Reason:\n\tFile not found."
);
continue;
}
try
{
idx++;
string tmpPath = Path.GetTempFileName();
if( File.Exists( tmpPath + ".html" ) )
{
FileInfo fi = new FileInfo( tmpPath + ".html" );
FileUtilities.DeleteFile( fi, true );
}
File.Move( tmpPath, tmpPath + ".html" );
tmpPath += ".html";
xslt.Transform( reportPath, tmpPath );
SingleFileAttachment sfa = sfwb.AddAttachment( tmpPath );
sfa.UseName = "Sheet" + idx.ToString( "00" ) + ".html";
sfa.Name = NameSheet( reportHeader.ShortName, sheetNames );
sheetNames[ sfa.Name ] = 1;
}
catch( Exception ex )
{
Trace.TraceWarning(
"Warning: An error occurred while creating the Excel Workbook.\n" +
"Sheet '" + reportHeader.ShortName + "' may be missing.\n"+
"Reason:\n\t" + ex.Message
);
}
}
string[] charts = Directory.GetFiles( reportDirectory, "*.jpg" );
foreach( string chart in charts )
{
try
{
SingleFileAttachment sfa = sfwb.AddAttachment( chart );
}
catch (Exception ex )
{
Trace.TraceWarning(
"Warning: An error occurred while creating the Excel Workbook.\n" +
"This attachment could not be embedded:\n\t"+ Path.GetFileName( chart )+
"\nReason:\n\t"+ ex.Message
);
}
}
return sfwb;
}
protected FilingSummary BuildFiling( string accessionNumberIn, string instanceNameIn, string taxonomyNameIn )
{
if( !RB_Tests.ConcurrentTestsRunning )
{
Type myType = MethodInfo.GetCurrentMethod().DeclaringType;
MethodBase method = null;
StackTrace trace = new StackTrace();
StackFrame[] frames = trace.GetFrames();
for( int f = 0; f < frames.Length; f++ )
{
method = frames[ f ].GetMethod();
if( method.DeclaringType != myType )
break;
}
if( RB_Tests.ConcurrentTestsResults.ContainsKey( method.Name ) )
{
//RB_Tests.IsConcurrentTestResult = true;
if( RB_Tests.ConcurrentTestsResults[ method.Name ] != null )
throw RB_Tests.ConcurrentTestsResults[ method.Name ];
else
return null;
}
}
string error;
FilingSummary filingSummary = null;
string folderPath = Path.Combine( baseDir, accessionNumberIn );
bool buildSuccess = BuildReports( folderPath, instanceNameIn, taxonomyNameIn, "10-Q", out filingSummary, out error );
Assert.IsTrue( buildSuccess, "Failed to generate reports: " + error );
this.myFilingSummary = filingSummary;
return filingSummary;
}
public bool BuildReports( string instanceFolder, string instanceName, string taxonomyName, string formType, out FilingSummary filingSummary, out string error )
{
Console.WriteLine( "=================================================" );
Console.WriteLine( " BUILD " + instanceFolder + "..." );
Console.WriteLine( "=================================================" );
this.processingTime = TimeSpan.Zero;
DateTime startTime = DateTime.Now;
string instance = PathCombine( instanceFolder, instanceName + ".xml" );
string taxonomy = PathCombine( instanceFolder, taxonomyName + ".xsd" );
string reportDir = PathCombine( instanceFolder, "Reports" );
string filingSummaryPath = PathCombine( reportDir, "FilingSummary.xml" );
CleanFolder( reportDir );
XRB.ReportBuilder myReportBuilder = new XRB.ReportBuilder();
myReportBuilder.ReportFormat = RB_Tests.OutputFormat;
myReportBuilder.HtmlReportFormat = RB_Tests.HtmlFormat;
myReportBuilder.RemoteFileCachePolicy = RB_Tests.CachePolicy;
if( string.IsNullOrEmpty( RB_Tests.ResourcePath ) )
RulesEngineUtils.SetBaseResourcePath( null );
else
RulesEngineUtils.SetBaseResourcePath( RB_Tests.ResourcePath );
bool status = myReportBuilder.BuildReports( instance, taxonomy, filingSummaryPath, reportDir, formType, out filingSummary, out error );
this.myFilingSummary = status ? filingSummary : null;
DateTime endTime = DateTime.Now;
this.processingTime = ( endTime - startTime );
return status;
}
/// <summary>
/// Creates a new instance of the trace wrapper with <paramref
/// name="fs"/> as the base.
/// </summary>
/// <param name="fs">The <see cref="FilingSummary"/> to use for
/// creating the trace wrapper.</param>
public FilingSummaryTraceWrapper( FilingSummary fs )
{
this.fs = fs;
this.Attributes[ "switchValue" ] = "All";
}
/// <summary>
/// <para>Performs the "building of reports" based on the <paramref name="instancePath"/> and <paramref name="taxonomy"/> provided.</para>
/// <para>As a result, <paramref name="filingSummary"/>.<see>MyReports</see> (<see cref="FilingSummary"/>) will be populated with instances of <see cref="ReportHeader"/>,</para>
/// <para>and <paramref name="filingSummary"/>.<see>MyReports</see> will be XML serialized to <paramref name="filingSummaryPath"/></para>
/// <para>If an error occurs, <paramref name="error" /> will be populated.</para>
/// </summary>
/// <param name="instancePath">The path to the instance document.</param>
/// <param name="taxonomy">The loaded and parsed taxonomy object.</param>
/// <param name="filingSummaryPath">The path where the generated <see cref="FilingSummary"/> object should be saved.</param>
/// <param name="reportDirectory">The path where the generated content is saved.</param>
/// <param name="filingSummary">The <see cref="FilingSummary"/> object to populate.</param>
/// <param name="error">The error message for any critical errors which might occur.</param>
/// <returns>True on success or false for fail.</returns>
public bool BuildReports( string instancePath, Taxonomy taxonomy,
string filingSummaryPath, string reportDirectory,
out FilingSummary filingSummary, out string error )
{
error = string.Empty;
filingSummary = null;
DateTime dtStart = DateTime.Now;
try
{
this.currentFilingSummary = new FilingSummary();
this.currentInstancePath = instancePath;
this.currentTaxonomy = taxonomy;
this.currentReportDirectory = reportDirectory;
if( !this.ValidateSettings( out error ) )
return false;
//create the reports directory so that defnref can be generated
if( !Directory.Exists( this.currentReportDirectory ) )
Directory.CreateDirectory( this.currentReportDirectory );
if( !string.IsNullOrEmpty( this.CurrencyMappingFile ) )
this.LoadCurrencies();
//set up this symbol for reuse throughout
InstanceUtils.USDCurrencySymbol = InstanceUtils.GetCurrencySymbolFromCode( InstanceUtils.USDCurrencyCode );
if( string.IsNullOrEmpty( this.preferredLanguage ) )
this.preferredLanguage = this.GetPreferredLanguage();
bool isIMBased = this.CheckIsIMBased();
//DateTime startInstance = DateTime.Now;
ArrayList errors = null;
if( !InstanceUtils.TryLoadInstanceDocument( this.currentInstancePath, out this.currentInstance, out errors ) )
{
string[] arrErrors = new string[ errors.Count ];
for( int i = 0; i < errors.Count; i++ )
{
arrErrors[ i ] = ( (ParserMessage)errors[ i ] ).Message;
}
string instanceDocErrors = string.Join( "\r\n ", arrErrors );
Regex splitPoint = new Regex( @"\S\s+at" );
arrErrors = splitPoint.Split( instanceDocErrors );
instanceDocErrors = arrErrors[ 0 ];
error = "Unable to load the instance document:\r\n " + instanceDocErrors;
return false;
}
ArrayList taxonomies = new ArrayList();
taxonomies.Add( this.currentTaxonomy );
this.currentInstance.FixPrefixInInstanceDocument( taxonomies );
//this.currentFilingSummary.InstanceLoadTime = DateTime.Now - startInstance;
//this.currentFilingSummary.FactCount = this.currentInstance.markups.Count;
this.PopulateMarkupDictionaries();
this.BuildFilingSummary();
this.FireBuildReportsProcessing();
this.currentFilingSummary.MyReports.Clear();
ArrayList topNodes = currentTaxonomy.GetNodesByPresentation( false, this.ExcludedReports );
foreach( Node topNode in topNodes )
{
InstanceReport report = null;
try
{
if( this.BuildReport( topNode, out report ) )
{
if( report.IsEmbedReport || report.HasEmbeddedReports )
{
this.roleAxes[ report.RoleURI ] = report.AxisByPresentation;
this.roleAxisDefaults[ report.RoleURI ] = report.AxisMemberDefaults;
this.roleAxisMembers[ report.RoleURI ] = report.AxisMembersByPresentation;
}
ReportHeader header = this.currentFilingSummary.AddReport( report );
this.ApplyRulesToReport( report );
report.OnRuleProcessing -= this.OnRuleProcessing;
report.OnRuleProcessing += this.OnRuleProcessing;
report.OnRuleProcessed -= this.OnRuleProcessed;
report.OnRuleProcessed += this.OnRuleProcessed;
report.UnitDictionary = this.unitDictionary;
string fullPath = Path.Combine( this.currentReportDirectory, header.XmlFileName );
report.BuildXMLDocument( fullPath, true, isIMBased, this.currentFilingSummary );
report.UnitDictionary = null;
report.OnRuleProcessing -= this.OnRuleProcessing;
report.OnRuleProcessed -= this.OnRuleProcessed;
if( header.HasEmbeddedReports )
this.reportsWithEmbeds.Add( header );
}
}
finally
{
if( report != null )
report.Dispose();
}
}
//Build Missing Elements (Uncategorized) Report before we flush the markups
InstanceReport missingReport;
if( this.BuildMissingDataReport( out missingReport ) )
{
ReportHeader uncatHeader = this.currentFilingSummary.AddReport( missingReport );
uncatHeader.XmlFileName = InstanceUtils.TOP_LEVEL_REPORT_INDICATOR+ _missingReportIndex +".xml";
}
//Free up some resources
this.currentInstance = null;
if( this.internalReports.Count == 0 )
{
this.currentTaxonomy.Close();
this.currentTaxonomy = null;
}
//clear the dictionaries after checking internalReports
this.ClearMarkupDictionaries();
#region Clean up Columns
//if the company has filed earning release, do not touch the reports.
//Based on request from SEC, do not remove any reports if the base taxonomy is the new GAAP taxonomy (2208)
if( !this.HasEarningRelease() )
{
//DO NOT REMOVE - These are used in rule processing
isGAAP2005 = this.TaxonomyIsGAAP2005();
isNewGAAP = ( !isGAAP2005 );
ProcessFlowThroughColumnsReports();
}
#endregion
#region Build All Reports
//Build book
ReportHeader r1 = new ReportHeader();
r1.LongName = _allReports;
r1.ShortName = _allReports;
r1.ReportType = ReportHeaderType.Book;
r1.IsDefault = this.currentFilingSummary.MyReports.Count == 0;
this.currentFilingSummary.AddReports( r1 );
#endregion
#region Process Embeeded Reports
bool isRuleEnabled = this.FireRuleProcessing( RulesEngineUtils.EMBED_REPORTS_RULE );
if( isRuleEnabled && this.reportsWithEmbeds.Count > 0 )
{
List<ReportHeader> embedReports = this.ProcessEmbeddedReports();
if( embedReports.Count > 0 )
{
foreach( ReportHeader embedReport in embedReports )
{
if( this.currentFilingSummary.MyReports.Contains( embedReport ) )
{
this.currentFilingSummary.MyReports.Remove( embedReport );
string reportNameToDelete = Path.Combine( this.currentReportDirectory, embedReport.XmlFileName );
if( File.Exists( reportNameToDelete ) )
File.Delete( reportNameToDelete );
}
}
}
this.FireRuleProcessed( RulesEngineUtils.EMBED_REPORTS_RULE );
}
#endregion
#region Generate Excel Workbook
isRuleEnabled = this.FireRuleProcessing( RulesEngineUtils.GENERATE_EXCEL_RULE );
if( isRuleEnabled )
{
string excelReportName = "Financial_Report";
bool processed = ExcelUtility.GenerateExcelWorkbook( this.currentFilingSummary, this.currentReportDirectory, excelReportName );
this.FireRuleProcessed( RulesEngineUtils.GENERATE_EXCEL_RULE );
if( !processed )
{
error = "Failed to generate Excel Workbook for report: " + this.currentReportDirectory + " " + excelReportName;
return false;
}
}
#endregion
#region Generate Output Formats
if( ( this.ReportFormat & ReportFormat.Html ) == ReportFormat.Html )
{
if( !this.GenerateHtmlFiles() )
{
//The error was logged to the tracer
}
}
if( ( this.ReportFormat & ReportFormat.Xml ) == ReportFormat.Xml )
{
//this format already exists
}
else
{
// this.currentFilingSummary.TraceInformation( "Information: Report Formart does not include 'Xml'. Removing references and files." );
foreach( ReportHeader rh in this.currentFilingSummary.MyReports )
{
if( !string.IsNullOrEmpty( rh.HtmlFileName ) )
{
string deleteFile = Path.Combine( this.currentReportDirectory, rh.XmlFileName );
if( File.Exists( deleteFile ) )
File.Delete( deleteFile );
rh.XmlFileName = null;
}
}
}
#endregion
#region Check and fix default report
//Recheck the default report just in case it was deleted
bool foundDefault = false;
foreach( ReportHeader header in this.currentFilingSummary.MyReports )
{
if( header.IsDefault )
{
foundDefault = true;
break;
}
}
if( !foundDefault )
{
foreach( ReportHeader header in this.currentFilingSummary.MyReports )
{
if( header.IsBalanceSheet() || header.ReportType == ReportHeaderType.Book )
{
header.IsDefault = true;
break;
}
}
}
#endregion
return true;
}
catch( Exception ex )
{
error = string.Format( "Exception thrown in BuildReports: {0}", ex.Message );
return false;
}
finally
{
this.FireBuildReportsProcessed();
this.currentFilingSummary.ProcessingTime = DateTime.Now - dtStart;
this.currentFilingSummary.SaveAsXml( filingSummaryPath );
filingSummary = this.currentFilingSummary;
this.currentFilingSummary = null;
if( this.currentTaxonomy != null )
this.currentTaxonomy.Close();
this.currentTaxonomy = null;
}
}
/// <summary>
/// <para>Performs the "building of reports" based on the <paramref name="instancePath"/> and <paramref name="taxonomyPath"/> provided.</para>
/// <para>As a result, <paramref name="filingSummary"/>.<see>MyReports</see> (<see cref="FilingSummary"/>) will be populated with instances of <see cref="ReportHeader"/>,</para>
/// <para>and <paramref name="filingSummary"/>.<see>MyReports</see> will be XML serialized to <paramref name="filingSummaryPath"/></para>
/// <para>If an error occurs, <paramref name="error" /> will be populated.</para>
/// </summary>
/// <param name="instancePath">The path to the instance document.</param>
/// <param name="taxonomyPath">The path to the taxonomy document.</param>
/// <param name="filingSummaryPath">The path where the generated <see cref="FilingSummary"/> object should be saved.</param>
/// <param name="reportDirectory">The path where the generated content is saved.</param>
/// <param name="filingSummary">The <see cref="FilingSummary"/> object to populate.</param>
/// <param name="error">The error message for any critical errors which might occur.</param>
/// <returns>True on success or false for fail.</returns>
public bool BuildReports( string instancePath,
string taxonomyPath,
string filingSummaryPath,
string reportDirectory,
out FilingSummary filingSummary, out string error )
{
error = string.Empty;
filingSummary = null;
if( !instancePath.StartsWith( "http", StringComparison.InvariantCultureIgnoreCase ) && !File.Exists( instancePath ) )
{
error = string.Format( "{0} : {1}", _instanceNotExist, instancePath );
return false;
}
if( !taxonomyPath.StartsWith( "http", StringComparison.InvariantCultureIgnoreCase ) && !File.Exists( taxonomyPath ) )
{
error = string.Format( "{0} : {1}", _taxonomyNotExist, taxonomyPath );
return false;
}
try
{
string baseHRef = string.Empty;
string curDir = Environment.CurrentDirectory;
if( instancePath.StartsWith( "http", StringComparison.InvariantCultureIgnoreCase ) )
{
int len = instancePath.Length - Path.GetFileName( instancePath ).Length;
baseHRef = instancePath.Substring( 0, len );
}
else
{
curDir = Path.GetDirectoryName( instancePath );
}
//this.XmlCatalog
int numErrors = 0;
Taxonomy taxonomy = null;
string errorMsg = string.Empty;
if( !TaxonomyUtils.TryLoadTaxonomy( taxonomyPath, this.RemoteFileCachePolicy, out taxonomy, out numErrors, out errorMsg ) )
{
error = "Error parsing the taxonomy: " + errorMsg;
return false;
}
this.currentTaxonomyPath = taxonomyPath;
bool isBuilt = this.BuildReports( instancePath, taxonomy, filingSummaryPath, reportDirectory, out filingSummary, out error );
return isBuilt;
}
catch( Exception ex )
{
error = string.Format( "Exception thrown in BuildReports: {0}", ex.Message );
}
finally
{
this.currentTaxonomyPath = null;
}
return false;
}
protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error)
{
error = null;
bool foundFiles = true;
string errorMsg = string.Empty;
string filingPath = Path.Combine(filing.ParentFolder, filing.AccessionNumber);
string filingReportsPath = Path.Combine(filingPath, "Reports");
string filingErrorFile = Path.Combine(filingReportsPath, ERROR_FILE_NAME);
string instancePath = filing.GetInstanceDocPath();
string taxonomyPath = filing.GetTaxonomyPath();
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
else if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
bool buildSucceeded = false;
if (foundFiles)
{
string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}", baseResourcePath), EventLogEntryType.Information);
RulesEngineUtils.SetBaseResourcePath(baseResourcePath);
ReportBuilder.SetSynchronizedResources(true);
FilingProcessorManager.TheMgr.WriteLogEntry("Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName(instancePath) + "'", EventLogEntryType.Information);
ReportBuilder rb = new ReportBuilder(FinancialRuleFile, ReportFormat, HtmlReportFormat);
rb.CurrencyMappingFile = CurrencyMappingFile;
rb.RemoteFileCachePolicy = RemoteFileCachePolicy;
//if( XmlCatalog != null )
// rb.XmlCatalog = XmlCatalog;
//Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder,
//this will allow us to only copy complete sets of R files to the Reports Folder.
string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(filingReportsPath))
{
Directory.Delete(filingReportsPath, true);
}
Directory.CreateDirectory(filingReportsPath);
FilingSummary summary = null;
buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error);
if (!buildSucceeded)
{
errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
if (!Directory.Exists(filingReportsPath))
{
Directory.CreateDirectory(filingReportsPath);
}
File.WriteAllText(filingErrorFile, errorMsg);
}
}
else
{
if (!Directory.Exists(filingReportsPath))
{
Directory.CreateDirectory(filingReportsPath);
}
File.WriteAllText(filingErrorFile, errorMsg);
}
try
{
string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName(filingErrorFile);
string reportErrorFile = Path.Combine(reportsFolder, errorFileName);
string reportZipFile = Path.Combine(reportsFolder, filing.AccessionNumber + ".zip");
if (File.Exists(reportErrorFile))
{
FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true);
}
if (File.Exists(reportZipFile))
{
FileUtilities.DeleteFile(new FileInfo(reportZipFile), true);
}
if (buildSucceeded)
{
string[] filePathsToZip = Directory.GetFiles(filingReportsPath);
string[] filesToZip = new string[filePathsToZip.Length];
for (int i = 0; i < filesToZip.Length; i++)
{
filesToZip[i] = Path.GetFileName(filePathsToZip[i]);
}
string zipFile = Path.Combine(filingReportsPath, filing.AccessionNumber + ".zip");
if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip))
{
File.Copy(zipFile, reportZipFile);
}
}
else
{
File.Copy(filingErrorFile, reportErrorFile);
}
if (DeleteProcessedFilings)
{
DirectoryInfo di = new DirectoryInfo(filingPath);
FileUtilities.DeleteDirectory(di, true, true);
di = new DirectoryInfo(filing.ParentFolder);
if (di.GetDirectories().Length == 0 && di.GetFiles().Length == 0)
{
FileUtilities.DeleteDirectory(di, true, true);
}
}
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
}
if (buildSucceeded)
{
FilingProcessorManager.TheMgr.IncrementCompletedCount();
}
return(buildSucceeded);
}
public void Test_29039()
{
Test_Abstract.OutputFormat = ( Test_Abstract.OutputFormat & ReportFormat.Xml ) | ReportFormat.Html;
Test_Abstract.HtmlFormat = HtmlReportFormat.Complete;
const string accessionNumber = "29039";
string reportsFolder = PathCombine( this.baseDir, accessionNumber, "Reports" );
CleanAndPrepareFolder( reportsFolder );
string resultsFolder = PathCombine( this.baseDir, accessionNumber, "Results" );
FilingSummary fs = new FilingSummary();
foreach( string from in Directory.GetFiles( resultsFolder, "R1_*.xml" ) )
{
string file = Path.GetFileName( from );
string to = PathCombine( reportsFolder, file );
FileUtilities.Copy( from, to );
ReportHeader header = new ReportHeader( file, file );
fs.MyReports.Add( header );
}
XRB.ReportBuilder rb = new XRB.ReportBuilder();
rb.ReportFormat = ReportFormat.Html;
rb.HtmlReportFormat = HtmlReportFormat.Complete;
rb.GetType().GetField( "currentFilingSummary", BindingFlags.NonPublic | BindingFlags.Instance ).SetValue( rb, fs );
rb.GetType().GetField( "currentReportDirectory", BindingFlags.NonPublic | BindingFlags.Instance ).SetValue( rb, reportsFolder );
rb.GetType().GetMethod( "GenerateHtmlFiles", BindingFlags.NonPublic | BindingFlags.Instance ).Invoke( rb, null );
foreach( string file in Directory.GetFiles( reportsFolder, "R1_*.htm" ) )
{
string html = File.ReadAllText( file );
int start = html.IndexOf( "<body" );
int end = html.IndexOf( "</body" );
end = html.IndexOf( '>', end ) + 1;
html = html.Substring( start, end - start );
html = html.Replace( "<br>", "<br />" );
XmlDocument xDoc = new XmlDocument();
xDoc.LoadXml( html );
XmlNodeList headerRows = xDoc.SelectNodes( "/body/table/tr[ th ]" );
Assert.AreEqual( 2, headerRows.Count, "The transform must generate 2 headers rows for the right display." );
}
}
public void PrependLogs( FilingSummary tmpFS )
{
}
