private bool BuildReports(string instanceFolder, string formType)
{
string accessionNumber = Path.GetFileName(instanceFolder);
string reportDir = string.Format(@"{0}{1}Reports", instanceFolder, Path.DirectorySeparatorChar);
string filingSummaryPath = string.Format(@"{0}{1}FilingSummary.xml", reportDir, Path.DirectorySeparatorChar);
if (Directory.Exists(reportDir))
{
//Make sure the reports dir is empty before starting. That way we can
//be sure the test actually created the correct files.
Directory.Delete(reportDir, true);
}
Directory.CreateDirectory(reportDir);
if (File.Exists(filingSummaryPath))
{
File.Delete(filingSummaryPath);
}
FilingInfo fi = new FilingInfo();
fi.AccessionNumber = accessionNumber;
fi.ParentFolder = testRoot;
fi.FormType = formType;
string error;
return this.BuildReports(fi, reportsFolder, out error);
}
protected bool IsFilingComplete(FilingInfo filing)
{
//TODO: we need logic here and some of it depends upon the SEC and what they write
//with the filing
return true;
}
private void ProcessNewFilings()
{
try
{
lock (filingsFolderInfo)
{
//Get all of the new sub folders in the filings folder
DirectoryInfo[] filingDirs = filingsFolderInfo.GetDirectories();
if (filingDirs.Length == 0)
{
//There are no new directories, so there is nothing to process, simply return
return;
}
myEventLog.WriteEntry("Found new filings, prepping the folders for processing.");
string processingSubDir = Guid.NewGuid().ToString();
DirectoryInfo targetFolderInfo = processingFolderInfo.CreateSubdirectory(processingSubDir);
List<FilingInfo> filings = new List<FilingInfo>();
foreach (DirectoryInfo subDir in filingsFolderInfo.GetDirectories())
{
if (TryMoveFilingToProcessingFolder(subDir, targetFolderInfo))
{
FilingInfo filing = new FilingInfo();
filing.AccessionNumber = subDir.Name;
filing.ParentFolder = targetFolderInfo.FullName;
filings.Add(filing);
}
}
this.ProcessFilings(filings);
}
}
catch (Exception ex)
{
myEventLog.WriteEntry("Exception while processing new filings: " + ex.Message);
return;
}
}
protected string GetTaxonomyPath(FilingInfo filing)
{
string taxonomyFile = string.Empty;
string instanceFile = GetInstanceDocPath(filing);
if (string.IsNullOrEmpty(instanceFile))
{
return string.Empty;
}
XmlTextReader xReader = new XmlTextReader(instanceFile);
try
{
xReader.MoveToContent();
if (xReader.LocalName == "xbrl")
{
while (xReader.MoveToNextAttribute())
{
if (xReader.Value == "http://ici.org/rr/2006")
{
break;
}
}
xReader.Read();
while (xReader.LocalName != "schemaRef")
{
xReader.Read();
}
if (xReader.LocalName == "schemaRef")
{
while (xReader.MoveToNextAttribute())
{
if (xReader.NodeType == XmlNodeType.Attribute &&
xReader.Name == "xlink:href")
{
taxonomyFile = Path.GetFileName(xReader.Value);
break;
}
}
}
}
taxonomyFile = Path.Combine(Path.GetDirectoryName(instanceFile), taxonomyFile);
}
catch (Exception ex)
{
myEventLog.WriteEntry("Exception in GetTaxonomy Path: " + ex.Message, EventLogEntryType.Warning);
taxonomyFile = string.Empty;
}
return taxonomyFile;
}
protected string GetInstanceDocPath(FilingInfo filing)
{
string filingDir = string.Format("{0}{1}{2}", filing.ParentFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
string[] files = Directory.GetFiles(filingDir, "*.xml");
Regex instanceRegex = new Regex(@"\d{8}\.xml$");
string retValue = string.Empty;
foreach (string file in files)
{
Match regexMatch = instanceRegex.Match(file);
if (regexMatch != null && regexMatch.Success)
{
//There should only be one file that matches the regex in each filing, so
//once we find it break and return
retValue = file;
break;
}
}
return retValue;
}
public bool BuildReports(FilingInfo filing)
{
string instancePath = GetInstanceDocPath(filing);
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
myEventLog.WriteEntry("Can not find instance document for filing: " + filing.AccessionNumber);
return false;
}
string taxonomyPath = GetTaxonomyPath(filing);
if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
myEventLog.WriteEntry("Can not find taxonomy file for filing: " + filing.AccessionNumber);
return false;
}
ReportBuilder rb = new ReportBuilder();
string reportPath = string.Format("{0}{1}{2}", reportsFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
string filingFile = string.Format("{0}{1}{2}", reportPath, Path.DirectorySeparatorChar, FilingSummary._FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(reportPath))
{
Directory.Delete(reportPath, true);
}
Directory.CreateDirectory(reportPath);
//TODO: get this info from the manifest once we have it
//rb.PeriodEnding = filing.period_ending;
//rb.FilingDate = filing.filing_date;
//rb.TickerSymbol = filing.ticker_symbol;
//rb.CompanyName = filing.company_name;
//rb.AccessionNumber = filing.accession_number;
//rb.FiscalYearEnd = filing.fiscal_year_end;
string error = null;
FilingSummary summary = null;
if (!rb.BuildReports(instancePath, taxonomyPath, filingFile, reportPath, filing.FormType, out summary, out error))
{
myEventLog.WriteEntry("build reports failed! " + error, EventLogEntryType.Error);
return false;
}
//Increment the number of filings that were successfully process. This needs to be done
//using Interlocked because other worker threads could be accessing the property as well
Interlocked.Increment(ref completedFilings);
return true;
}
protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error)
{
error = null;
bool foundFiles = true;
string errorMsg = string.Empty;
string filingPath = Path.Combine( filing.ParentFolder, filing.AccessionNumber );
string filingReportsPath = Path.Combine( filingPath, "Reports" );
string filingErrorFile = Path.Combine( filingReportsPath, ERROR_FILE_NAME );
string instancePath = filing.GetInstanceDocPath();
string taxonomyPath = filing.GetTaxonomyPath();
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
else if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
bool buildSucceeded = false;
if (foundFiles)
{
string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}",baseResourcePath), EventLogEntryType.Information);
RulesEngineUtils.SetBaseResourcePath(baseResourcePath);
ReportBuilder.SetSynchronizedResources( true );
FilingProcessorManager.TheMgr.WriteLogEntry( "Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName( instancePath ) + "'", EventLogEntryType.Information );
ReportBuilder rb = new ReportBuilder( FinancialRuleFile, ReportFormat, HtmlReportFormat );
rb.CurrencyMappingFile = CurrencyMappingFile;
rb.RemoteFileCachePolicy = RemoteFileCachePolicy;
//if( XmlCatalog != null )
// rb.XmlCatalog = XmlCatalog;
//Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder,
//this will allow us to only copy complete sets of R files to the Reports Folder.
string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(filingReportsPath))
Directory.Delete(filingReportsPath, true);
Directory.CreateDirectory(filingReportsPath);
FilingSummary summary = null;
buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error);
if (!buildSucceeded)
{
errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
if (!Directory.Exists(filingReportsPath))
Directory.CreateDirectory(filingReportsPath);
File.WriteAllText(filingErrorFile, errorMsg);
}
}
else
{
if (!Directory.Exists(filingReportsPath))
Directory.CreateDirectory(filingReportsPath);
File.WriteAllText(filingErrorFile, errorMsg);
}
try
{
string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName( filingErrorFile );
string reportErrorFile = Path.Combine( reportsFolder, errorFileName );
string reportZipFile = Path.Combine( reportsFolder, filing.AccessionNumber + ".zip" );
if (File.Exists(reportErrorFile))
FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true);
if (File.Exists(reportZipFile))
FileUtilities.DeleteFile(new FileInfo(reportZipFile), true);
if (buildSucceeded)
{
string[] filePathsToZip = Directory.GetFiles(filingReportsPath);
string[] filesToZip = new string[filePathsToZip.Length];
for (int i = 0; i < filesToZip.Length; i++)
{
filesToZip[i] = Path.GetFileName(filePathsToZip[i]);
}
string zipFile = Path.Combine( filingReportsPath, filing.AccessionNumber +".zip");
if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip))
File.Copy(zipFile, reportZipFile);
}
else
{
File.Copy(filingErrorFile, reportErrorFile);
}
if( DeleteProcessedFilings )
{
DirectoryInfo di = new DirectoryInfo( filingPath );
FileUtilities.DeleteDirectory( di, true, true );
di = new DirectoryInfo( filing.ParentFolder );
if( di.GetDirectories().Length == 0 && di.GetFiles().Length == 0 )
FileUtilities.DeleteDirectory( di, true, true );
}
}
catch(Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
}
if (buildSucceeded)
FilingProcessorManager.TheMgr.IncrementCompletedCount();
return buildSucceeded;
}
public bool TryMoveFilingsToProcessingFolder(out List<FilingInfo> filings)
{
filings = new List<FilingInfo>();
FileInfo[] zipFiles = FilingProcessorManager.TheMgr.FilingsFolderInfo.GetFiles("*.zip", SearchOption.TopDirectoryOnly);
Array.Sort( zipFiles, ( left, right ) => DateTime.Compare( left.CreationTime, right.CreationTime ) );
Guid batchId = Guid.NewGuid();
DirectoryInfo processingBatchFolder = FilingProcessorManager.TheMgr.ProcessingFolderInfo.CreateSubdirectory(batchId.ToString());
foreach (FileInfo zipFile in zipFiles)
{
string filingName = Path.GetFileNameWithoutExtension(zipFile.Name);
if( string.IsNullOrEmpty( filingName ) )
{
try
{
string errorFile = Path.Combine( FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, "UNKNOWN.txt" );
string errorMessage = "Cannot process a filing zip which has no base name.";
FilingProcessorManager.TheMgr.WriteLogEntry( errorMessage, EventLogEntryType.Error );
File.WriteAllText( errorFile, errorMessage );
zipFile.Delete();
} catch { }
continue;
}
DirectoryInfo filingProcessingFolder = processingBatchFolder.CreateSubdirectory(filingName);
string[] unzippedFiles;
string filingZipFile = string.Format("{0}{1}{2}", filingProcessingFolder.FullName, Path.DirectorySeparatorChar,Path.GetFileName(zipFile.Name));
if (filingRetryCount.ContainsKey(filingName) &&
filingRetryCount[filingName] >= maxUnzipAttempts)
{
filingRetryCount.Remove(filingName);
//We have failed to extract the filing multiple times, stop trying to make it work,
//remove the zip file from the filing folder, log an error in the reports folder,
//and reomve the entry from the dictionary
string errorFilename = filingName +"_"+ ERROR_FILE_NAME;
string errorFile = Path.Combine( FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, filingName );
string errorMsg = string.Format("Cannot extract files for filing {0}. The max number of retries has been reached, removing the zip file from the Filings folder. ", Path.GetFileNameWithoutExtension(zipFile.FullName));
if (File.Exists(errorFile))
{
FileUtilities.DeleteFile(new FileInfo(errorFile), true);
}
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
File.WriteAllText(errorFile, errorMsg);
zipFile.CopyTo(filingZipFile);
FileUtilities.DeleteFile(zipFile, true);
}
else
{
string zipError;
if (ZipUtilities.TryUnzipAndUncompressFiles(zipFile.FullName, filingProcessingFolder.FullName, out unzippedFiles, out zipError))
{
FilingInfo filing = new FilingInfo();
filing.AccessionNumber = filingProcessingFolder.Name;
filing.ParentFolder = processingBatchFolder.FullName;
filings.Add(filing);
zipFile.CopyTo(filingZipFile, true);
FileUtilities.DeleteFile(zipFile, true);
//SEC0145 - Only queue one filing at a time
break;
}
else
{
//Delete the folder from the processing batch folder sense it doesn't contain a valid filing
FileUtilities.DeleteDirectory(filingProcessingFolder, true, true);
if (!filingRetryCount.ContainsKey(filingName))
{
filingRetryCount.Add(filingName, 0);
}
filingRetryCount[filingName]++;
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Can not extract files for filing {0}. The zip file may not be complete. ", Path.GetFileNameWithoutExtension(zipFile.FullName)), EventLogEntryType.Warning);
}
}
}
if (processingBatchFolder.GetDirectories().Length == 0)
{
//There were not any valid filings in this batch, remove the batch folder
FileUtilities.DeleteDirectory(processingBatchFolder, true, true);
}
return true;
}