private bool BuildReports(string instanceFolder, string formType) { string accessionNumber = Path.GetFileName(instanceFolder); string reportDir = string.Format(@"{0}{1}Reports", instanceFolder, Path.DirectorySeparatorChar); string filingSummaryPath = string.Format(@"{0}{1}FilingSummary.xml", reportDir, Path.DirectorySeparatorChar); if (Directory.Exists(reportDir)) { //Make sure the reports dir is empty before starting. That way we can //be sure the test actually created the correct files. Directory.Delete(reportDir, true); } Directory.CreateDirectory(reportDir); if (File.Exists(filingSummaryPath)) { File.Delete(filingSummaryPath); } FilingInfo fi = new FilingInfo(); fi.AccessionNumber = accessionNumber; fi.ParentFolder = testRoot; fi.FormType = formType; string error; return this.BuildReports(fi, reportsFolder, out error); }
protected bool IsFilingComplete(FilingInfo filing) { //TODO: we need logic here and some of it depends upon the SEC and what they write //with the filing return true; }
private void ProcessNewFilings() { try { lock (filingsFolderInfo) { //Get all of the new sub folders in the filings folder DirectoryInfo[] filingDirs = filingsFolderInfo.GetDirectories(); if (filingDirs.Length == 0) { //There are no new directories, so there is nothing to process, simply return return; } myEventLog.WriteEntry("Found new filings, prepping the folders for processing."); string processingSubDir = Guid.NewGuid().ToString(); DirectoryInfo targetFolderInfo = processingFolderInfo.CreateSubdirectory(processingSubDir); List<FilingInfo> filings = new List<FilingInfo>(); foreach (DirectoryInfo subDir in filingsFolderInfo.GetDirectories()) { if (TryMoveFilingToProcessingFolder(subDir, targetFolderInfo)) { FilingInfo filing = new FilingInfo(); filing.AccessionNumber = subDir.Name; filing.ParentFolder = targetFolderInfo.FullName; filings.Add(filing); } } this.ProcessFilings(filings); } } catch (Exception ex) { myEventLog.WriteEntry("Exception while processing new filings: " + ex.Message); return; } }
protected string GetTaxonomyPath(FilingInfo filing) { string taxonomyFile = string.Empty; string instanceFile = GetInstanceDocPath(filing); if (string.IsNullOrEmpty(instanceFile)) { return string.Empty; } XmlTextReader xReader = new XmlTextReader(instanceFile); try { xReader.MoveToContent(); if (xReader.LocalName == "xbrl") { while (xReader.MoveToNextAttribute()) { if (xReader.Value == "http://ici.org/rr/2006") { break; } } xReader.Read(); while (xReader.LocalName != "schemaRef") { xReader.Read(); } if (xReader.LocalName == "schemaRef") { while (xReader.MoveToNextAttribute()) { if (xReader.NodeType == XmlNodeType.Attribute && xReader.Name == "xlink:href") { taxonomyFile = Path.GetFileName(xReader.Value); break; } } } } taxonomyFile = Path.Combine(Path.GetDirectoryName(instanceFile), taxonomyFile); } catch (Exception ex) { myEventLog.WriteEntry("Exception in GetTaxonomy Path: " + ex.Message, EventLogEntryType.Warning); taxonomyFile = string.Empty; } return taxonomyFile; }
protected string GetInstanceDocPath(FilingInfo filing) { string filingDir = string.Format("{0}{1}{2}", filing.ParentFolder, Path.DirectorySeparatorChar, filing.AccessionNumber); string[] files = Directory.GetFiles(filingDir, "*.xml"); Regex instanceRegex = new Regex(@"\d{8}\.xml$"); string retValue = string.Empty; foreach (string file in files) { Match regexMatch = instanceRegex.Match(file); if (regexMatch != null && regexMatch.Success) { //There should only be one file that matches the regex in each filing, so //once we find it break and return retValue = file; break; } } return retValue; }
public bool BuildReports(FilingInfo filing) { string instancePath = GetInstanceDocPath(filing); if (string.IsNullOrEmpty(instancePath) || !File.Exists(instancePath)) { myEventLog.WriteEntry("Can not find instance document for filing: " + filing.AccessionNumber); return false; } string taxonomyPath = GetTaxonomyPath(filing); if (string.IsNullOrEmpty(taxonomyPath) || !File.Exists(taxonomyPath)) { myEventLog.WriteEntry("Can not find taxonomy file for filing: " + filing.AccessionNumber); return false; } ReportBuilder rb = new ReportBuilder(); string reportPath = string.Format("{0}{1}{2}", reportsFolder, Path.DirectorySeparatorChar, filing.AccessionNumber); string filingFile = string.Format("{0}{1}{2}", reportPath, Path.DirectorySeparatorChar, FilingSummary._FilingSummaryXmlName); //Make sure there is a clean folder where the reports will be written to. if (Directory.Exists(reportPath)) { Directory.Delete(reportPath, true); } Directory.CreateDirectory(reportPath); //TODO: get this info from the manifest once we have it //rb.PeriodEnding = filing.period_ending; //rb.FilingDate = filing.filing_date; //rb.TickerSymbol = filing.ticker_symbol; //rb.CompanyName = filing.company_name; //rb.AccessionNumber = filing.accession_number; //rb.FiscalYearEnd = filing.fiscal_year_end; string error = null; FilingSummary summary = null; if (!rb.BuildReports(instancePath, taxonomyPath, filingFile, reportPath, filing.FormType, out summary, out error)) { myEventLog.WriteEntry("build reports failed! " + error, EventLogEntryType.Error); return false; } //Increment the number of filings that were successfully process. This needs to be done //using Interlocked because other worker threads could be accessing the property as well Interlocked.Increment(ref completedFilings); return true; }
protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error) { error = null; bool foundFiles = true; string errorMsg = string.Empty; string filingPath = Path.Combine( filing.ParentFolder, filing.AccessionNumber ); string filingReportsPath = Path.Combine( filingPath, "Reports" ); string filingErrorFile = Path.Combine( filingReportsPath, ERROR_FILE_NAME ); string instancePath = filing.GetInstanceDocPath(); string taxonomyPath = filing.GetTaxonomyPath(); if (string.IsNullOrEmpty(instancePath) || !File.Exists(instancePath)) { errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber); FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); foundFiles = false; } else if (string.IsNullOrEmpty(taxonomyPath) || !File.Exists(taxonomyPath)) { errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber); FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); foundFiles = false; } bool buildSucceeded = false; if (foundFiles) { string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}",baseResourcePath), EventLogEntryType.Information); RulesEngineUtils.SetBaseResourcePath(baseResourcePath); ReportBuilder.SetSynchronizedResources( true ); FilingProcessorManager.TheMgr.WriteLogEntry( "Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName( instancePath ) + "'", EventLogEntryType.Information ); ReportBuilder rb = new ReportBuilder( FinancialRuleFile, ReportFormat, HtmlReportFormat ); rb.CurrencyMappingFile = CurrencyMappingFile; rb.RemoteFileCachePolicy = RemoteFileCachePolicy; //if( XmlCatalog != null ) // rb.XmlCatalog = XmlCatalog; //Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder, //this will allow us to only copy complete sets of R files to the Reports Folder. string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName); //Make sure there is a clean folder where the reports will be written to. if (Directory.Exists(filingReportsPath)) Directory.Delete(filingReportsPath, true); Directory.CreateDirectory(filingReportsPath); FilingSummary summary = null; buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error); if (!buildSucceeded) { errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error); FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); if (!Directory.Exists(filingReportsPath)) Directory.CreateDirectory(filingReportsPath); File.WriteAllText(filingErrorFile, errorMsg); } } else { if (!Directory.Exists(filingReportsPath)) Directory.CreateDirectory(filingReportsPath); File.WriteAllText(filingErrorFile, errorMsg); } try { string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName( filingErrorFile ); string reportErrorFile = Path.Combine( reportsFolder, errorFileName ); string reportZipFile = Path.Combine( reportsFolder, filing.AccessionNumber + ".zip" ); if (File.Exists(reportErrorFile)) FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true); if (File.Exists(reportZipFile)) FileUtilities.DeleteFile(new FileInfo(reportZipFile), true); if (buildSucceeded) { string[] filePathsToZip = Directory.GetFiles(filingReportsPath); string[] filesToZip = new string[filePathsToZip.Length]; for (int i = 0; i < filesToZip.Length; i++) { filesToZip[i] = Path.GetFileName(filePathsToZip[i]); } string zipFile = Path.Combine( filingReportsPath, filing.AccessionNumber +".zip"); if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip)) File.Copy(zipFile, reportZipFile); } else { File.Copy(filingErrorFile, reportErrorFile); } if( DeleteProcessedFilings ) { DirectoryInfo di = new DirectoryInfo( filingPath ); FileUtilities.DeleteDirectory( di, true, true ); di = new DirectoryInfo( filing.ParentFolder ); if( di.GetDirectories().Length == 0 && di.GetFiles().Length == 0 ) FileUtilities.DeleteDirectory( di, true, true ); } } catch(Exception ex) { System.Diagnostics.Debug.WriteLine(ex.Message); } if (buildSucceeded) FilingProcessorManager.TheMgr.IncrementCompletedCount(); return buildSucceeded; }
public bool TryMoveFilingsToProcessingFolder(out List<FilingInfo> filings) { filings = new List<FilingInfo>(); FileInfo[] zipFiles = FilingProcessorManager.TheMgr.FilingsFolderInfo.GetFiles("*.zip", SearchOption.TopDirectoryOnly); Array.Sort( zipFiles, ( left, right ) => DateTime.Compare( left.CreationTime, right.CreationTime ) ); Guid batchId = Guid.NewGuid(); DirectoryInfo processingBatchFolder = FilingProcessorManager.TheMgr.ProcessingFolderInfo.CreateSubdirectory(batchId.ToString()); foreach (FileInfo zipFile in zipFiles) { string filingName = Path.GetFileNameWithoutExtension(zipFile.Name); if( string.IsNullOrEmpty( filingName ) ) { try { string errorFile = Path.Combine( FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, "UNKNOWN.txt" ); string errorMessage = "Cannot process a filing zip which has no base name."; FilingProcessorManager.TheMgr.WriteLogEntry( errorMessage, EventLogEntryType.Error ); File.WriteAllText( errorFile, errorMessage ); zipFile.Delete(); } catch { } continue; } DirectoryInfo filingProcessingFolder = processingBatchFolder.CreateSubdirectory(filingName); string[] unzippedFiles; string filingZipFile = string.Format("{0}{1}{2}", filingProcessingFolder.FullName, Path.DirectorySeparatorChar,Path.GetFileName(zipFile.Name)); if (filingRetryCount.ContainsKey(filingName) && filingRetryCount[filingName] >= maxUnzipAttempts) { filingRetryCount.Remove(filingName); //We have failed to extract the filing multiple times, stop trying to make it work, //remove the zip file from the filing folder, log an error in the reports folder, //and reomve the entry from the dictionary string errorFilename = filingName +"_"+ ERROR_FILE_NAME; string errorFile = Path.Combine( FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, filingName ); string errorMsg = string.Format("Cannot extract files for filing {0}. The max number of retries has been reached, removing the zip file from the Filings folder. ", Path.GetFileNameWithoutExtension(zipFile.FullName)); if (File.Exists(errorFile)) { FileUtilities.DeleteFile(new FileInfo(errorFile), true); } FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); File.WriteAllText(errorFile, errorMsg); zipFile.CopyTo(filingZipFile); FileUtilities.DeleteFile(zipFile, true); } else { string zipError; if (ZipUtilities.TryUnzipAndUncompressFiles(zipFile.FullName, filingProcessingFolder.FullName, out unzippedFiles, out zipError)) { FilingInfo filing = new FilingInfo(); filing.AccessionNumber = filingProcessingFolder.Name; filing.ParentFolder = processingBatchFolder.FullName; filings.Add(filing); zipFile.CopyTo(filingZipFile, true); FileUtilities.DeleteFile(zipFile, true); //SEC0145 - Only queue one filing at a time break; } else { //Delete the folder from the processing batch folder sense it doesn't contain a valid filing FileUtilities.DeleteDirectory(filingProcessingFolder, true, true); if (!filingRetryCount.ContainsKey(filingName)) { filingRetryCount.Add(filingName, 0); } filingRetryCount[filingName]++; FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Can not extract files for filing {0}. The zip file may not be complete. ", Path.GetFileNameWithoutExtension(zipFile.FullName)), EventLogEntryType.Warning); } } } if (processingBatchFolder.GetDirectories().Length == 0) { //There were not any valid filings in this batch, remove the batch folder FileUtilities.DeleteDirectory(processingBatchFolder, true, true); } return true; }