private bool BuildReports(string instanceFolder, string formType) { string accessionNumber = Path.GetFileName(instanceFolder); string reportDir = string.Format(@"{0}{1}Reports", instanceFolder, Path.DirectorySeparatorChar); string filingSummaryPath = string.Format(@"{0}{1}FilingSummary.xml", reportDir, Path.DirectorySeparatorChar); if (Directory.Exists(reportDir)) { //Make sure the reports dir is empty before starting. That way we can //be sure the test actually created the correct files. Directory.Delete(reportDir, true); } Directory.CreateDirectory(reportDir); if (File.Exists(filingSummaryPath)) { File.Delete(filingSummaryPath); } FilingInfo fi = new FilingInfo(); fi.AccessionNumber = accessionNumber; fi.ParentFolder = testRoot; fi.FormType = formType; string error; return(this.BuildReports(fi, reportsFolder, out error)); }
protected string GetTaxonomyPath(FilingInfo filing) { string taxonomyFile = string.Empty; string instanceFile = GetInstanceDocPath(filing); if (string.IsNullOrEmpty(instanceFile)) { return(string.Empty); } XmlTextReader xReader = new XmlTextReader(instanceFile); try { xReader.MoveToContent(); if (xReader.LocalName == "xbrl") { while (xReader.MoveToNextAttribute()) { if (xReader.Value == "http://ici.org/rr/2006") { break; } } xReader.Read(); while (xReader.LocalName != "schemaRef") { xReader.Read(); } if (xReader.LocalName == "schemaRef") { while (xReader.MoveToNextAttribute()) { if (xReader.NodeType == XmlNodeType.Attribute && xReader.Name == "xlink:href") { taxonomyFile = Path.GetFileName(xReader.Value); break; } } } } taxonomyFile = Path.Combine(Path.GetDirectoryName(instanceFile), taxonomyFile); } catch (Exception ex) { myEventLog.WriteEntry("Exception in GetTaxonomy Path: " + ex.Message, EventLogEntryType.Warning); taxonomyFile = string.Empty; } return(taxonomyFile); }
public void BuildReportsCallBack(Object objFiling) { FilingInfo filing = objFiling as FilingInfo; if (filing == null) { return; } BuildReports(filing); }
public bool BuildReports(FilingInfo filing) { string instancePath = GetInstanceDocPath(filing); if (string.IsNullOrEmpty(instancePath) || !File.Exists(instancePath)) { myEventLog.WriteEntry("Can not find instance document for filing: " + filing.AccessionNumber); return(false); } string taxonomyPath = GetTaxonomyPath(filing); if (string.IsNullOrEmpty(taxonomyPath) || !File.Exists(taxonomyPath)) { myEventLog.WriteEntry("Can not find taxonomy file for filing: " + filing.AccessionNumber); return(false); } ReportBuilder rb = new ReportBuilder(); string reportPath = string.Format("{0}{1}{2}", reportsFolder, Path.DirectorySeparatorChar, filing.AccessionNumber); string filingFile = string.Format("{0}{1}{2}", reportPath, Path.DirectorySeparatorChar, FilingSummary._FilingSummaryXmlName); //Make sure there is a clean folder where the reports will be written to. if (Directory.Exists(reportPath)) { Directory.Delete(reportPath, true); } Directory.CreateDirectory(reportPath); //TODO: get this info from the manifest once we have it //rb.PeriodEnding = filing.period_ending; //rb.FilingDate = filing.filing_date; //rb.TickerSymbol = filing.ticker_symbol; //rb.CompanyName = filing.company_name; //rb.AccessionNumber = filing.accession_number; //rb.FiscalYearEnd = filing.fiscal_year_end; string error = null; FilingSummary summary = null; if (!rb.BuildReports(instancePath, taxonomyPath, filingFile, reportPath, filing.FormType, out summary, out error)) { myEventLog.WriteEntry("build reports failed! " + error, EventLogEntryType.Error); return(false); } //Increment the number of filings that were successfully process. This needs to be done //using Interlocked because other worker threads could be accessing the property as well Interlocked.Increment(ref completedFilings); return(true); }
public void ProcessFilingCallback(Object objFiling) { FilingInfo filing = objFiling as FilingInfo; if (filing == null) { return; } string error; BuildReports(filing, FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, out error); }
private void ProcessNewFilings() { try { lock (filingsFolderInfo) { //Get all of the new sub folders in the filings folder DirectoryInfo[] filingDirs = filingsFolderInfo.GetDirectories(); if (filingDirs.Length == 0) { //There are no new directories, so there is nothing to process, simply return return; } myEventLog.WriteEntry("Found new filings, prepping the folders for processing."); string processingSubDir = Guid.NewGuid().ToString(); DirectoryInfo targetFolderInfo = processingFolderInfo.CreateSubdirectory(processingSubDir); List <FilingInfo> filings = new List <FilingInfo>(); foreach (DirectoryInfo subDir in filingsFolderInfo.GetDirectories()) { if (TryMoveFilingToProcessingFolder(subDir, targetFolderInfo)) { FilingInfo filing = new FilingInfo(); filing.AccessionNumber = subDir.Name; filing.ParentFolder = targetFolderInfo.FullName; filings.Add(filing); } } this.ProcessFilings(filings); } } catch (Exception ex) { myEventLog.WriteEntry("Exception while processing new filings: " + ex.Message); return; } }
protected string GetInstanceDocPath(FilingInfo filing) { string filingDir = string.Format("{0}{1}{2}", filing.ParentFolder, Path.DirectorySeparatorChar, filing.AccessionNumber); string[] files = Directory.GetFiles(filingDir, "*.xml"); Regex instanceRegex = new Regex(@"\d{8}\.xml$"); string retValue = string.Empty; foreach (string file in files) { Match regexMatch = instanceRegex.Match(file); if (regexMatch != null && regexMatch.Success) { //There should only be one file that matches the regex in each filing, so //once we find it break and return retValue = file; break; } } return(retValue); }
public void TestGetTaxonomyPath() { string[][] testFilings = new string[10][]; testFilings[0] = new string[] { "0000008670-08-000046", "adp-20080331.xsd" }; testFilings[1] = new string[] { "0000040545-08-000037", "ge-20080630.xsd" }; testFilings[2] = new string[] { "0000796343-08-000005", "adbe-20080530.xsd" }; testFilings[3] = new string[] { "0000893220-08-002188", "dd-20080630.xsd" }; testFilings[4] = new string[] { "0000950133-08-003012", "nihd-20080331.xsd" }; testFilings[5] = new string[] { "0000950135-08-002601", "gis-20080224.xsd" }; testFilings[6] = new string[] { "0000950135-08-003083", "dd-20080331.xsd" }; testFilings[7] = new string[] { "0000950135-08-004978", "trow-20071231.xsd" }; testFilings[8] = new string[] { "0000950137-08-010830", "gis-20080525.xsd" }; testFilings[9] = new string[] { "0000950137-08-012088", "gis-20080824.xsd" }; List <string> errors = new List <string>(); foreach (string[] filing in testFilings) { FilingInfo fi = new FilingInfo(); fi.AccessionNumber = filing[0]; fi.ParentFolder = testRoot; string taxonomyPath = fi.GetTaxonomyPath(); string expectedPath = string.Format("{0}{1}{2}{1}{3}", testRoot, System.IO.Path.DirectorySeparatorChar, filing[0], filing[1]); if (!expectedPath.Equals(taxonomyPath, StringComparison.CurrentCultureIgnoreCase)) { errors.Add("Paths do not match. Expected: '" + expectedPath + "'. Actual: '" + taxonomyPath + "'"); } } if (errors.Count > 0) { string strErrors = string.Join(Environment.NewLine + "\t", errors.ToArray()); Assert.Fail("The following errors were found: " + Environment.NewLine + strErrors); } }
public void TestGetInstanceDocPath() { string[][] testFilings = new string[10][]; testFilings[0] = new string[] { "0000008670-08-000046", "adp-20080331.xml" }; testFilings[1] = new string[] { "0000040545-08-000037", "ge-20080630.xml" }; testFilings[2] = new string[] { "0000796343-08-000005", "adbe-20080616.xml" }; testFilings[3] = new string[] { "0000893220-08-002188", "dd-20080630.xml" }; testFilings[4] = new string[] { "0000950133-08-003012", "nihd-20080331.xml" }; testFilings[5] = new string[] { "0000950135-08-002601", "gis-20080224.xml" }; testFilings[6] = new string[] { "0000950135-08-003083", "dd-20080331.xml" }; testFilings[7] = new string[] { "0000950135-08-004978", "trow-20071231.xml" }; testFilings[8] = new string[] { "0000950137-08-010830", "gis-20080525.xml" }; testFilings[9] = new string[] { "0000950137-08-012088", "gis-20080824.xml" }; List<string> errors = new List<string>(); foreach (string[] filing in testFilings) { FilingInfo fi = new FilingInfo(); fi.AccessionNumber = filing[0]; fi.ParentFolder = testRoot; string instanceDocPath = fi.GetInstanceDocPath(); string expectedPath = string.Format("{0}{1}{2}{1}{3}", testRoot, System.IO.Path.DirectorySeparatorChar, filing[0], filing[1]); if (!expectedPath.Equals(instanceDocPath, StringComparison.CurrentCultureIgnoreCase)) { errors.Add("Paths do not match. Expected: '" + expectedPath + "'. Actual: '" + instanceDocPath + "'"); } } if (errors.Count > 0) { string strErrors = string.Join(Environment.NewLine + "\t", errors.ToArray()); Assert.Fail("The following errors were found: " + Environment.NewLine + "\t" + strErrors); } }
protected bool IsFilingComplete(FilingInfo filing) { //TODO: we need logic here and some of it depends upon the SEC and what they write //with the filing return(true); }
protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error) { error = null; bool foundFiles = true; string errorMsg = string.Empty; string filingPath = Path.Combine(filing.ParentFolder, filing.AccessionNumber); string filingReportsPath = Path.Combine(filingPath, "Reports"); string filingErrorFile = Path.Combine(filingReportsPath, ERROR_FILE_NAME); string instancePath = filing.GetInstanceDocPath(); string taxonomyPath = filing.GetTaxonomyPath(); if (string.IsNullOrEmpty(instancePath) || !File.Exists(instancePath)) { errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber); FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); foundFiles = false; } else if (string.IsNullOrEmpty(taxonomyPath) || !File.Exists(taxonomyPath)) { errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber); FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); foundFiles = false; } bool buildSucceeded = false; if (foundFiles) { string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}", baseResourcePath), EventLogEntryType.Information); RulesEngineUtils.SetBaseResourcePath(baseResourcePath); ReportBuilder.SetSynchronizedResources(true); FilingProcessorManager.TheMgr.WriteLogEntry("Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName(instancePath) + "'", EventLogEntryType.Information); ReportBuilder rb = new ReportBuilder(FinancialRuleFile, ReportFormat, HtmlReportFormat); rb.CurrencyMappingFile = CurrencyMappingFile; rb.RemoteFileCachePolicy = RemoteFileCachePolicy; //if( XmlCatalog != null ) // rb.XmlCatalog = XmlCatalog; //Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder, //this will allow us to only copy complete sets of R files to the Reports Folder. string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName); //Make sure there is a clean folder where the reports will be written to. if (Directory.Exists(filingReportsPath)) { Directory.Delete(filingReportsPath, true); } Directory.CreateDirectory(filingReportsPath); FilingSummary summary = null; buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error); if (!buildSucceeded) { errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error); FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); if (!Directory.Exists(filingReportsPath)) { Directory.CreateDirectory(filingReportsPath); } File.WriteAllText(filingErrorFile, errorMsg); } } else { if (!Directory.Exists(filingReportsPath)) { Directory.CreateDirectory(filingReportsPath); } File.WriteAllText(filingErrorFile, errorMsg); } try { string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName(filingErrorFile); string reportErrorFile = Path.Combine(reportsFolder, errorFileName); string reportZipFile = Path.Combine(reportsFolder, filing.AccessionNumber + ".zip"); if (File.Exists(reportErrorFile)) { FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true); } if (File.Exists(reportZipFile)) { FileUtilities.DeleteFile(new FileInfo(reportZipFile), true); } if (buildSucceeded) { string[] filePathsToZip = Directory.GetFiles(filingReportsPath); string[] filesToZip = new string[filePathsToZip.Length]; for (int i = 0; i < filesToZip.Length; i++) { filesToZip[i] = Path.GetFileName(filePathsToZip[i]); } string zipFile = Path.Combine(filingReportsPath, filing.AccessionNumber + ".zip"); if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip)) { File.Copy(zipFile, reportZipFile); } } else { File.Copy(filingErrorFile, reportErrorFile); } if (DeleteProcessedFilings) { DirectoryInfo di = new DirectoryInfo(filingPath); FileUtilities.DeleteDirectory(di, true, true); di = new DirectoryInfo(filing.ParentFolder); if (di.GetDirectories().Length == 0 && di.GetFiles().Length == 0) { FileUtilities.DeleteDirectory(di, true, true); } } } catch (Exception ex) { System.Diagnostics.Debug.WriteLine(ex.Message); } if (buildSucceeded) { FilingProcessorManager.TheMgr.IncrementCompletedCount(); } return(buildSucceeded); }
public bool TryMoveFilingsToProcessingFolder(out List <FilingInfo> filings) { filings = new List <FilingInfo>(); FileInfo[] zipFiles = FilingProcessorManager.TheMgr.FilingsFolderInfo.GetFiles("*.zip", SearchOption.TopDirectoryOnly); Array.Sort(zipFiles, (left, right) => DateTime.Compare(left.CreationTime, right.CreationTime)); Guid batchId = Guid.NewGuid(); DirectoryInfo processingBatchFolder = FilingProcessorManager.TheMgr.ProcessingFolderInfo.CreateSubdirectory(batchId.ToString()); foreach (FileInfo zipFile in zipFiles) { string filingName = Path.GetFileNameWithoutExtension(zipFile.Name); if (string.IsNullOrEmpty(filingName)) { try { string errorFile = Path.Combine(FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, "UNKNOWN.txt"); string errorMessage = "Cannot process a filing zip which has no base name."; FilingProcessorManager.TheMgr.WriteLogEntry(errorMessage, EventLogEntryType.Error); File.WriteAllText(errorFile, errorMessage); zipFile.Delete(); } catch { } continue; } DirectoryInfo filingProcessingFolder = processingBatchFolder.CreateSubdirectory(filingName); string[] unzippedFiles; string filingZipFile = string.Format("{0}{1}{2}", filingProcessingFolder.FullName, Path.DirectorySeparatorChar, Path.GetFileName(zipFile.Name)); if (filingRetryCount.ContainsKey(filingName) && filingRetryCount[filingName] >= maxUnzipAttempts) { filingRetryCount.Remove(filingName); //We have failed to extract the filing multiple times, stop trying to make it work, //remove the zip file from the filing folder, log an error in the reports folder, //and reomve the entry from the dictionary string errorFilename = filingName + "_" + ERROR_FILE_NAME; string errorFile = Path.Combine(FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, filingName); string errorMsg = string.Format("Cannot extract files for filing {0}. The max number of retries has been reached, removing the zip file from the Filings folder. ", Path.GetFileNameWithoutExtension(zipFile.FullName)); if (File.Exists(errorFile)) { FileUtilities.DeleteFile(new FileInfo(errorFile), true); } FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error); File.WriteAllText(errorFile, errorMsg); zipFile.CopyTo(filingZipFile); FileUtilities.DeleteFile(zipFile, true); } else { string zipError; if (ZipUtilities.TryUnzipAndUncompressFiles(zipFile.FullName, filingProcessingFolder.FullName, out unzippedFiles, out zipError)) { FilingInfo filing = new FilingInfo(); filing.AccessionNumber = filingProcessingFolder.Name; filing.ParentFolder = processingBatchFolder.FullName; filings.Add(filing); zipFile.CopyTo(filingZipFile, true); FileUtilities.DeleteFile(zipFile, true); //SEC0145 - Only queue one filing at a time break; } else { //Delete the folder from the processing batch folder sense it doesn't contain a valid filing FileUtilities.DeleteDirectory(filingProcessingFolder, true, true); if (!filingRetryCount.ContainsKey(filingName)) { filingRetryCount.Add(filingName, 0); } filingRetryCount[filingName]++; FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Can not extract files for filing {0}. The zip file may not be complete. ", Path.GetFileNameWithoutExtension(zipFile.FullName)), EventLogEntryType.Warning); } } } if (processingBatchFolder.GetDirectories().Length == 0) { //There were not any valid filings in this batch, remove the batch folder FileUtilities.DeleteDirectory(processingBatchFolder, true, true); } return(true); }