private bool BuildReports(string instanceFolder, string formType)
{
string accessionNumber = Path.GetFileName(instanceFolder);
string reportDir = string.Format(@"{0}{1}Reports", instanceFolder, Path.DirectorySeparatorChar);
string filingSummaryPath = string.Format(@"{0}{1}FilingSummary.xml", reportDir, Path.DirectorySeparatorChar);
if (Directory.Exists(reportDir))
{
//Make sure the reports dir is empty before starting. That way we can
//be sure the test actually created the correct files.
Directory.Delete(reportDir, true);
}
Directory.CreateDirectory(reportDir);
if (File.Exists(filingSummaryPath))
{
File.Delete(filingSummaryPath);
}
FilingInfo fi = new FilingInfo();
fi.AccessionNumber = accessionNumber;
fi.ParentFolder = testRoot;
fi.FormType = formType;
string error;
return(this.BuildReports(fi, reportsFolder, out error));
}
protected string GetTaxonomyPath(FilingInfo filing)
{
string taxonomyFile = string.Empty;
string instanceFile = GetInstanceDocPath(filing);
if (string.IsNullOrEmpty(instanceFile))
{
return(string.Empty);
}
XmlTextReader xReader = new XmlTextReader(instanceFile);
try
{
xReader.MoveToContent();
if (xReader.LocalName == "xbrl")
{
while (xReader.MoveToNextAttribute())
{
if (xReader.Value == "http://ici.org/rr/2006")
{
break;
}
}
xReader.Read();
while (xReader.LocalName != "schemaRef")
{
xReader.Read();
}
if (xReader.LocalName == "schemaRef")
{
while (xReader.MoveToNextAttribute())
{
if (xReader.NodeType == XmlNodeType.Attribute &&
xReader.Name == "xlink:href")
{
taxonomyFile = Path.GetFileName(xReader.Value);
break;
}
}
}
}
taxonomyFile = Path.Combine(Path.GetDirectoryName(instanceFile), taxonomyFile);
}
catch (Exception ex)
{
myEventLog.WriteEntry("Exception in GetTaxonomy Path: " + ex.Message, EventLogEntryType.Warning);
taxonomyFile = string.Empty;
}
return(taxonomyFile);
}
public void BuildReportsCallBack(Object objFiling)
{
FilingInfo filing = objFiling as FilingInfo;
if (filing == null)
{
return;
}
BuildReports(filing);
}
public bool BuildReports(FilingInfo filing)
{
string instancePath = GetInstanceDocPath(filing);
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
myEventLog.WriteEntry("Can not find instance document for filing: " + filing.AccessionNumber);
return(false);
}
string taxonomyPath = GetTaxonomyPath(filing);
if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
myEventLog.WriteEntry("Can not find taxonomy file for filing: " + filing.AccessionNumber);
return(false);
}
ReportBuilder rb = new ReportBuilder();
string reportPath = string.Format("{0}{1}{2}", reportsFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
string filingFile = string.Format("{0}{1}{2}", reportPath, Path.DirectorySeparatorChar, FilingSummary._FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(reportPath))
{
Directory.Delete(reportPath, true);
}
Directory.CreateDirectory(reportPath);
//TODO: get this info from the manifest once we have it
//rb.PeriodEnding = filing.period_ending;
//rb.FilingDate = filing.filing_date;
//rb.TickerSymbol = filing.ticker_symbol;
//rb.CompanyName = filing.company_name;
//rb.AccessionNumber = filing.accession_number;
//rb.FiscalYearEnd = filing.fiscal_year_end;
string error = null;
FilingSummary summary = null;
if (!rb.BuildReports(instancePath, taxonomyPath, filingFile, reportPath, filing.FormType, out summary, out error))
{
myEventLog.WriteEntry("build reports failed! " + error, EventLogEntryType.Error);
return(false);
}
//Increment the number of filings that were successfully process. This needs to be done
//using Interlocked because other worker threads could be accessing the property as well
Interlocked.Increment(ref completedFilings);
return(true);
}
public void ProcessFilingCallback(Object objFiling)
{
FilingInfo filing = objFiling as FilingInfo;
if (filing == null)
{
return;
}
string error;
BuildReports(filing, FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, out error);
}
private void ProcessNewFilings()
{
try
{
lock (filingsFolderInfo)
{
//Get all of the new sub folders in the filings folder
DirectoryInfo[] filingDirs = filingsFolderInfo.GetDirectories();
if (filingDirs.Length == 0)
{
//There are no new directories, so there is nothing to process, simply return
return;
}
myEventLog.WriteEntry("Found new filings, prepping the folders for processing.");
string processingSubDir = Guid.NewGuid().ToString();
DirectoryInfo targetFolderInfo = processingFolderInfo.CreateSubdirectory(processingSubDir);
List <FilingInfo> filings = new List <FilingInfo>();
foreach (DirectoryInfo subDir in filingsFolderInfo.GetDirectories())
{
if (TryMoveFilingToProcessingFolder(subDir, targetFolderInfo))
{
FilingInfo filing = new FilingInfo();
filing.AccessionNumber = subDir.Name;
filing.ParentFolder = targetFolderInfo.FullName;
filings.Add(filing);
}
}
this.ProcessFilings(filings);
}
}
catch (Exception ex)
{
myEventLog.WriteEntry("Exception while processing new filings: " + ex.Message);
return;
}
}
protected string GetInstanceDocPath(FilingInfo filing)
{
string filingDir = string.Format("{0}{1}{2}", filing.ParentFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
string[] files = Directory.GetFiles(filingDir, "*.xml");
Regex instanceRegex = new Regex(@"\d{8}\.xml$");
string retValue = string.Empty;
foreach (string file in files)
{
Match regexMatch = instanceRegex.Match(file);
if (regexMatch != null && regexMatch.Success)
{
//There should only be one file that matches the regex in each filing, so
//once we find it break and return
retValue = file;
break;
}
}
return(retValue);
}
public void TestGetTaxonomyPath()
{
string[][] testFilings = new string[10][];
testFilings[0] = new string[] { "0000008670-08-000046", "adp-20080331.xsd" };
testFilings[1] = new string[] { "0000040545-08-000037", "ge-20080630.xsd" };
testFilings[2] = new string[] { "0000796343-08-000005", "adbe-20080530.xsd" };
testFilings[3] = new string[] { "0000893220-08-002188", "dd-20080630.xsd" };
testFilings[4] = new string[] { "0000950133-08-003012", "nihd-20080331.xsd" };
testFilings[5] = new string[] { "0000950135-08-002601", "gis-20080224.xsd" };
testFilings[6] = new string[] { "0000950135-08-003083", "dd-20080331.xsd" };
testFilings[7] = new string[] { "0000950135-08-004978", "trow-20071231.xsd" };
testFilings[8] = new string[] { "0000950137-08-010830", "gis-20080525.xsd" };
testFilings[9] = new string[] { "0000950137-08-012088", "gis-20080824.xsd" };
List <string> errors = new List <string>();
foreach (string[] filing in testFilings)
{
FilingInfo fi = new FilingInfo();
fi.AccessionNumber = filing[0];
fi.ParentFolder = testRoot;
string taxonomyPath = fi.GetTaxonomyPath();
string expectedPath = string.Format("{0}{1}{2}{1}{3}", testRoot, System.IO.Path.DirectorySeparatorChar, filing[0], filing[1]);
if (!expectedPath.Equals(taxonomyPath, StringComparison.CurrentCultureIgnoreCase))
{
errors.Add("Paths do not match. Expected: '" + expectedPath + "'. Actual: '" + taxonomyPath + "'");
}
}
if (errors.Count > 0)
{
string strErrors = string.Join(Environment.NewLine + "\t", errors.ToArray());
Assert.Fail("The following errors were found: " + Environment.NewLine + strErrors);
}
}
public void TestGetInstanceDocPath()
{
string[][] testFilings = new string[10][];
testFilings[0] = new string[] { "0000008670-08-000046", "adp-20080331.xml" };
testFilings[1] = new string[] { "0000040545-08-000037", "ge-20080630.xml" };
testFilings[2] = new string[] { "0000796343-08-000005", "adbe-20080616.xml" };
testFilings[3] = new string[] { "0000893220-08-002188", "dd-20080630.xml" };
testFilings[4] = new string[] { "0000950133-08-003012", "nihd-20080331.xml" };
testFilings[5] = new string[] { "0000950135-08-002601", "gis-20080224.xml" };
testFilings[6] = new string[] { "0000950135-08-003083", "dd-20080331.xml" };
testFilings[7] = new string[] { "0000950135-08-004978", "trow-20071231.xml" };
testFilings[8] = new string[] { "0000950137-08-010830", "gis-20080525.xml" };
testFilings[9] = new string[] { "0000950137-08-012088", "gis-20080824.xml" };
List<string> errors = new List<string>();
foreach (string[] filing in testFilings)
{
FilingInfo fi = new FilingInfo();
fi.AccessionNumber = filing[0];
fi.ParentFolder = testRoot;
string instanceDocPath = fi.GetInstanceDocPath();
string expectedPath = string.Format("{0}{1}{2}{1}{3}", testRoot, System.IO.Path.DirectorySeparatorChar, filing[0], filing[1]);
if (!expectedPath.Equals(instanceDocPath, StringComparison.CurrentCultureIgnoreCase))
{
errors.Add("Paths do not match. Expected: '" + expectedPath + "'. Actual: '" + instanceDocPath + "'");
}
}
if (errors.Count > 0)
{
string strErrors = string.Join(Environment.NewLine + "\t", errors.ToArray());
Assert.Fail("The following errors were found: " + Environment.NewLine + "\t" + strErrors);
}
}
protected bool IsFilingComplete(FilingInfo filing)
{
//TODO: we need logic here and some of it depends upon the SEC and what they write
//with the filing
return(true);
}
protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error)
{
error = null;
bool foundFiles = true;
string errorMsg = string.Empty;
string filingPath = Path.Combine(filing.ParentFolder, filing.AccessionNumber);
string filingReportsPath = Path.Combine(filingPath, "Reports");
string filingErrorFile = Path.Combine(filingReportsPath, ERROR_FILE_NAME);
string instancePath = filing.GetInstanceDocPath();
string taxonomyPath = filing.GetTaxonomyPath();
if (string.IsNullOrEmpty(instancePath) ||
!File.Exists(instancePath))
{
errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
else if (string.IsNullOrEmpty(taxonomyPath) ||
!File.Exists(taxonomyPath))
{
errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
foundFiles = false;
}
bool buildSucceeded = false;
if (foundFiles)
{
string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}", baseResourcePath), EventLogEntryType.Information);
RulesEngineUtils.SetBaseResourcePath(baseResourcePath);
ReportBuilder.SetSynchronizedResources(true);
FilingProcessorManager.TheMgr.WriteLogEntry("Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName(instancePath) + "'", EventLogEntryType.Information);
ReportBuilder rb = new ReportBuilder(FinancialRuleFile, ReportFormat, HtmlReportFormat);
rb.CurrencyMappingFile = CurrencyMappingFile;
rb.RemoteFileCachePolicy = RemoteFileCachePolicy;
//if( XmlCatalog != null )
// rb.XmlCatalog = XmlCatalog;
//Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder,
//this will allow us to only copy complete sets of R files to the Reports Folder.
string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName);
//Make sure there is a clean folder where the reports will be written to.
if (Directory.Exists(filingReportsPath))
{
Directory.Delete(filingReportsPath, true);
}
Directory.CreateDirectory(filingReportsPath);
FilingSummary summary = null;
buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error);
if (!buildSucceeded)
{
errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error);
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
if (!Directory.Exists(filingReportsPath))
{
Directory.CreateDirectory(filingReportsPath);
}
File.WriteAllText(filingErrorFile, errorMsg);
}
}
else
{
if (!Directory.Exists(filingReportsPath))
{
Directory.CreateDirectory(filingReportsPath);
}
File.WriteAllText(filingErrorFile, errorMsg);
}
try
{
string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName(filingErrorFile);
string reportErrorFile = Path.Combine(reportsFolder, errorFileName);
string reportZipFile = Path.Combine(reportsFolder, filing.AccessionNumber + ".zip");
if (File.Exists(reportErrorFile))
{
FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true);
}
if (File.Exists(reportZipFile))
{
FileUtilities.DeleteFile(new FileInfo(reportZipFile), true);
}
if (buildSucceeded)
{
string[] filePathsToZip = Directory.GetFiles(filingReportsPath);
string[] filesToZip = new string[filePathsToZip.Length];
for (int i = 0; i < filesToZip.Length; i++)
{
filesToZip[i] = Path.GetFileName(filePathsToZip[i]);
}
string zipFile = Path.Combine(filingReportsPath, filing.AccessionNumber + ".zip");
if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip))
{
File.Copy(zipFile, reportZipFile);
}
}
else
{
File.Copy(filingErrorFile, reportErrorFile);
}
if (DeleteProcessedFilings)
{
DirectoryInfo di = new DirectoryInfo(filingPath);
FileUtilities.DeleteDirectory(di, true, true);
di = new DirectoryInfo(filing.ParentFolder);
if (di.GetDirectories().Length == 0 && di.GetFiles().Length == 0)
{
FileUtilities.DeleteDirectory(di, true, true);
}
}
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.Message);
}
if (buildSucceeded)
{
FilingProcessorManager.TheMgr.IncrementCompletedCount();
}
return(buildSucceeded);
}
public bool TryMoveFilingsToProcessingFolder(out List <FilingInfo> filings)
{
filings = new List <FilingInfo>();
FileInfo[] zipFiles = FilingProcessorManager.TheMgr.FilingsFolderInfo.GetFiles("*.zip", SearchOption.TopDirectoryOnly);
Array.Sort(zipFiles, (left, right) => DateTime.Compare(left.CreationTime, right.CreationTime));
Guid batchId = Guid.NewGuid();
DirectoryInfo processingBatchFolder = FilingProcessorManager.TheMgr.ProcessingFolderInfo.CreateSubdirectory(batchId.ToString());
foreach (FileInfo zipFile in zipFiles)
{
string filingName = Path.GetFileNameWithoutExtension(zipFile.Name);
if (string.IsNullOrEmpty(filingName))
{
try
{
string errorFile = Path.Combine(FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, "UNKNOWN.txt");
string errorMessage = "Cannot process a filing zip which has no base name.";
FilingProcessorManager.TheMgr.WriteLogEntry(errorMessage, EventLogEntryType.Error);
File.WriteAllText(errorFile, errorMessage);
zipFile.Delete();
} catch { }
continue;
}
DirectoryInfo filingProcessingFolder = processingBatchFolder.CreateSubdirectory(filingName);
string[] unzippedFiles;
string filingZipFile = string.Format("{0}{1}{2}", filingProcessingFolder.FullName, Path.DirectorySeparatorChar, Path.GetFileName(zipFile.Name));
if (filingRetryCount.ContainsKey(filingName) &&
filingRetryCount[filingName] >= maxUnzipAttempts)
{
filingRetryCount.Remove(filingName);
//We have failed to extract the filing multiple times, stop trying to make it work,
//remove the zip file from the filing folder, log an error in the reports folder,
//and reomve the entry from the dictionary
string errorFilename = filingName + "_" + ERROR_FILE_NAME;
string errorFile = Path.Combine(FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, filingName);
string errorMsg = string.Format("Cannot extract files for filing {0}. The max number of retries has been reached, removing the zip file from the Filings folder. ", Path.GetFileNameWithoutExtension(zipFile.FullName));
if (File.Exists(errorFile))
{
FileUtilities.DeleteFile(new FileInfo(errorFile), true);
}
FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
File.WriteAllText(errorFile, errorMsg);
zipFile.CopyTo(filingZipFile);
FileUtilities.DeleteFile(zipFile, true);
}
else
{
string zipError;
if (ZipUtilities.TryUnzipAndUncompressFiles(zipFile.FullName, filingProcessingFolder.FullName, out unzippedFiles, out zipError))
{
FilingInfo filing = new FilingInfo();
filing.AccessionNumber = filingProcessingFolder.Name;
filing.ParentFolder = processingBatchFolder.FullName;
filings.Add(filing);
zipFile.CopyTo(filingZipFile, true);
FileUtilities.DeleteFile(zipFile, true);
//SEC0145 - Only queue one filing at a time
break;
}
else
{
//Delete the folder from the processing batch folder sense it doesn't contain a valid filing
FileUtilities.DeleteDirectory(filingProcessingFolder, true, true);
if (!filingRetryCount.ContainsKey(filingName))
{
filingRetryCount.Add(filingName, 0);
}
filingRetryCount[filingName]++;
FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Can not extract files for filing {0}. The zip file may not be complete. ", Path.GetFileNameWithoutExtension(zipFile.FullName)), EventLogEntryType.Warning);
}
}
}
if (processingBatchFolder.GetDirectories().Length == 0)
{
//There were not any valid filings in this batch, remove the batch folder
FileUtilities.DeleteDirectory(processingBatchFolder, true, true);
}
return(true);
}
