private bool BuildReports(string instanceFolder, string formType)
        {
            string accessionNumber = Path.GetFileName(instanceFolder);

            string reportDir = string.Format(@"{0}{1}Reports", instanceFolder, Path.DirectorySeparatorChar);
            string filingSummaryPath = string.Format(@"{0}{1}FilingSummary.xml", reportDir, Path.DirectorySeparatorChar);

            if (Directory.Exists(reportDir))
            {
                //Make sure the reports dir is empty before starting.  That way we can
                //be sure the test actually created the correct files.
                Directory.Delete(reportDir, true);
            }
            Directory.CreateDirectory(reportDir);

            if (File.Exists(filingSummaryPath))
            {
                File.Delete(filingSummaryPath);
            }

            FilingInfo fi = new FilingInfo();
            fi.AccessionNumber = accessionNumber;
            fi.ParentFolder = testRoot;
            fi.FormType = formType;

            string error;
            return this.BuildReports(fi, reportsFolder, out error);
        }
Ejemplo n.º 2
0
 protected bool IsFilingComplete(FilingInfo filing)
 {
     //TODO:  we need logic here and some of it depends upon the SEC and what they write
     //with the filing
     return true;
 }
Ejemplo n.º 3
0
        private void ProcessNewFilings()
        {
            try
            {
                lock (filingsFolderInfo)
                {
                    //Get all of the new sub folders in the filings folder
                    DirectoryInfo[] filingDirs = filingsFolderInfo.GetDirectories();
                    if (filingDirs.Length == 0)
                    {
                        //There are no new directories, so there is nothing to process, simply return
                        return;
                    }

                    myEventLog.WriteEntry("Found new filings, prepping the folders for processing.");

                    string processingSubDir = Guid.NewGuid().ToString();
                    DirectoryInfo targetFolderInfo = processingFolderInfo.CreateSubdirectory(processingSubDir);
                    List<FilingInfo> filings = new List<FilingInfo>();

                    foreach (DirectoryInfo subDir in filingsFolderInfo.GetDirectories())
                    {
                        if (TryMoveFilingToProcessingFolder(subDir, targetFolderInfo))
                        {
                            FilingInfo filing = new FilingInfo();
                            filing.AccessionNumber = subDir.Name;
                            filing.ParentFolder = targetFolderInfo.FullName;
                            filings.Add(filing);
                        }
                    }
                    this.ProcessFilings(filings);
                }
            }
            catch (Exception ex)
            {
                myEventLog.WriteEntry("Exception while processing new filings: " + ex.Message);
                return;
            }
        }
Ejemplo n.º 4
0
        protected string GetTaxonomyPath(FilingInfo filing)
        {
            string taxonomyFile = string.Empty;

            string instanceFile = GetInstanceDocPath(filing);
            if (string.IsNullOrEmpty(instanceFile))
            {
                return string.Empty;
            }

            XmlTextReader xReader = new XmlTextReader(instanceFile);
            try
            {
                xReader.MoveToContent();
                if (xReader.LocalName == "xbrl")
                {
                    while (xReader.MoveToNextAttribute())
                    {
                        if (xReader.Value == "http://ici.org/rr/2006")
                        {
                            break;
                        }
                    }

                    xReader.Read();
                    while (xReader.LocalName != "schemaRef")
                    {
                        xReader.Read();
                    }

                    if (xReader.LocalName == "schemaRef")
                    {
                        while (xReader.MoveToNextAttribute())
                        {
                            if (xReader.NodeType == XmlNodeType.Attribute &&
                                  xReader.Name == "xlink:href")
                            {
                                taxonomyFile = Path.GetFileName(xReader.Value);
                                break;
                            }
                        }
                    }
                }

                taxonomyFile = Path.Combine(Path.GetDirectoryName(instanceFile), taxonomyFile);
            }
            catch (Exception ex)
            {
                myEventLog.WriteEntry("Exception in GetTaxonomy Path: " + ex.Message, EventLogEntryType.Warning);
                taxonomyFile = string.Empty;
            }

            return taxonomyFile;
        }
Ejemplo n.º 5
0
        protected string GetInstanceDocPath(FilingInfo filing)
        {
            string filingDir = string.Format("{0}{1}{2}", filing.ParentFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
            string[] files = Directory.GetFiles(filingDir, "*.xml");
            Regex instanceRegex = new Regex(@"\d{8}\.xml$");

            string retValue = string.Empty;
            foreach (string file in files)
            {
                Match regexMatch = instanceRegex.Match(file);
                if (regexMatch != null && regexMatch.Success)
                {
                    //There should only be one file that matches the regex in each filing, so
                    //once we find it break and return
                    retValue = file;
                    break;
                }
            }

            return retValue;
        }
Ejemplo n.º 6
0
        public bool BuildReports(FilingInfo filing)
        {
            string instancePath = GetInstanceDocPath(filing);
            if (string.IsNullOrEmpty(instancePath) ||
                !File.Exists(instancePath))
            {
                myEventLog.WriteEntry("Can not find instance document for filing: " + filing.AccessionNumber);
                return false;
            }

            string taxonomyPath = GetTaxonomyPath(filing);
            if (string.IsNullOrEmpty(taxonomyPath) ||
                !File.Exists(taxonomyPath))
            {
                myEventLog.WriteEntry("Can not find taxonomy file for filing: " + filing.AccessionNumber);
                return false;
            }

            ReportBuilder rb = new ReportBuilder();

            string reportPath = string.Format("{0}{1}{2}", reportsFolder, Path.DirectorySeparatorChar, filing.AccessionNumber);
            string filingFile = string.Format("{0}{1}{2}", reportPath, Path.DirectorySeparatorChar, FilingSummary._FilingSummaryXmlName);

            //Make sure there is a clean folder where the reports will be written to.
            if (Directory.Exists(reportPath))
            {
                Directory.Delete(reportPath, true);
            }
            Directory.CreateDirectory(reportPath);

            //TODO:  get this info from the manifest once we have it
            //rb.PeriodEnding = filing.period_ending;
            //rb.FilingDate = filing.filing_date;
            //rb.TickerSymbol = filing.ticker_symbol;
            //rb.CompanyName = filing.company_name;
            //rb.AccessionNumber = filing.accession_number;
            //rb.FiscalYearEnd = filing.fiscal_year_end;

            string error = null;
            FilingSummary summary = null;
            if (!rb.BuildReports(instancePath, taxonomyPath, filingFile, reportPath, filing.FormType, out summary, out error))
            {
                myEventLog.WriteEntry("build reports failed! " + error, EventLogEntryType.Error);
                return false;
            }
            //Increment the number of filings that were successfully process.  This needs to be done
            //using Interlocked because other worker threads could be accessing the property as well
            Interlocked.Increment(ref completedFilings);
            return true;
        }
Ejemplo n.º 7
0
        protected bool BuildReports(FilingInfo filing, string reportsFolder, out string error)
        {
            error = null;

            bool foundFiles = true;
            string errorMsg = string.Empty;

            string filingPath = Path.Combine( filing.ParentFolder, filing.AccessionNumber );
            string filingReportsPath = Path.Combine( filingPath, "Reports" );
            string filingErrorFile = Path.Combine( filingReportsPath, ERROR_FILE_NAME );

            string instancePath = filing.GetInstanceDocPath();
            string taxonomyPath = filing.GetTaxonomyPath();

            if (string.IsNullOrEmpty(instancePath) ||
                !File.Exists(instancePath))
            {
                errorMsg = string.Format("Can not find instance document for filing: {0}", filing.AccessionNumber);
                FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
                foundFiles = false;
            }
            else if (string.IsNullOrEmpty(taxonomyPath) ||
                !File.Exists(taxonomyPath))
            {
                errorMsg = string.Format("Can not find taxonomy file for filing: {0}", filing.AccessionNumber);
                FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
                foundFiles = false;
            }

            bool buildSucceeded = false;
            if (foundFiles)
            {
                string baseResourcePath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
                FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Setting base path for Rules Engine. Path: {0}",baseResourcePath), EventLogEntryType.Information);

                RulesEngineUtils.SetBaseResourcePath(baseResourcePath);
                ReportBuilder.SetSynchronizedResources( true );

                FilingProcessorManager.TheMgr.WriteLogEntry( "Selected rule file: '" + FinancialRuleFile + "' for instance document: '" + Path.GetFileName( instancePath ) + "'", EventLogEntryType.Information );

                ReportBuilder rb = new ReportBuilder( FinancialRuleFile, ReportFormat, HtmlReportFormat );
                rb.CurrencyMappingFile = CurrencyMappingFile;
                rb.RemoteFileCachePolicy = RemoteFileCachePolicy;

                //if( XmlCatalog != null )
                //	rb.XmlCatalog = XmlCatalog;

                //Reports will be saves under the Filing's Processing folder then copied out to the actual reports folder,
                //this will allow us to only copy complete sets of R files to the Reports Folder.
                string filingSummaryFile = string.Format("{0}{1}{2}", filingReportsPath, Path.DirectorySeparatorChar, FilingSummary.FilingSummaryXmlName);

                //Make sure there is a clean folder where the reports will be written to.
                if (Directory.Exists(filingReportsPath))
                    Directory.Delete(filingReportsPath, true);

                Directory.CreateDirectory(filingReportsPath);
                FilingSummary summary = null;
                buildSucceeded = rb.BuildReports(instancePath, taxonomyPath, filingSummaryFile, filingReportsPath, out summary, out error);

                if (!buildSucceeded)
                {
                    errorMsg = string.Format("Call to BuildReports failed for Filing {0}: {1}", filing.AccessionNumber, error);
                    FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);

                    if (!Directory.Exists(filingReportsPath))
                        Directory.CreateDirectory(filingReportsPath);

                    File.WriteAllText(filingErrorFile, errorMsg);
                }
            }
            else
            {
                if (!Directory.Exists(filingReportsPath))
                    Directory.CreateDirectory(filingReportsPath);

                File.WriteAllText(filingErrorFile, errorMsg);
            }

            try
            {
                string errorFileName = filing.AccessionNumber + "_" + Path.GetFileName( filingErrorFile );
                string reportErrorFile = Path.Combine( reportsFolder, errorFileName );
                string reportZipFile = Path.Combine( reportsFolder, filing.AccessionNumber + ".zip" );
                if (File.Exists(reportErrorFile))
                    FileUtilities.DeleteFile(new FileInfo(reportErrorFile), true);

                if (File.Exists(reportZipFile))
                    FileUtilities.DeleteFile(new FileInfo(reportZipFile), true);

                if (buildSucceeded)
                {
                    string[] filePathsToZip = Directory.GetFiles(filingReportsPath);
                    string[] filesToZip = new string[filePathsToZip.Length];
                    for (int i = 0; i < filesToZip.Length; i++)
                    {
                        filesToZip[i] = Path.GetFileName(filePathsToZip[i]);
                    }

                    string zipFile = Path.Combine( filingReportsPath, filing.AccessionNumber +".zip");
                    if (ZipUtilities.TryZipAndCompressFiles(zipFile, filingReportsPath, filesToZip))
                        File.Copy(zipFile, reportZipFile);
                }
                else
                {
                    File.Copy(filingErrorFile, reportErrorFile);
                }

                if( DeleteProcessedFilings )
                {
                    DirectoryInfo di = new DirectoryInfo( filingPath );
                    FileUtilities.DeleteDirectory( di, true, true );

                    di = new DirectoryInfo( filing.ParentFolder );
                    if( di.GetDirectories().Length == 0 && di.GetFiles().Length == 0 )
                        FileUtilities.DeleteDirectory( di, true, true );
                }
            }
            catch(Exception ex)
            {
                System.Diagnostics.Debug.WriteLine(ex.Message);
            }

            if (buildSucceeded)
                FilingProcessorManager.TheMgr.IncrementCompletedCount();

            return buildSucceeded;
        }
Ejemplo n.º 8
0
        public bool TryMoveFilingsToProcessingFolder(out List<FilingInfo> filings)
        {
            filings = new List<FilingInfo>();

            FileInfo[] zipFiles = FilingProcessorManager.TheMgr.FilingsFolderInfo.GetFiles("*.zip", SearchOption.TopDirectoryOnly);
            Array.Sort( zipFiles, ( left, right ) => DateTime.Compare( left.CreationTime, right.CreationTime ) );

            Guid batchId = Guid.NewGuid();
            DirectoryInfo processingBatchFolder = FilingProcessorManager.TheMgr.ProcessingFolderInfo.CreateSubdirectory(batchId.ToString());

            foreach (FileInfo zipFile in zipFiles)
            {
                string filingName = Path.GetFileNameWithoutExtension(zipFile.Name);
                if( string.IsNullOrEmpty( filingName ) )
                {
                    try
                    {
                        string errorFile = Path.Combine( FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, "UNKNOWN.txt" );
                        string errorMessage = "Cannot process a filing zip which has no base name.";
                        FilingProcessorManager.TheMgr.WriteLogEntry( errorMessage, EventLogEntryType.Error );
                        File.WriteAllText( errorFile, errorMessage );
                        zipFile.Delete();
                    } catch { }

                    continue;
                }

                DirectoryInfo filingProcessingFolder = processingBatchFolder.CreateSubdirectory(filingName);

                string[] unzippedFiles;
                string filingZipFile = string.Format("{0}{1}{2}", filingProcessingFolder.FullName, Path.DirectorySeparatorChar,Path.GetFileName(zipFile.Name));

                if (filingRetryCount.ContainsKey(filingName) &&
                    filingRetryCount[filingName] >= maxUnzipAttempts)
                {
                    filingRetryCount.Remove(filingName);

                    //We have failed to extract the filing multiple times, stop trying to make it work,
                    //remove the zip file from the filing folder, log an error in the reports folder,
                    //and reomve the entry from the dictionary
                    string errorFilename = filingName +"_"+ ERROR_FILE_NAME;
                    string errorFile = Path.Combine( FilingProcessorManager.TheMgr.ReportsFolderInfo.FullName, filingName );

                    string errorMsg = string.Format("Cannot extract files for filing {0}.  The max number of retries has been reached, removing the zip file from the Filings folder. ", Path.GetFileNameWithoutExtension(zipFile.FullName));
                    if (File.Exists(errorFile))
                    {
                        FileUtilities.DeleteFile(new FileInfo(errorFile), true);
                    }
                    FilingProcessorManager.TheMgr.WriteLogEntry(errorMsg, EventLogEntryType.Error);
                    File.WriteAllText(errorFile, errorMsg);

                    zipFile.CopyTo(filingZipFile);
                    FileUtilities.DeleteFile(zipFile, true);
                }
                else
                {
                    string zipError;
                    if (ZipUtilities.TryUnzipAndUncompressFiles(zipFile.FullName, filingProcessingFolder.FullName, out unzippedFiles, out zipError))
                    {
                        FilingInfo filing = new FilingInfo();
                        filing.AccessionNumber = filingProcessingFolder.Name;
                        filing.ParentFolder = processingBatchFolder.FullName;
                        filings.Add(filing);

                        zipFile.CopyTo(filingZipFile, true);
                        FileUtilities.DeleteFile(zipFile, true);

                        //SEC0145 - Only queue one filing at a time
                        break;
                    }
                    else
                    {
                        //Delete the folder from the processing batch folder sense it doesn't contain a valid filing
                        FileUtilities.DeleteDirectory(filingProcessingFolder, true, true);
                        if (!filingRetryCount.ContainsKey(filingName))
                        {
                            filingRetryCount.Add(filingName, 0);
                        }
                        filingRetryCount[filingName]++;
                        FilingProcessorManager.TheMgr.WriteLogEntry(string.Format("Can not extract files for filing {0}.  The zip file may not be complete. ", Path.GetFileNameWithoutExtension(zipFile.FullName)), EventLogEntryType.Warning);
                    }
                }
            }

            if (processingBatchFolder.GetDirectories().Length == 0)
            {
                //There were not any valid filings in this batch, remove the batch folder
                FileUtilities.DeleteDirectory(processingBatchFolder, true, true);
            }

            return true;
        }