static void Run(Options options) { options.Print(Console.Out); string inputFolder = Path.GetFullPath(options.InputFolder); if (!Directory.Exists(inputFolder)) { Console.WriteLine("Input file folder {0} does not exist.", inputFolder); return; } DataDictionary dataDictionary = new DataDictionary(); if (!string.IsNullOrEmpty(options.DataDictionaryFile)) { dataDictionary.Load(options.DataDictionaryFile); } DataDictionary newDataDictionary = new DataDictionary(dataDictionary); IReportParser parser = ReportParserFactory.Create(options.FinanceReportFileType, dataDictionary, Console.Error); List <FinanceReport> reports = new List <FinanceReport>(); foreach (var file in Directory.EnumerateFiles(inputFolder)) { string code = Path.GetFileNameWithoutExtension(file); if (string.IsNullOrWhiteSpace(code)) { Console.WriteLine("The file name {0} is not expected", file); } FinanceReport report = parser.ParseReport(code, file); if (report == null) { Console.WriteLine("Parse report file {0} failed.", file); } else { Console.WriteLine("Parse report for {0}:{1} succeeded.", report.CompanyCode, report.CompanyName); AddReportMetadataToDataDictionary(report, newDataDictionary); reports.Add(report); } } // normalize each report according to data dictionary foreach (var report in reports) { report.Normalize(newDataDictionary); } // expand and merge tables in reports ExpandAndMergeTables(reports); // create revenue table for last 12 months CreateRevenueTableForLast12Months(reports); // output reports OutputReports(reports, options.OutputFolder); // save new data dictionary if necessary. if (!string.IsNullOrEmpty(options.GenerateDataDictionaryFile)) { newDataDictionary.Save(options.GenerateDataDictionaryFile); } Console.WriteLine("Done."); }
public bool FetchMailbox() { _logger.LogInformation("Checking mailbox..."); using (var client = new ImapClient()) { // For demo-purposes, accept all SSL certificates client.ServerCertificateValidationCallback = (s, c, h, e) => true; client.Connect(_host, _port, _useSSL); try { client.Authenticate(_username, _password); // Find folders: parsed, failed var folders = client.GetFolders(client.PersonalNamespaces[0]); MailKit.IMailFolder parsed = null; MailKit.IMailFolder failed = null; foreach (var folder in folders) { if (folder.Name == "parsed") { parsed = folder; } else if (folder.Name == "failed") { failed = folder; } } if (parsed == null || failed == null) { _logger.LogError("Couldn't find 'parsed' or 'failed' folder in the mailbox"); client.Disconnect(true); return(false); } else { _logger.LogInformation("Successfully found 'parsed' and 'failed' folders"); parsed.Open(FolderAccess.ReadWrite); failed.Open(FolderAccess.ReadWrite); } var inbox = client.Inbox; inbox.Open(FolderAccess.ReadWrite); _logger.LogInformation("Total messages: {0}", inbox.Count); _logger.LogInformation("Recent messages: {0}", inbox.Recent); // Not sure if we can iterate forwards and move the messages at the same time. // Thus iterating backwards. for (int i = inbox.Count - 1; i >= 0; --i) { var message = inbox.GetMessage(i); string sender; string subject; string body; if (message.Sender != null) { sender = message.Sender.Address; } else { sender = "<null>"; } if (message.Subject != null) { subject = message.Subject; } else { subject = "<null>"; } if (message.Body != null) { body = message.Body.ToString(); } else { body = "<null>"; } Report report = _reportParser.ParseReport(sender, subject, body); if (report != null) { //download attachments and save to chosen folder char[] separators = { (char)47, (char)92 }; string etr = report.EtrNumber.Replace("\r", ""); etr = string.Join("", etr.Split(separators)); foreach (var attachment in message.Attachments) { string fileName = string.Join("_", attachment.ContentDisposition.FileName.Split(Path.GetInvalidFileNameChars())); fileName = string.Join("_", fileName.Split(separators)); string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"..\..\..\attachments\" + etr); try { if (!Directory.Exists(path)) { DirectoryInfo di = Directory.CreateDirectory(path); } path = path + @"\" + fileName; using (var stream = File.Create(path)) { if (attachment is MessagePart) { var part = (MessagePart)attachment; part.Message.WriteTo(stream); } else { var part = (MimePart)attachment; part.ContentObject.DecodeTo(stream); _logger.LogInformation("Attachment " + fileName + " stored for report: " + etr); } } } catch (IOException e) { _logger.LogError("Could not create directory or file: ", e); }; } //end of attachments storage part try { inbox.AddFlags(i, MessageFlags.Seen, true); inbox.MoveTo(i, parsed); } catch (MailKit.Net.Imap.ImapCommandException e) { _logger.LogError("Exception occured while trying to mark mail " + "as read or move it: {0}", e); } using (IServiceScope scope = _serviceProvider.CreateScope()) { var context = scope.ServiceProvider .GetRequiredService <ApplicationDbContext>(); // If there is an existing report in the database that matches // ETR Number of this report, ignore this report (don't put // duplicates) Report alreadyExisting = context.Report.SingleOrDefault(existing => existing.EtrNumber == report.EtrNumber); if (alreadyExisting == null) { _logger.LogInformation("Successfully parsed report '{0}'" + " from {1} - inserting to database", subject, sender); context.Report.Add(report); context.SaveChanges(); } else { _logger.LogInformation("Not inserting duplicate report {0}", report.EtrNumber); } } } else { _logger.LogWarning("Failed to parse report '{0}' from {1}", subject, sender); inbox.AddFlags(i, MessageFlags.Seen, true); inbox.MoveTo(i, failed); } } client.Disconnect(true); } catch (ImapProtocolException e) { _logger.LogError("An exception occured while trying to fetch mailbox: {0}", e); } } _logger.LogInformation("Finished checking."); return(true); }