static void Run(Options options)
{
options.Print(Console.Out);
string inputFolder = Path.GetFullPath(options.InputFolder);
if (!Directory.Exists(inputFolder))
{
Console.WriteLine("Input file folder {0} does not exist.", inputFolder);
return;
}
DataDictionary dataDictionary = new DataDictionary();
if (!string.IsNullOrEmpty(options.DataDictionaryFile))
{
dataDictionary.Load(options.DataDictionaryFile);
}
DataDictionary newDataDictionary = new DataDictionary(dataDictionary);
IReportParser parser = ReportParserFactory.Create(options.FinanceReportFileType, dataDictionary, Console.Error);
List <FinanceReport> reports = new List <FinanceReport>();
foreach (var file in Directory.EnumerateFiles(inputFolder))
{
string code = Path.GetFileNameWithoutExtension(file);
if (string.IsNullOrWhiteSpace(code))
{
Console.WriteLine("The file name {0} is not expected", file);
}
FinanceReport report = parser.ParseReport(code, file);
if (report == null)
{
Console.WriteLine("Parse report file {0} failed.", file);
}
else
{
Console.WriteLine("Parse report for {0}:{1} succeeded.", report.CompanyCode, report.CompanyName);
AddReportMetadataToDataDictionary(report, newDataDictionary);
reports.Add(report);
}
}
// normalize each report according to data dictionary
foreach (var report in reports)
{
report.Normalize(newDataDictionary);
}
// expand and merge tables in reports
ExpandAndMergeTables(reports);
// create revenue table for last 12 months
CreateRevenueTableForLast12Months(reports);
// output reports
OutputReports(reports, options.OutputFolder);
// save new data dictionary if necessary.
if (!string.IsNullOrEmpty(options.GenerateDataDictionaryFile))
{
newDataDictionary.Save(options.GenerateDataDictionaryFile);
}
Console.WriteLine("Done.");
}
public bool FetchMailbox()
{
_logger.LogInformation("Checking mailbox...");
using (var client = new ImapClient()) {
// For demo-purposes, accept all SSL certificates
client.ServerCertificateValidationCallback = (s, c, h, e) => true;
client.Connect(_host, _port, _useSSL);
try {
client.Authenticate(_username, _password);
// Find folders: parsed, failed
var folders = client.GetFolders(client.PersonalNamespaces[0]);
MailKit.IMailFolder parsed = null;
MailKit.IMailFolder failed = null;
foreach (var folder in folders)
{
if (folder.Name == "parsed")
{
parsed = folder;
}
else if (folder.Name == "failed")
{
failed = folder;
}
}
if (parsed == null || failed == null)
{
_logger.LogError("Couldn't find 'parsed' or 'failed' folder in the mailbox");
client.Disconnect(true);
return(false);
}
else
{
_logger.LogInformation("Successfully found 'parsed' and 'failed' folders");
parsed.Open(FolderAccess.ReadWrite);
failed.Open(FolderAccess.ReadWrite);
}
var inbox = client.Inbox;
inbox.Open(FolderAccess.ReadWrite);
_logger.LogInformation("Total messages: {0}", inbox.Count);
_logger.LogInformation("Recent messages: {0}", inbox.Recent);
// Not sure if we can iterate forwards and move the messages at the same time.
// Thus iterating backwards.
for (int i = inbox.Count - 1; i >= 0; --i)
{
var message = inbox.GetMessage(i);
string sender;
string subject;
string body;
if (message.Sender != null)
{
sender = message.Sender.Address;
}
else
{
sender = "<null>";
}
if (message.Subject != null)
{
subject = message.Subject;
}
else
{
subject = "<null>";
}
if (message.Body != null)
{
body = message.Body.ToString();
}
else
{
body = "<null>";
}
Report report = _reportParser.ParseReport(sender, subject, body);
if (report != null)
{
//download attachments and save to chosen folder
char[] separators = { (char)47, (char)92 };
string etr = report.EtrNumber.Replace("\r", "");
etr = string.Join("", etr.Split(separators));
foreach (var attachment in message.Attachments)
{
string fileName = string.Join("_", attachment.ContentDisposition.FileName.Split(Path.GetInvalidFileNameChars()));
fileName = string.Join("_", fileName.Split(separators));
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @"..\..\..\attachments\" + etr);
try
{
if (!Directory.Exists(path))
{
DirectoryInfo di = Directory.CreateDirectory(path);
}
path = path + @"\" + fileName;
using (var stream = File.Create(path))
{
if (attachment is MessagePart)
{
var part = (MessagePart)attachment;
part.Message.WriteTo(stream);
}
else
{
var part = (MimePart)attachment;
part.ContentObject.DecodeTo(stream);
_logger.LogInformation("Attachment " + fileName + " stored for report: " + etr);
}
}
}
catch (IOException e) { _logger.LogError("Could not create directory or file: ", e); };
}
//end of attachments storage part
try
{
inbox.AddFlags(i, MessageFlags.Seen, true);
inbox.MoveTo(i, parsed);
} catch (MailKit.Net.Imap.ImapCommandException e) {
_logger.LogError("Exception occured while trying to mark mail " +
"as read or move it: {0}", e);
}
using (IServiceScope scope = _serviceProvider.CreateScope()) {
var context = scope.ServiceProvider
.GetRequiredService <ApplicationDbContext>();
// If there is an existing report in the database that matches
// ETR Number of this report, ignore this report (don't put
// duplicates)
Report alreadyExisting = context.Report.SingleOrDefault(existing
=> existing.EtrNumber == report.EtrNumber);
if (alreadyExisting == null)
{
_logger.LogInformation("Successfully parsed report '{0}'" +
" from {1} - inserting to database",
subject, sender);
context.Report.Add(report);
context.SaveChanges();
}
else
{
_logger.LogInformation("Not inserting duplicate report {0}",
report.EtrNumber);
}
}
}
else
{
_logger.LogWarning("Failed to parse report '{0}' from {1}",
subject, sender);
inbox.AddFlags(i, MessageFlags.Seen, true);
inbox.MoveTo(i, failed);
}
}
client.Disconnect(true);
} catch (ImapProtocolException e) {
_logger.LogError("An exception occured while trying to fetch mailbox: {0}", e);
}
}
_logger.LogInformation("Finished checking.");
return(true);
}