private void ResetInProgress() { using (var ctx = new EmailImportDataContext()) { foreach (var email in ctx.Emails.Where(e => e.InProgress == true)) { email.InProgress = null; } ctx.SubmitChanges(); } }
private void UpdateStatus(EmailStatus status, XElement errors) { int count = 0; while (true) { try { using (EmailImportDataContext ctx = new EmailImportDataContext()) { var eml = ctx.Emails.Single(e => e.EmailID == email.EmailID); if (status == EmailStatus.InProgress) { eml.InProgress = true; eml.StartTime = DateTime.Now; eml.EndTime = null; } else { eml.Status = status.ToString(); eml.ProcessedCount = eml.ProcessedCount.GetValueOrDefault() + 1; eml.EndTime = DateTime.Now; eml.InProgress = null; eml.Errors = errors; } ctx.SubmitChanges(); } break; } catch (Exception e) { // Increment the retry counter count++; // Log the exception ConfigLogger.Instance.Log(count < 5 ? LogSeverity.Warning : LogSeverity.Error, email.EmailID, e); // Retry up to 5 times if (count >= 5) { break; } // Sleep for a fraction Thread.Sleep(50); } } }
private void HouseKeeping() { ConfigLogger.Instance.LogInfo("EmailMonitor", "Performing house keeping tasks..."); try { using (var ctx = new EmailImportDataContext()) { // Loop through each profile foreach (var profile in Settings.MailboxProfiles.Values) { try { // Determine date to purge from based on retention setting var date = DateTime.Today.AddDays(0 - Math.Abs(profile.StorageRetention)); // Get emails to be purged var emails = ctx.Emails.Where(e => (e.MailboxGUID == profile.MailboxGUID) && (e.InProgress == null || e.InProgress == false) && (e.Status == "Complete" || e.Status == "Empty" || e.Status == "Rejected" || e.Status == "Ignored") && (e.EndTime != null && e.EndTime.Value < date)).ToList(); // Delete each email msg file foreach (var email in emails) { File.Delete(email.MessageFilePath); } // Delete the email entries from the Emails table ctx.Emails.DeleteAllOnSubmit(emails); // Submit the changes ctx.SubmitChanges(); // Log how many emails were deleted ConfigLogger.Instance.LogInfo("EmailMonitor", String.Format("Deleted {0} stored email{1} from {2}.", emails.Count, emails.Count == 1 ? "" : "s", profile.Description)); // Remove empty folders from the storage path RemoveEmptyFolders(profile.StoragePath); // Archive path deletion if (!String.IsNullOrWhiteSpace(profile.ArchivePath) && Directory.Exists(profile.ArchivePath)) { // Determine date to purge from based on retention setting date = DateTime.Today.AddDays(0 - Math.Abs(profile.ArchiveRetention)); // Delete dated archive folders earlier than the retention date foreach (var folder in Directory.GetDirectories(profile.ArchivePath)) { DateTime dateValue; if (DateTime.TryParseExact(Path.GetFileName(folder), "yyyyMMdd", null, System.Globalization.DateTimeStyles.None, out dateValue)) { if (dateValue < date) { Directory.Delete(folder, true); ConfigLogger.Instance.LogInfo("EmailMonitor", String.Format("Deleted archive folder {0}.", folder)); } } } } } catch (Exception e) { // Add custom data for logging purposes e.Data.Add("MailboxGUID", profile.MailboxGUID); // Log the warning (not error) ConfigLogger.Instance.LogWarning("EmailMonitor", e); } } } // Remove empty folders from the default storage path RemoveEmptyFolders(Settings.DefaultStoragePath); } catch (Exception e) { // We don't want housekeeping to cause a failure, but we want someone to // know about it so that they can investigate the issue ConfigLogger.Instance.LogWarning("EmailMonitor", e); } }
private int DownloadEmail(Imap imap, MailboxProfile profile, String storagePath) { int count = 0; // Build the MailQuery var query = new MailQuery(String.IsNullOrEmpty(profile.ImapQuery) ? "('Deleted' = 'False')" : String.Format("({0}&('Deleted' = 'False'))", profile.ImapQuery)); // Get all messages matching to the query var infos = imap.ListMessages(query); // If there are any messages to process, then process them if (infos.Any()) { ConfigLogger.Instance.LogInfo("ImapCollector", String.Format("Downloading {0} message{1} from {2}.", infos.Count, infos.Count == 1 ? "" : "s", profile.Description)); // Download each message foreach (var info in infos) { if (!timer.Enabled) { break; } // Just check to ensure its valid if (info.Deleted || String.IsNullOrWhiteSpace(info.UniqueId)) { continue; } // Track the start time for debug purposes var start = DateTime.Now; MailMessage message = null; try { // Download the message message = imap.FetchMessage(info.UniqueId); // Calculate the time taken to fetch the message var fetchTime = DateTime.Now.Subtract(start); // Process the message (So long as the fetch succeeded) if (message != null) { // Setup the data context using (var ctx = new EmailImportDataContext()) { long emailID = 0; // Truncate the subject to avoid data commit errors message.Subject = Truncate(message.Subject, 500); // Check for duplicate if (IsDuplicate(ctx, profile.MailboxGUID, message, ref emailID)) { // Log the duplicate error ConfigLogger.Instance.LogWarning("ImapCollector", String.Format("Message already downloaded, moved to duplicate folder (existing EmailID = {0}).", emailID)); // Move the message to the duplicate sub folder imap.MoveMessage(info.UniqueId, "Duplicate", true, false); } else { // Create an instance of the email database object var email = new Email(); // Assign properties email.MailboxGUID = profile.MailboxGUID; email.DateSent = message.DateSent(); email.DateReceived = message.DateReceived(); email.From = message.From.GetAddressOrDisplayName(); email.MessageID = message.MessageId; if (CreditCardHelper.ExistsCCNumber(message.Subject)) { email.Subject = CreditCardHelper.MaskCCNumbers(message.Subject, '#'); } else { email.Subject = message.Subject; } email.Timestamp = DateTime.Now; // Create the dated storage path var path = Path.Combine(storagePath, email.Timestamp.Value.ToString("yyyyMMdd")); FileSystemHelper.CreateDirectory(path); // Insert the new record ctx.Emails.InsertOnSubmit(email); // Submit the email (we need to get the email ID) using (TransactionScope scope = new TransactionScope()) { // Initial submit of changes ctx.SubmitChanges(); // Build the mail message file name email.MessageFilePath = Path.Combine(path, String.Format("{0:00000000}.eml", email.EmailID)); // Save to disk (delete anything that already exists) message.Save(email.MessageFilePath, MessageFormat.Eml); // Get the batch number - THIS SHOULD NEVER HAPPEN IN A MULTI THREAD SCENARIO WITHOUT A LOCK var batchNumber = ctx.BatchNumbers.SingleOrDefault(b => b.Group == profile.Group); // If there is no batchNumber defined yet, create and insert one if (batchNumber == null) { batchNumber = new BatchNumber(); batchNumber.Group = profile.Group; ctx.BatchNumbers.InsertOnSubmit(batchNumber); } // Init to 0 if null if (!batchNumber.Value.HasValue) { batchNumber.Value = 0; } // Set the new batch number to this email email.BatchNumber = String.Format(String.IsNullOrWhiteSpace(profile.BatchNumberFormat) ? "{0:00000000}" : profile.BatchNumberFormat, ++batchNumber.Value); // Final submit of updates ctx.SubmitChanges(); // Finally, commit to the database scope.Complete(); } // Move the email to the archive (if this fails, but the download is complete this // will just result in a duplicate next time round if the deleted flag is not set) imap.MoveMessage(info.UniqueId, "Archive", true, false); // Log message level download stats ConfigLogger.Instance.LogDebug("ImapCollector", String.Format("Message downloaded (EmailID = {0}, Fetch Time = {1}s, Total Time = {2}s).", email.EmailID, (int)fetchTime.TotalSeconds, (int)DateTime.Now.Subtract(start).TotalSeconds)); // Increment the download count count++; } } } } catch (OutOfMemoryException) { throw; } catch (Exception e) { ConfigLogger.Instance.LogError("ImapCollector", e); } finally { if (message != null) { message.Dispose(); } } } } return(count); }