private Boolean IsDuplicate(EmailImportDataContext ctx, Guid guid, MailMessage message, ref long emailID) { var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject; var email = ctx.Emails.FirstOrDefault(e => Object.Equals(e.MailboxGUID, guid) && Object.Equals(e.MessageID, message.MessageId) && Object.Equals(e.From, message.From.GetAddressOrDisplayName()) && Object.Equals(e.DateSent, message.DateSent()) && Object.Equals(e.DateReceived, message.DateReceived()) && Object.Equals(e.Subject, subject)); if (email != null) { emailID = email.EmailID; } return(email != null); }
private void Worker(Object waiter) { Batch batch = null; Interlocked.Increment(ref count); ((ManualResetEvent)waiter).Set(); try { var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject; ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Started..."); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Profile: {0}", profile.Description)); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Mailbox GUID: {0}", profile.MailboxGUID)); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Message-ID: {0}", message.MessageId)); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Subject: {0}", subject.Replace("\"", "'"))); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> From: {0}", message.From.GetAddressOrDisplayName().Replace("\"", "'"))); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Received: {0}", message.DateReceived())); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Batch Number: {0}", email.BatchNumber)); ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Email-ID: {0}", email.EmailID)); batch = new Batch(email, message, profile); if (batch.Documents.Any()) { var startTime = DateTime.Now; foreach (var document in batch.Documents) { // Get the filetype config element if one is defined var fileType = profile.FileTypes.FirstOrDefault(f => Regex.IsMatch(document.OriginalName, f.Pattern ?? string.Empty, RegexOptions.IgnoreCase)); // Setup common properties document.Resolution = profile.Resolution; document.BitDepth = (fileType != null && fileType.BitDepth.GetValueOrDefault() > 0) ? fileType.BitDepth.GetValueOrDefault() : profile.BitDepth; document.Passthrough = (profile.AttachmentConversion == AttachmentConversion.Passthrough && !document.IsBody) || (fileType != null && fileType.Passthrough.GetValueOrDefault()); document.Mailbox = profile.ImapUserName; document.BinarisationAlgorithm = (fileType != null) ? fileType.BinarisationAlgorithm.GetValueOrDefault(profile.BinarisationAlgorithm) : profile.BinarisationAlgorithm; document.AutoDeskew = (fileType != null) ? fileType.AutoDeskew : null; document.AutoRotate = (fileType != null) ? fileType.AutoRotate : null; document.RemoveFaxHeader = profile.RemoveFaxHeader; document.ProcessAs = (fileType != null) ? fileType.ProcessAs : null; document.PdfConversion = profile.PdfConversion; if (!document.Passthrough) { document.ErrorHandling.Unknown = profile.ErrorHandling.Unknown; document.ErrorHandling.Unsupported = profile.ErrorHandling.Unsupported; document.ErrorHandling.Unprocessable = profile.ErrorHandling.Unprocessable; // Check for error handling overrides if (email.Errors != null) { var error = email.Errors.Elements().Where(e => (String)e.Attribute("key") == Path.GetFileName(document.AttachmentFile)).FirstOrDefault(); if (error != null) { ErrorHandlingActions value; if (Enum.TryParse <ErrorHandlingActions>((String)error.Attribute("override"), out value)) { document.ErrorHandling.Unknown = value; document.ErrorHandling.Unsupported = value; document.ErrorHandling.Unprocessable = value; } // Set ProcessAs var processAs = (String)error.Attribute("processAs"); if (!String.IsNullOrWhiteSpace(processAs)) { document.ProcessAs = processAs; } } } // Get the converter document.Converter = GetConverter(document); } } // Convert documents in parallel Parallel.ForEach(batch.Documents, document => { document.Convert(); }); // XElement to store errors XElement errors = null; // If there are any documents containing error message, add these to an xml document and add to the database if (batch.Documents.Any(d => !String.IsNullOrWhiteSpace(d.ErrorMessage))) { errors = new XElement("Errors", new XAttribute("retry", batch.Documents.Any(d => d.ShouldRetry && !String.IsNullOrWhiteSpace(d.ErrorMessage))), from document in batch.Documents.Where(d => !String.IsNullOrEmpty(d.ErrorMessage)) select new XElement("Error", new XAttribute("key", Path.GetFileName(document.AttachmentFile)), new XAttribute("fileName", document.OriginalName), new XAttribute("reason", document.FailureReason), new XAttribute("message", RemoveInvalidXmlChars(document.ErrorMessage)), new XAttribute("action", document.ActionToTake.Value) ) ); } // Remove all documents where ActionToTake is Ignore batch.Documents.RemoveAll(d => d.ActionToTake.HasValue && d.ActionToTake.Value == ErrorHandlingActions.Ignore); // If IgnoreErrors is false and one or more documents contains errors, if (batch.Documents.Any(d => d.Success == false)) { UpdateStatus(EmailStatus.Error, errors); // Delete the working folder try { FileSystemHelper.DeleteDirectory(batch.OutputPath, true); } catch (Exception e) { ConfigLogger.Instance.LogWarning(email.EmailID, e); } ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Failed."); } else { ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Documents Converted in {0:c}.", DateTime.Now.Subtract(startTime))); // Check and remove blank pages (and documents) if enabled if (profile.RemoveBlankPages) { // No need to check the email body as they will always contain the header details // Blank check only applies to tif images (pdf document via passthrough excluded) foreach (var document in batch.Documents.Where(d => d.Source == DocumentSource.Attachment)) { // Remove pages that are blank var blanks = document.Pages.RemoveAll(p => p.IsBlank); // If any blanks, log the number removed if (blanks > 0) { ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Removed {0} blank page{1} from document {2}.", blanks, (blanks > 1) ? "s" : "", document.RelativePath)); } } } // Apply any filtering rules based on resulting image attributes foreach (var document in batch.Documents) { // Get the filetype config element if one is defined var fileType = profile.FileTypes.FirstOrDefault(f => Regex.IsMatch(document.OriginalName, f.Pattern ?? string.Empty, RegexOptions.IgnoreCase)); // Only proceed if there is a file type element for the current extension if (fileType != null) { // Remove all pages that fall under the Minimum Pixel threshold var filtered = document.Pages.RemoveAll(p => p.Area < fileType.MinPixels); // If any pages were filtered out, log how many if (filtered > 0) { ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Filtered out {0} page{1} from document {2}.", filtered, (filtered > 1) ? "s" : "", document.RelativePath)); } } } // Remove any empty documents following blank page removal and filtering int empty = batch.Documents.RemoveAll(d => !d.Pages.Any()); // If any blanks, log the number removed if (empty > 0) { ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Removed {0} empty document{1} from batch.", empty, (empty > 1) ? "s" : "")); } // Remove Email Body on batch empty (if enabled) if (profile.BodyConversion == BodyConversion.OnBatchEmpty) { // Check if the count of body documents equals the total count // If not, this means there is at least 1 non-body document therefore // the body documents can be removed from the batch if (batch.Documents.Count(d => d.IsBody) != batch.Documents.Count) { // Remove the email body from the batch batch.Documents.RemoveAll(d => d.IsBody); ConfigLogger.Instance.LogInfo(email.EmailID, "Removed email body from batch."); } } // Check for empty batch after removing documents if (batch.Documents.Any()) { // Create the output batch.CreateOutput(); // Update the tracking table to mark the email as complete UpdateStatus(EmailStatus.Complete, errors); ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Completed."); } else { HandleEmptyBatch(batch); } } } else { HandleEmptyBatch(batch); } } catch (Exception e) { // Add custom data for logging purposes e.Data.Add("MailboxGUID", profile.MailboxGUID); // Log the error ConfigLogger.Instance.LogError(email.EmailID, e); // Create the error xml based on the exception thrown var errors = new XElement("Errors", new XAttribute("retry", "true"), new XElement("Error", new XAttribute("key", "MailMessage.msg"), new XAttribute("fileName", "MailMessage.msg"), new XAttribute("reason", "Unknown error."), new XAttribute("message", RemoveInvalidXmlChars(e.ToString())), new XAttribute("action", profile.ErrorHandling.Unknown) ) ); // If the batch was created, include document level errors if (batch != null) { errors.Element("Errors").Add(from document in batch.Documents.Where(d => !String.IsNullOrWhiteSpace(d.ErrorMessage)) select new XElement("Error", new XAttribute("key", Path.GetFileName(document.AttachmentFile)), new XAttribute("fileName", document.OriginalName), new XAttribute("reason", document.FailureReason), new XAttribute("message", RemoveInvalidXmlChars(document.ErrorMessage)), new XAttribute("action", document.ActionToTake.Value) ) ); } // Update the email status UpdateStatus(EmailStatus.Error, errors); // Delete the working folder if (batch != null) { try { FileSystemHelper.DeleteDirectory(batch.OutputPath, true); } catch (Exception ex) { ConfigLogger.Instance.LogWarning(email.EmailID, ex); } } ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Failed."); } finally { Interlocked.Decrement(ref count); } }
private int DownloadEmail(Imap imap, MailboxProfile profile, String storagePath) { int count = 0; // Build the MailQuery var query = new MailQuery(String.IsNullOrEmpty(profile.ImapQuery) ? "('Deleted' = 'False')" : String.Format("({0}&('Deleted' = 'False'))", profile.ImapQuery)); // Get all messages matching to the query var infos = imap.ListMessages(query); // If there are any messages to process, then process them if (infos.Any()) { ConfigLogger.Instance.LogInfo("ImapCollector", String.Format("Downloading {0} message{1} from {2}.", infos.Count, infos.Count == 1 ? "" : "s", profile.Description)); // Download each message foreach (var info in infos) { if (!timer.Enabled) { break; } // Just check to ensure its valid if (info.Deleted || String.IsNullOrWhiteSpace(info.UniqueId)) { continue; } // Track the start time for debug purposes var start = DateTime.Now; MailMessage message = null; try { // Download the message message = imap.FetchMessage(info.UniqueId); // Calculate the time taken to fetch the message var fetchTime = DateTime.Now.Subtract(start); // Process the message (So long as the fetch succeeded) if (message != null) { // Setup the data context using (var ctx = new EmailImportDataContext()) { long emailID = 0; // Truncate the subject to avoid data commit errors message.Subject = Truncate(message.Subject, 500); // Check for duplicate if (IsDuplicate(ctx, profile.MailboxGUID, message, ref emailID)) { // Log the duplicate error ConfigLogger.Instance.LogWarning("ImapCollector", String.Format("Message already downloaded, moved to duplicate folder (existing EmailID = {0}).", emailID)); // Move the message to the duplicate sub folder imap.MoveMessage(info.UniqueId, "Duplicate", true, false); } else { // Create an instance of the email database object var email = new Email(); // Assign properties email.MailboxGUID = profile.MailboxGUID; email.DateSent = message.DateSent(); email.DateReceived = message.DateReceived(); email.From = message.From.GetAddressOrDisplayName(); email.MessageID = message.MessageId; if (CreditCardHelper.ExistsCCNumber(message.Subject)) { email.Subject = CreditCardHelper.MaskCCNumbers(message.Subject, '#'); } else { email.Subject = message.Subject; } email.Timestamp = DateTime.Now; // Create the dated storage path var path = Path.Combine(storagePath, email.Timestamp.Value.ToString("yyyyMMdd")); FileSystemHelper.CreateDirectory(path); // Insert the new record ctx.Emails.InsertOnSubmit(email); // Submit the email (we need to get the email ID) using (TransactionScope scope = new TransactionScope()) { // Initial submit of changes ctx.SubmitChanges(); // Build the mail message file name email.MessageFilePath = Path.Combine(path, String.Format("{0:00000000}.eml", email.EmailID)); // Save to disk (delete anything that already exists) message.Save(email.MessageFilePath, MessageFormat.Eml); // Get the batch number - THIS SHOULD NEVER HAPPEN IN A MULTI THREAD SCENARIO WITHOUT A LOCK var batchNumber = ctx.BatchNumbers.SingleOrDefault(b => b.Group == profile.Group); // If there is no batchNumber defined yet, create and insert one if (batchNumber == null) { batchNumber = new BatchNumber(); batchNumber.Group = profile.Group; ctx.BatchNumbers.InsertOnSubmit(batchNumber); } // Init to 0 if null if (!batchNumber.Value.HasValue) { batchNumber.Value = 0; } // Set the new batch number to this email email.BatchNumber = String.Format(String.IsNullOrWhiteSpace(profile.BatchNumberFormat) ? "{0:00000000}" : profile.BatchNumberFormat, ++batchNumber.Value); // Final submit of updates ctx.SubmitChanges(); // Finally, commit to the database scope.Complete(); } // Move the email to the archive (if this fails, but the download is complete this // will just result in a duplicate next time round if the deleted flag is not set) imap.MoveMessage(info.UniqueId, "Archive", true, false); // Log message level download stats ConfigLogger.Instance.LogDebug("ImapCollector", String.Format("Message downloaded (EmailID = {0}, Fetch Time = {1}s, Total Time = {2}s).", email.EmailID, (int)fetchTime.TotalSeconds, (int)DateTime.Now.Subtract(start).TotalSeconds)); // Increment the download count count++; } } } } catch (OutOfMemoryException) { throw; } catch (Exception e) { ConfigLogger.Instance.LogError("ImapCollector", e); } finally { if (message != null) { message.Dispose(); } } } } return(count); }