Пример #1
0
        private Boolean IsDuplicate(EmailImportDataContext ctx, Guid guid, MailMessage message, ref long emailID)
        {
            var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject;

            var email = ctx.Emails.FirstOrDefault(e => Object.Equals(e.MailboxGUID, guid) &&
                                                  Object.Equals(e.MessageID, message.MessageId) &&
                                                  Object.Equals(e.From, message.From.GetAddressOrDisplayName()) &&
                                                  Object.Equals(e.DateSent, message.DateSent()) &&
                                                  Object.Equals(e.DateReceived, message.DateReceived()) &&
                                                  Object.Equals(e.Subject, subject));

            if (email != null)
            {
                emailID = email.EmailID;
            }

            return(email != null);
        }
Пример #2
0
        private void Worker(Object waiter)
        {
            Batch batch = null;

            Interlocked.Increment(ref count);

            ((ManualResetEvent)waiter).Set();

            try
            {
                var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject;

                ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Started...");
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Profile:      {0}", profile.Description));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Mailbox GUID: {0}", profile.MailboxGUID));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Message-ID:   {0}", message.MessageId));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Subject:      {0}", subject.Replace("\"", "'")));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> From:         {0}", message.From.GetAddressOrDisplayName().Replace("\"", "'")));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Received:     {0}", message.DateReceived()));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Batch Number: {0}", email.BatchNumber));
                ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Email-ID:     {0}", email.EmailID));

                batch = new Batch(email, message, profile);

                if (batch.Documents.Any())
                {
                    var startTime = DateTime.Now;

                    foreach (var document in batch.Documents)
                    {
                        // Get the filetype config element if one is defined
                        var fileType = profile.FileTypes.FirstOrDefault(f => Regex.IsMatch(document.OriginalName, f.Pattern ?? string.Empty, RegexOptions.IgnoreCase));

                        // Setup common properties
                        document.Resolution            = profile.Resolution;
                        document.BitDepth              = (fileType != null && fileType.BitDepth.GetValueOrDefault() > 0) ? fileType.BitDepth.GetValueOrDefault() : profile.BitDepth;
                        document.Passthrough           = (profile.AttachmentConversion == AttachmentConversion.Passthrough && !document.IsBody) || (fileType != null && fileType.Passthrough.GetValueOrDefault());
                        document.Mailbox               = profile.ImapUserName;
                        document.BinarisationAlgorithm = (fileType != null) ? fileType.BinarisationAlgorithm.GetValueOrDefault(profile.BinarisationAlgorithm) : profile.BinarisationAlgorithm;
                        document.AutoDeskew            = (fileType != null) ? fileType.AutoDeskew : null;
                        document.AutoRotate            = (fileType != null) ? fileType.AutoRotate : null;
                        document.RemoveFaxHeader       = profile.RemoveFaxHeader;
                        document.ProcessAs             = (fileType != null) ? fileType.ProcessAs : null;
                        document.PdfConversion         = profile.PdfConversion;

                        if (!document.Passthrough)
                        {
                            document.ErrorHandling.Unknown       = profile.ErrorHandling.Unknown;
                            document.ErrorHandling.Unsupported   = profile.ErrorHandling.Unsupported;
                            document.ErrorHandling.Unprocessable = profile.ErrorHandling.Unprocessable;

                            // Check for error handling overrides
                            if (email.Errors != null)
                            {
                                var error = email.Errors.Elements().Where(e => (String)e.Attribute("key") == Path.GetFileName(document.AttachmentFile)).FirstOrDefault();

                                if (error != null)
                                {
                                    ErrorHandlingActions value;

                                    if (Enum.TryParse <ErrorHandlingActions>((String)error.Attribute("override"), out value))
                                    {
                                        document.ErrorHandling.Unknown       = value;
                                        document.ErrorHandling.Unsupported   = value;
                                        document.ErrorHandling.Unprocessable = value;
                                    }

                                    // Set ProcessAs
                                    var processAs = (String)error.Attribute("processAs");

                                    if (!String.IsNullOrWhiteSpace(processAs))
                                    {
                                        document.ProcessAs = processAs;
                                    }
                                }
                            }

                            // Get the converter
                            document.Converter = GetConverter(document);
                        }
                    }

                    // Convert documents in parallel
                    Parallel.ForEach(batch.Documents, document =>
                    {
                        document.Convert();
                    });

                    // XElement to store errors
                    XElement errors = null;

                    // If there are any documents containing error message, add these to an xml document and add to the database
                    if (batch.Documents.Any(d => !String.IsNullOrWhiteSpace(d.ErrorMessage)))
                    {
                        errors = new XElement("Errors",
                                              new XAttribute("retry", batch.Documents.Any(d => d.ShouldRetry && !String.IsNullOrWhiteSpace(d.ErrorMessage))),
                                              from document in batch.Documents.Where(d => !String.IsNullOrEmpty(d.ErrorMessage))
                                              select new XElement("Error",
                                                                  new XAttribute("key", Path.GetFileName(document.AttachmentFile)),
                                                                  new XAttribute("fileName", document.OriginalName),
                                                                  new XAttribute("reason", document.FailureReason),
                                                                  new XAttribute("message", RemoveInvalidXmlChars(document.ErrorMessage)),
                                                                  new XAttribute("action", document.ActionToTake.Value)
                                                                  )
                                              );
                    }

                    // Remove all documents where ActionToTake is Ignore
                    batch.Documents.RemoveAll(d => d.ActionToTake.HasValue && d.ActionToTake.Value == ErrorHandlingActions.Ignore);

                    // If IgnoreErrors is false and one or more documents contains errors,
                    if (batch.Documents.Any(d => d.Success == false))
                    {
                        UpdateStatus(EmailStatus.Error, errors);

                        // Delete the working folder
                        try
                        {
                            FileSystemHelper.DeleteDirectory(batch.OutputPath, true);
                        }
                        catch (Exception e)
                        {
                            ConfigLogger.Instance.LogWarning(email.EmailID, e);
                        }

                        ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Failed.");
                    }
                    else
                    {
                        ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Documents Converted in {0:c}.", DateTime.Now.Subtract(startTime)));

                        // Check and remove blank pages (and documents) if enabled
                        if (profile.RemoveBlankPages)
                        {
                            // No need to check the email body as they will always contain the header details
                            // Blank check only applies to tif images (pdf document via passthrough excluded)
                            foreach (var document in batch.Documents.Where(d => d.Source == DocumentSource.Attachment))
                            {
                                // Remove pages that are blank
                                var blanks = document.Pages.RemoveAll(p => p.IsBlank);

                                // If any blanks, log the number removed
                                if (blanks > 0)
                                {
                                    ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Removed {0} blank page{1} from document {2}.", blanks, (blanks > 1) ? "s" : "", document.RelativePath));
                                }
                            }
                        }

                        // Apply any filtering rules based on resulting image attributes
                        foreach (var document in batch.Documents)
                        {
                            // Get the filetype config element if one is defined
                            var fileType = profile.FileTypes.FirstOrDefault(f => Regex.IsMatch(document.OriginalName, f.Pattern ?? string.Empty, RegexOptions.IgnoreCase));

                            // Only proceed if there is a file type element for the current extension
                            if (fileType != null)
                            {
                                // Remove all pages that fall under the Minimum Pixel threshold
                                var filtered = document.Pages.RemoveAll(p => p.Area < fileType.MinPixels);

                                // If any pages were filtered out, log how many
                                if (filtered > 0)
                                {
                                    ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Filtered out {0} page{1} from document {2}.", filtered, (filtered > 1) ? "s" : "", document.RelativePath));
                                }
                            }
                        }

                        // Remove any empty documents following blank page removal and filtering
                        int empty = batch.Documents.RemoveAll(d => !d.Pages.Any());

                        // If any blanks, log the number removed
                        if (empty > 0)
                        {
                            ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Removed {0} empty document{1} from batch.", empty, (empty > 1) ? "s" : ""));
                        }

                        // Remove Email Body on batch empty (if enabled)
                        if (profile.BodyConversion == BodyConversion.OnBatchEmpty)
                        {
                            // Check if the count of body documents equals the total count
                            // If not, this means there is at least 1 non-body document therefore
                            // the body documents can be removed from the batch
                            if (batch.Documents.Count(d => d.IsBody) != batch.Documents.Count)
                            {
                                // Remove the email body from the batch
                                batch.Documents.RemoveAll(d => d.IsBody);
                                ConfigLogger.Instance.LogInfo(email.EmailID, "Removed email body from batch.");
                            }
                        }

                        // Check for empty batch after removing documents
                        if (batch.Documents.Any())
                        {
                            // Create the output
                            batch.CreateOutput();

                            // Update the tracking table to mark the email as complete
                            UpdateStatus(EmailStatus.Complete, errors);

                            ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Completed.");
                        }
                        else
                        {
                            HandleEmptyBatch(batch);
                        }
                    }
                }
                else
                {
                    HandleEmptyBatch(batch);
                }
            }
            catch (Exception e)
            {
                // Add custom data for logging purposes
                e.Data.Add("MailboxGUID", profile.MailboxGUID);

                // Log the error
                ConfigLogger.Instance.LogError(email.EmailID, e);

                // Create the error xml based on the exception thrown
                var errors = new XElement("Errors",
                                          new XAttribute("retry", "true"),
                                          new XElement("Error",
                                                       new XAttribute("key", "MailMessage.msg"),
                                                       new XAttribute("fileName", "MailMessage.msg"),
                                                       new XAttribute("reason", "Unknown error."),
                                                       new XAttribute("message", RemoveInvalidXmlChars(e.ToString())),
                                                       new XAttribute("action", profile.ErrorHandling.Unknown)
                                                       )
                                          );

                // If the batch was created, include document level errors
                if (batch != null)
                {
                    errors.Element("Errors").Add(from document in batch.Documents.Where(d => !String.IsNullOrWhiteSpace(d.ErrorMessage))
                                                 select new XElement("Error",
                                                                     new XAttribute("key", Path.GetFileName(document.AttachmentFile)),
                                                                     new XAttribute("fileName", document.OriginalName),
                                                                     new XAttribute("reason", document.FailureReason),
                                                                     new XAttribute("message", RemoveInvalidXmlChars(document.ErrorMessage)),
                                                                     new XAttribute("action", document.ActionToTake.Value)
                                                                     )
                                                 );
                }

                // Update the email status
                UpdateStatus(EmailStatus.Error, errors);

                // Delete the working folder
                if (batch != null)
                {
                    try
                    {
                        FileSystemHelper.DeleteDirectory(batch.OutputPath, true);
                    }
                    catch (Exception ex)
                    {
                        ConfigLogger.Instance.LogWarning(email.EmailID, ex);
                    }
                }

                ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Failed.");
            }
            finally
            {
                Interlocked.Decrement(ref count);
            }
        }
Пример #3
0
        private int DownloadEmail(Imap imap, MailboxProfile profile, String storagePath)
        {
            int count = 0;

            // Build the MailQuery
            var query = new MailQuery(String.IsNullOrEmpty(profile.ImapQuery) ? "('Deleted' = 'False')" : String.Format("({0}&('Deleted' = 'False'))", profile.ImapQuery));

            // Get all messages matching to the query
            var infos = imap.ListMessages(query);

            // If there are any messages to process, then process them
            if (infos.Any())
            {
                ConfigLogger.Instance.LogInfo("ImapCollector", String.Format("Downloading {0} message{1} from {2}.", infos.Count, infos.Count == 1 ? "" : "s", profile.Description));

                // Download each message
                foreach (var info in infos)
                {
                    if (!timer.Enabled)
                    {
                        break;
                    }

                    // Just check to ensure its valid
                    if (info.Deleted || String.IsNullOrWhiteSpace(info.UniqueId))
                    {
                        continue;
                    }

                    // Track the start time for debug purposes
                    var start = DateTime.Now;

                    MailMessage message = null;

                    try
                    {
                        // Download the message
                        message = imap.FetchMessage(info.UniqueId);

                        // Calculate the time taken to fetch the message
                        var fetchTime = DateTime.Now.Subtract(start);

                        // Process the message (So long as the fetch succeeded)
                        if (message != null)
                        {
                            // Setup the data context
                            using (var ctx = new EmailImportDataContext())
                            {
                                long emailID = 0;

                                // Truncate the subject to avoid data commit errors
                                message.Subject = Truncate(message.Subject, 500);

                                // Check for duplicate
                                if (IsDuplicate(ctx, profile.MailboxGUID, message, ref emailID))
                                {
                                    // Log the duplicate error
                                    ConfigLogger.Instance.LogWarning("ImapCollector", String.Format("Message already downloaded, moved to duplicate folder (existing EmailID = {0}).", emailID));

                                    // Move the message to the duplicate sub folder
                                    imap.MoveMessage(info.UniqueId, "Duplicate", true, false);
                                }
                                else
                                {
                                    // Create an instance of the email database object
                                    var email = new Email();

                                    // Assign properties
                                    email.MailboxGUID  = profile.MailboxGUID;
                                    email.DateSent     = message.DateSent();
                                    email.DateReceived = message.DateReceived();
                                    email.From         = message.From.GetAddressOrDisplayName();
                                    email.MessageID    = message.MessageId;
                                    if (CreditCardHelper.ExistsCCNumber(message.Subject))
                                    {
                                        email.Subject = CreditCardHelper.MaskCCNumbers(message.Subject, '#');
                                    }
                                    else
                                    {
                                        email.Subject = message.Subject;
                                    }
                                    email.Timestamp = DateTime.Now;

                                    // Create the dated storage path
                                    var path = Path.Combine(storagePath, email.Timestamp.Value.ToString("yyyyMMdd"));
                                    FileSystemHelper.CreateDirectory(path);

                                    // Insert the new record
                                    ctx.Emails.InsertOnSubmit(email);

                                    // Submit the email (we need to get the email ID)
                                    using (TransactionScope scope = new TransactionScope())
                                    {
                                        // Initial submit of changes
                                        ctx.SubmitChanges();

                                        // Build the mail message file name
                                        email.MessageFilePath = Path.Combine(path, String.Format("{0:00000000}.eml", email.EmailID));

                                        // Save to disk (delete anything that already exists)
                                        message.Save(email.MessageFilePath, MessageFormat.Eml);

                                        // Get the batch number - THIS SHOULD NEVER HAPPEN IN A MULTI THREAD SCENARIO WITHOUT A LOCK
                                        var batchNumber = ctx.BatchNumbers.SingleOrDefault(b => b.Group == profile.Group);

                                        // If there is no batchNumber defined yet, create and insert one
                                        if (batchNumber == null)
                                        {
                                            batchNumber       = new BatchNumber();
                                            batchNumber.Group = profile.Group;
                                            ctx.BatchNumbers.InsertOnSubmit(batchNumber);
                                        }

                                        // Init to 0 if null
                                        if (!batchNumber.Value.HasValue)
                                        {
                                            batchNumber.Value = 0;
                                        }

                                        // Set the new batch number to this email
                                        email.BatchNumber = String.Format(String.IsNullOrWhiteSpace(profile.BatchNumberFormat) ? "{0:00000000}" : profile.BatchNumberFormat, ++batchNumber.Value);

                                        // Final submit of updates
                                        ctx.SubmitChanges();

                                        // Finally, commit to the database
                                        scope.Complete();
                                    }

                                    // Move the email to the archive (if this fails, but the download is complete this
                                    // will just result in a duplicate next time round if the deleted flag is not set)
                                    imap.MoveMessage(info.UniqueId, "Archive", true, false);

                                    // Log message level download stats
                                    ConfigLogger.Instance.LogDebug("ImapCollector", String.Format("Message downloaded (EmailID = {0}, Fetch Time = {1}s, Total Time = {2}s).", email.EmailID, (int)fetchTime.TotalSeconds, (int)DateTime.Now.Subtract(start).TotalSeconds));

                                    // Increment the download count
                                    count++;
                                }
                            }
                        }
                    }
                    catch (OutOfMemoryException)
                    {
                        throw;
                    }
                    catch (Exception e)
                    {
                        ConfigLogger.Instance.LogError("ImapCollector", e);
                    }
                    finally
                    {
                        if (message != null)
                        {
                            message.Dispose();
                        }
                    }
                }
            }

            return(count);
        }