private Boolean IsDuplicate(EmailImportDataContext ctx, Guid guid, MailMessage message, ref long emailID)
{
var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject;
var email = ctx.Emails.FirstOrDefault(e => Object.Equals(e.MailboxGUID, guid) &&
Object.Equals(e.MessageID, message.MessageId) &&
Object.Equals(e.From, message.From.GetAddressOrDisplayName()) &&
Object.Equals(e.DateSent, message.DateSent()) &&
Object.Equals(e.DateReceived, message.DateReceived()) &&
Object.Equals(e.Subject, subject));
if (email != null)
{
emailID = email.EmailID;
}
return(email != null);
}
private void Worker(Object waiter)
{
Batch batch = null;
Interlocked.Increment(ref count);
((ManualResetEvent)waiter).Set();
try
{
var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject;
ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Started...");
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Profile: {0}", profile.Description));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Mailbox GUID: {0}", profile.MailboxGUID));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Message-ID: {0}", message.MessageId));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Subject: {0}", subject.Replace("\"", "'")));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> From: {0}", message.From.GetAddressOrDisplayName().Replace("\"", "'")));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Received: {0}", message.DateReceived()));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Batch Number: {0}", email.BatchNumber));
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format(" >> Email-ID: {0}", email.EmailID));
batch = new Batch(email, message, profile);
if (batch.Documents.Any())
{
var startTime = DateTime.Now;
foreach (var document in batch.Documents)
{
// Get the filetype config element if one is defined
var fileType = profile.FileTypes.FirstOrDefault(f => Regex.IsMatch(document.OriginalName, f.Pattern ?? string.Empty, RegexOptions.IgnoreCase));
// Setup common properties
document.Resolution = profile.Resolution;
document.BitDepth = (fileType != null && fileType.BitDepth.GetValueOrDefault() > 0) ? fileType.BitDepth.GetValueOrDefault() : profile.BitDepth;
document.Passthrough = (profile.AttachmentConversion == AttachmentConversion.Passthrough && !document.IsBody) || (fileType != null && fileType.Passthrough.GetValueOrDefault());
document.Mailbox = profile.ImapUserName;
document.BinarisationAlgorithm = (fileType != null) ? fileType.BinarisationAlgorithm.GetValueOrDefault(profile.BinarisationAlgorithm) : profile.BinarisationAlgorithm;
document.AutoDeskew = (fileType != null) ? fileType.AutoDeskew : null;
document.AutoRotate = (fileType != null) ? fileType.AutoRotate : null;
document.RemoveFaxHeader = profile.RemoveFaxHeader;
document.ProcessAs = (fileType != null) ? fileType.ProcessAs : null;
document.PdfConversion = profile.PdfConversion;
if (!document.Passthrough)
{
document.ErrorHandling.Unknown = profile.ErrorHandling.Unknown;
document.ErrorHandling.Unsupported = profile.ErrorHandling.Unsupported;
document.ErrorHandling.Unprocessable = profile.ErrorHandling.Unprocessable;
// Check for error handling overrides
if (email.Errors != null)
{
var error = email.Errors.Elements().Where(e => (String)e.Attribute("key") == Path.GetFileName(document.AttachmentFile)).FirstOrDefault();
if (error != null)
{
ErrorHandlingActions value;
if (Enum.TryParse <ErrorHandlingActions>((String)error.Attribute("override"), out value))
{
document.ErrorHandling.Unknown = value;
document.ErrorHandling.Unsupported = value;
document.ErrorHandling.Unprocessable = value;
}
// Set ProcessAs
var processAs = (String)error.Attribute("processAs");
if (!String.IsNullOrWhiteSpace(processAs))
{
document.ProcessAs = processAs;
}
}
}
// Get the converter
document.Converter = GetConverter(document);
}
}
// Convert documents in parallel
Parallel.ForEach(batch.Documents, document =>
{
document.Convert();
});
// XElement to store errors
XElement errors = null;
// If there are any documents containing error message, add these to an xml document and add to the database
if (batch.Documents.Any(d => !String.IsNullOrWhiteSpace(d.ErrorMessage)))
{
errors = new XElement("Errors",
new XAttribute("retry", batch.Documents.Any(d => d.ShouldRetry && !String.IsNullOrWhiteSpace(d.ErrorMessage))),
from document in batch.Documents.Where(d => !String.IsNullOrEmpty(d.ErrorMessage))
select new XElement("Error",
new XAttribute("key", Path.GetFileName(document.AttachmentFile)),
new XAttribute("fileName", document.OriginalName),
new XAttribute("reason", document.FailureReason),
new XAttribute("message", RemoveInvalidXmlChars(document.ErrorMessage)),
new XAttribute("action", document.ActionToTake.Value)
)
);
}
// Remove all documents where ActionToTake is Ignore
batch.Documents.RemoveAll(d => d.ActionToTake.HasValue && d.ActionToTake.Value == ErrorHandlingActions.Ignore);
// If IgnoreErrors is false and one or more documents contains errors,
if (batch.Documents.Any(d => d.Success == false))
{
UpdateStatus(EmailStatus.Error, errors);
// Delete the working folder
try
{
FileSystemHelper.DeleteDirectory(batch.OutputPath, true);
}
catch (Exception e)
{
ConfigLogger.Instance.LogWarning(email.EmailID, e);
}
ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Failed.");
}
else
{
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Documents Converted in {0:c}.", DateTime.Now.Subtract(startTime)));
// Check and remove blank pages (and documents) if enabled
if (profile.RemoveBlankPages)
{
// No need to check the email body as they will always contain the header details
// Blank check only applies to tif images (pdf document via passthrough excluded)
foreach (var document in batch.Documents.Where(d => d.Source == DocumentSource.Attachment))
{
// Remove pages that are blank
var blanks = document.Pages.RemoveAll(p => p.IsBlank);
// If any blanks, log the number removed
if (blanks > 0)
{
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Removed {0} blank page{1} from document {2}.", blanks, (blanks > 1) ? "s" : "", document.RelativePath));
}
}
}
// Apply any filtering rules based on resulting image attributes
foreach (var document in batch.Documents)
{
// Get the filetype config element if one is defined
var fileType = profile.FileTypes.FirstOrDefault(f => Regex.IsMatch(document.OriginalName, f.Pattern ?? string.Empty, RegexOptions.IgnoreCase));
// Only proceed if there is a file type element for the current extension
if (fileType != null)
{
// Remove all pages that fall under the Minimum Pixel threshold
var filtered = document.Pages.RemoveAll(p => p.Area < fileType.MinPixels);
// If any pages were filtered out, log how many
if (filtered > 0)
{
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Filtered out {0} page{1} from document {2}.", filtered, (filtered > 1) ? "s" : "", document.RelativePath));
}
}
}
// Remove any empty documents following blank page removal and filtering
int empty = batch.Documents.RemoveAll(d => !d.Pages.Any());
// If any blanks, log the number removed
if (empty > 0)
{
ConfigLogger.Instance.LogInfo(email.EmailID, String.Format("Removed {0} empty document{1} from batch.", empty, (empty > 1) ? "s" : ""));
}
// Remove Email Body on batch empty (if enabled)
if (profile.BodyConversion == BodyConversion.OnBatchEmpty)
{
// Check if the count of body documents equals the total count
// If not, this means there is at least 1 non-body document therefore
// the body documents can be removed from the batch
if (batch.Documents.Count(d => d.IsBody) != batch.Documents.Count)
{
// Remove the email body from the batch
batch.Documents.RemoveAll(d => d.IsBody);
ConfigLogger.Instance.LogInfo(email.EmailID, "Removed email body from batch.");
}
}
// Check for empty batch after removing documents
if (batch.Documents.Any())
{
// Create the output
batch.CreateOutput();
// Update the tracking table to mark the email as complete
UpdateStatus(EmailStatus.Complete, errors);
ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Completed.");
}
else
{
HandleEmptyBatch(batch);
}
}
}
else
{
HandleEmptyBatch(batch);
}
}
catch (Exception e)
{
// Add custom data for logging purposes
e.Data.Add("MailboxGUID", profile.MailboxGUID);
// Log the error
ConfigLogger.Instance.LogError(email.EmailID, e);
// Create the error xml based on the exception thrown
var errors = new XElement("Errors",
new XAttribute("retry", "true"),
new XElement("Error",
new XAttribute("key", "MailMessage.msg"),
new XAttribute("fileName", "MailMessage.msg"),
new XAttribute("reason", "Unknown error."),
new XAttribute("message", RemoveInvalidXmlChars(e.ToString())),
new XAttribute("action", profile.ErrorHandling.Unknown)
)
);
// If the batch was created, include document level errors
if (batch != null)
{
errors.Element("Errors").Add(from document in batch.Documents.Where(d => !String.IsNullOrWhiteSpace(d.ErrorMessage))
select new XElement("Error",
new XAttribute("key", Path.GetFileName(document.AttachmentFile)),
new XAttribute("fileName", document.OriginalName),
new XAttribute("reason", document.FailureReason),
new XAttribute("message", RemoveInvalidXmlChars(document.ErrorMessage)),
new XAttribute("action", document.ActionToTake.Value)
)
);
}
// Update the email status
UpdateStatus(EmailStatus.Error, errors);
// Delete the working folder
if (batch != null)
{
try
{
FileSystemHelper.DeleteDirectory(batch.OutputPath, true);
}
catch (Exception ex)
{
ConfigLogger.Instance.LogWarning(email.EmailID, ex);
}
}
ConfigLogger.Instance.LogInfo(email.EmailID, "Batch Conversion Failed.");
}
finally
{
Interlocked.Decrement(ref count);
}
}
private int DownloadEmail(Imap imap, MailboxProfile profile, String storagePath)
{
int count = 0;
// Build the MailQuery
var query = new MailQuery(String.IsNullOrEmpty(profile.ImapQuery) ? "('Deleted' = 'False')" : String.Format("({0}&('Deleted' = 'False'))", profile.ImapQuery));
// Get all messages matching to the query
var infos = imap.ListMessages(query);
// If there are any messages to process, then process them
if (infos.Any())
{
ConfigLogger.Instance.LogInfo("ImapCollector", String.Format("Downloading {0} message{1} from {2}.", infos.Count, infos.Count == 1 ? "" : "s", profile.Description));
// Download each message
foreach (var info in infos)
{
if (!timer.Enabled)
{
break;
}
// Just check to ensure its valid
if (info.Deleted || String.IsNullOrWhiteSpace(info.UniqueId))
{
continue;
}
// Track the start time for debug purposes
var start = DateTime.Now;
MailMessage message = null;
try
{
// Download the message
message = imap.FetchMessage(info.UniqueId);
// Calculate the time taken to fetch the message
var fetchTime = DateTime.Now.Subtract(start);
// Process the message (So long as the fetch succeeded)
if (message != null)
{
// Setup the data context
using (var ctx = new EmailImportDataContext())
{
long emailID = 0;
// Truncate the subject to avoid data commit errors
message.Subject = Truncate(message.Subject, 500);
// Check for duplicate
if (IsDuplicate(ctx, profile.MailboxGUID, message, ref emailID))
{
// Log the duplicate error
ConfigLogger.Instance.LogWarning("ImapCollector", String.Format("Message already downloaded, moved to duplicate folder (existing EmailID = {0}).", emailID));
// Move the message to the duplicate sub folder
imap.MoveMessage(info.UniqueId, "Duplicate", true, false);
}
else
{
// Create an instance of the email database object
var email = new Email();
// Assign properties
email.MailboxGUID = profile.MailboxGUID;
email.DateSent = message.DateSent();
email.DateReceived = message.DateReceived();
email.From = message.From.GetAddressOrDisplayName();
email.MessageID = message.MessageId;
if (CreditCardHelper.ExistsCCNumber(message.Subject))
{
email.Subject = CreditCardHelper.MaskCCNumbers(message.Subject, '#');
}
else
{
email.Subject = message.Subject;
}
email.Timestamp = DateTime.Now;
// Create the dated storage path
var path = Path.Combine(storagePath, email.Timestamp.Value.ToString("yyyyMMdd"));
FileSystemHelper.CreateDirectory(path);
// Insert the new record
ctx.Emails.InsertOnSubmit(email);
// Submit the email (we need to get the email ID)
using (TransactionScope scope = new TransactionScope())
{
// Initial submit of changes
ctx.SubmitChanges();
// Build the mail message file name
email.MessageFilePath = Path.Combine(path, String.Format("{0:00000000}.eml", email.EmailID));
// Save to disk (delete anything that already exists)
message.Save(email.MessageFilePath, MessageFormat.Eml);
// Get the batch number - THIS SHOULD NEVER HAPPEN IN A MULTI THREAD SCENARIO WITHOUT A LOCK
var batchNumber = ctx.BatchNumbers.SingleOrDefault(b => b.Group == profile.Group);
// If there is no batchNumber defined yet, create and insert one
if (batchNumber == null)
{
batchNumber = new BatchNumber();
batchNumber.Group = profile.Group;
ctx.BatchNumbers.InsertOnSubmit(batchNumber);
}
// Init to 0 if null
if (!batchNumber.Value.HasValue)
{
batchNumber.Value = 0;
}
// Set the new batch number to this email
email.BatchNumber = String.Format(String.IsNullOrWhiteSpace(profile.BatchNumberFormat) ? "{0:00000000}" : profile.BatchNumberFormat, ++batchNumber.Value);
// Final submit of updates
ctx.SubmitChanges();
// Finally, commit to the database
scope.Complete();
}
// Move the email to the archive (if this fails, but the download is complete this
// will just result in a duplicate next time round if the deleted flag is not set)
imap.MoveMessage(info.UniqueId, "Archive", true, false);
// Log message level download stats
ConfigLogger.Instance.LogDebug("ImapCollector", String.Format("Message downloaded (EmailID = {0}, Fetch Time = {1}s, Total Time = {2}s).", email.EmailID, (int)fetchTime.TotalSeconds, (int)DateTime.Now.Subtract(start).TotalSeconds));
// Increment the download count
count++;
}
}
}
}
catch (OutOfMemoryException)
{
throw;
}
catch (Exception e)
{
ConfigLogger.Instance.LogError("ImapCollector", e);
}
finally
{
if (message != null)
{
message.Dispose();
}
}
}
}
return(count);
}
