private IEnumerable <Email> GetErrorEmails()
{
List <Email> emails = new List <Email>();
using (var ctx = new EmailImportDataContext())
{
foreach (var profile in Settings.MailboxProfiles.Values.Where(m => m.IsActive))
{
try
{
var errors = from email in ctx.Emails
where (email.MailboxGUID == profile.MailboxGUID) &&
(email.InProgress == null || email.InProgress == false) &&
(email.Status == "Error")
select email;
emails.AddRange(errors.ToList().Where(e => e.ProcessedCount.GetValueOrDefault() > profile.MaximumRetries || (e.Errors != null && (String)e.Errors.Attribute("retry") == "false")));
}
catch (Exception e)
{
// Add custom data for logging purposes
e.Data.Add("MailboxGUID", profile.MailboxGUID);
// Log the error
ConfigLogger.Instance.LogError("EmailMonitor", e);
}
}
}
return(emails);
}
private IEnumerable <Email> GetEmails()
{
List <Email> emails = new List <Email>();
using (var ctx = new EmailImportDataContext())
{
foreach (var profile in Settings.MailboxProfiles.Values.Where(m => m.IsActive))
{
var retryTime = DateTime.Now.Subtract(profile.TimeBetweenRetries);
try
{
// Get email with status == null || (status == error && lastProcessed > timespan)
emails.AddRange((from email in ctx.Emails
where (email.MailboxGUID == profile.MailboxGUID) &&
(email.InProgress == null || email.InProgress == false) &&
((email.Status == null) ||
(email.Status == "Error" && (email.EndTime == null || email.EndTime.Value < retryTime)))
select email));
}
catch (Exception e)
{
// Add custom data for logging purposes
e.Data.Add("MailboxGUID", profile.MailboxGUID);
// Log the error
ConfigLogger.Instance.LogError("EmailMonitor", e);
}
}
}
return(emails.OrderBy(e => e.EmailID));
}
private IEnumerable <Email> GetConversionErrorEmails()
{
List <Email> emails = new List <Email>();
using (var ctx = new EmailImportDataContext())
{
foreach (var profile in Settings.MailboxProfiles.Values.Where(m => m.IsActive))
{
try
{
var errors = from email in ctx.Emails
where (email.MailboxGUID == profile.MailboxGUID && profile.ProcessErrorBatch == true) &&
(email.InProgress == null || email.InProgress == false) &&
(email.Status == "Rejected" || email.Status == "Ignored")
select email;
emails.AddRange(errors.ToList());
}
catch (Exception e)
{
// Add custom data for logging purposes
e.Data.Add("MailboxGUID", profile.MailboxGUID);
// Log the error
ConfigLogger.Instance.LogError("EmailMonitor", e);
}
}
}
return(emails);
}
public static void LoadMailboxProfiles()
{
MailboxProfiles.Clear();
XmlSerializer serializer = new XmlSerializer(typeof(MailboxProfile));
using (var ctx = new EmailImportDataContext())
{
foreach (var mailbox in ctx.Mailboxes)
{
using (TextReader reader = new StringReader(mailbox.ProfileObject))
{
var profile = (MailboxProfile)serializer.Deserialize(reader);
if (profile.Enabled)
{
try
{
profile.ParseScript();
}
catch (Exception e)
{
ConfigLogger.Instance.LogError(e);
}
MailboxProfiles.Add(mailbox.MailboxGUID, profile);
}
}
}
}
}
private void ResetInProgress()
{
using (var ctx = new EmailImportDataContext())
{
foreach (var email in ctx.Emails.Where(e => e.InProgress == true))
{
email.InProgress = null;
}
ctx.SubmitChanges();
}
}
private void UpdateStatus(EmailStatus status, XElement errors)
{
int count = 0;
while (true)
{
try
{
using (EmailImportDataContext ctx = new EmailImportDataContext())
{
var eml = ctx.Emails.Single(e => e.EmailID == email.EmailID);
if (status == EmailStatus.InProgress)
{
eml.InProgress = true;
eml.StartTime = DateTime.Now;
eml.EndTime = null;
}
else
{
eml.Status = status.ToString();
eml.ProcessedCount = eml.ProcessedCount.GetValueOrDefault() + 1;
eml.EndTime = DateTime.Now;
eml.InProgress = null;
eml.Errors = errors;
}
ctx.SubmitChanges();
}
break;
}
catch (Exception e)
{
// Increment the retry counter
count++;
// Log the exception
ConfigLogger.Instance.Log(count < 5 ? LogSeverity.Warning : LogSeverity.Error, email.EmailID, e);
// Retry up to 5 times
if (count >= 5)
{
break;
}
// Sleep for a fraction
Thread.Sleep(50);
}
}
}
private Boolean IsDuplicate(EmailImportDataContext ctx, Guid guid, MailMessage message, ref long emailID)
{
var subject = CreditCardHelper.ExistsCCNumber(message.Subject) ? CreditCardHelper.MaskCCNumbers(message.Subject, '#') : message.Subject;
var email = ctx.Emails.FirstOrDefault(e => Object.Equals(e.MailboxGUID, guid) &&
Object.Equals(e.MessageID, message.MessageId) &&
Object.Equals(e.From, message.From.GetAddressOrDisplayName()) &&
Object.Equals(e.DateSent, message.DateSent()) &&
Object.Equals(e.DateReceived, message.DateReceived()) &&
Object.Equals(e.Subject, subject));
if (email != null)
{
emailID = email.EmailID;
}
return(email != null);
}
private void HouseKeeping()
{
ConfigLogger.Instance.LogInfo("EmailMonitor", "Performing house keeping tasks...");
try
{
using (var ctx = new EmailImportDataContext())
{
// Loop through each profile
foreach (var profile in Settings.MailboxProfiles.Values)
{
try
{
// Determine date to purge from based on retention setting
var date = DateTime.Today.AddDays(0 - Math.Abs(profile.StorageRetention));
// Get emails to be purged
var emails = ctx.Emails.Where(e => (e.MailboxGUID == profile.MailboxGUID) &&
(e.InProgress == null || e.InProgress == false) &&
(e.Status == "Complete" || e.Status == "Empty" || e.Status == "Rejected" || e.Status == "Ignored") &&
(e.EndTime != null && e.EndTime.Value < date)).ToList();
// Delete each email msg file
foreach (var email in emails)
{
File.Delete(email.MessageFilePath);
}
// Delete the email entries from the Emails table
ctx.Emails.DeleteAllOnSubmit(emails);
// Submit the changes
ctx.SubmitChanges();
// Log how many emails were deleted
ConfigLogger.Instance.LogInfo("EmailMonitor", String.Format("Deleted {0} stored email{1} from {2}.", emails.Count, emails.Count == 1 ? "" : "s", profile.Description));
// Remove empty folders from the storage path
RemoveEmptyFolders(profile.StoragePath);
// Archive path deletion
if (!String.IsNullOrWhiteSpace(profile.ArchivePath) && Directory.Exists(profile.ArchivePath))
{
// Determine date to purge from based on retention setting
date = DateTime.Today.AddDays(0 - Math.Abs(profile.ArchiveRetention));
// Delete dated archive folders earlier than the retention date
foreach (var folder in Directory.GetDirectories(profile.ArchivePath))
{
DateTime dateValue;
if (DateTime.TryParseExact(Path.GetFileName(folder), "yyyyMMdd", null, System.Globalization.DateTimeStyles.None, out dateValue))
{
if (dateValue < date)
{
Directory.Delete(folder, true);
ConfigLogger.Instance.LogInfo("EmailMonitor", String.Format("Deleted archive folder {0}.", folder));
}
}
}
}
}
catch (Exception e)
{
// Add custom data for logging purposes
e.Data.Add("MailboxGUID", profile.MailboxGUID);
// Log the warning (not error)
ConfigLogger.Instance.LogWarning("EmailMonitor", e);
}
}
}
// Remove empty folders from the default storage path
RemoveEmptyFolders(Settings.DefaultStoragePath);
}
catch (Exception e)
{
// We don't want housekeeping to cause a failure, but we want someone to
// know about it so that they can investigate the issue
ConfigLogger.Instance.LogWarning("EmailMonitor", e);
}
}
private int DownloadEmail(Imap imap, MailboxProfile profile, String storagePath)
{
int count = 0;
// Build the MailQuery
var query = new MailQuery(String.IsNullOrEmpty(profile.ImapQuery) ? "('Deleted' = 'False')" : String.Format("({0}&('Deleted' = 'False'))", profile.ImapQuery));
// Get all messages matching to the query
var infos = imap.ListMessages(query);
// If there are any messages to process, then process them
if (infos.Any())
{
ConfigLogger.Instance.LogInfo("ImapCollector", String.Format("Downloading {0} message{1} from {2}.", infos.Count, infos.Count == 1 ? "" : "s", profile.Description));
// Download each message
foreach (var info in infos)
{
if (!timer.Enabled)
{
break;
}
// Just check to ensure its valid
if (info.Deleted || String.IsNullOrWhiteSpace(info.UniqueId))
{
continue;
}
// Track the start time for debug purposes
var start = DateTime.Now;
MailMessage message = null;
try
{
// Download the message
message = imap.FetchMessage(info.UniqueId);
// Calculate the time taken to fetch the message
var fetchTime = DateTime.Now.Subtract(start);
// Process the message (So long as the fetch succeeded)
if (message != null)
{
// Setup the data context
using (var ctx = new EmailImportDataContext())
{
long emailID = 0;
// Truncate the subject to avoid data commit errors
message.Subject = Truncate(message.Subject, 500);
// Check for duplicate
if (IsDuplicate(ctx, profile.MailboxGUID, message, ref emailID))
{
// Log the duplicate error
ConfigLogger.Instance.LogWarning("ImapCollector", String.Format("Message already downloaded, moved to duplicate folder (existing EmailID = {0}).", emailID));
// Move the message to the duplicate sub folder
imap.MoveMessage(info.UniqueId, "Duplicate", true, false);
}
else
{
// Create an instance of the email database object
var email = new Email();
// Assign properties
email.MailboxGUID = profile.MailboxGUID;
email.DateSent = message.DateSent();
email.DateReceived = message.DateReceived();
email.From = message.From.GetAddressOrDisplayName();
email.MessageID = message.MessageId;
if (CreditCardHelper.ExistsCCNumber(message.Subject))
{
email.Subject = CreditCardHelper.MaskCCNumbers(message.Subject, '#');
}
else
{
email.Subject = message.Subject;
}
email.Timestamp = DateTime.Now;
// Create the dated storage path
var path = Path.Combine(storagePath, email.Timestamp.Value.ToString("yyyyMMdd"));
FileSystemHelper.CreateDirectory(path);
// Insert the new record
ctx.Emails.InsertOnSubmit(email);
// Submit the email (we need to get the email ID)
using (TransactionScope scope = new TransactionScope())
{
// Initial submit of changes
ctx.SubmitChanges();
// Build the mail message file name
email.MessageFilePath = Path.Combine(path, String.Format("{0:00000000}.eml", email.EmailID));
// Save to disk (delete anything that already exists)
message.Save(email.MessageFilePath, MessageFormat.Eml);
// Get the batch number - THIS SHOULD NEVER HAPPEN IN A MULTI THREAD SCENARIO WITHOUT A LOCK
var batchNumber = ctx.BatchNumbers.SingleOrDefault(b => b.Group == profile.Group);
// If there is no batchNumber defined yet, create and insert one
if (batchNumber == null)
{
batchNumber = new BatchNumber();
batchNumber.Group = profile.Group;
ctx.BatchNumbers.InsertOnSubmit(batchNumber);
}
// Init to 0 if null
if (!batchNumber.Value.HasValue)
{
batchNumber.Value = 0;
}
// Set the new batch number to this email
email.BatchNumber = String.Format(String.IsNullOrWhiteSpace(profile.BatchNumberFormat) ? "{0:00000000}" : profile.BatchNumberFormat, ++batchNumber.Value);
// Final submit of updates
ctx.SubmitChanges();
// Finally, commit to the database
scope.Complete();
}
// Move the email to the archive (if this fails, but the download is complete this
// will just result in a duplicate next time round if the deleted flag is not set)
imap.MoveMessage(info.UniqueId, "Archive", true, false);
// Log message level download stats
ConfigLogger.Instance.LogDebug("ImapCollector", String.Format("Message downloaded (EmailID = {0}, Fetch Time = {1}s, Total Time = {2}s).", email.EmailID, (int)fetchTime.TotalSeconds, (int)DateTime.Now.Subtract(start).TotalSeconds));
// Increment the download count
count++;
}
}
}
}
catch (OutOfMemoryException)
{
throw;
}
catch (Exception e)
{
ConfigLogger.Instance.LogError("ImapCollector", e);
}
finally
{
if (message != null)
{
message.Dispose();
}
}
}
}
return(count);
}
static Settings()
{
#region General Settings
using (var ctx = new EmailImportDataContext())
{
try
{
ImapCollectorInterval = TimeSpan.Parse(ctx.Settings.Single(s => s.Name == "Interval.ImapCollector").Value);
}
catch
{
ImapCollectorInterval = TimeSpan.FromMinutes(5);
}
try
{
EmailMonitorInterval = TimeSpan.Parse(ctx.Settings.Single(s => s.Name == "Interval.EmailMonitor").Value);
}
catch
{
EmailMonitorInterval = TimeSpan.FromMinutes(1);
}
try
{
ConcurrencyLevel = int.Parse(ctx.Settings.Single(s => s.Name == "ConcurrencyLevel").Value);
}
catch
{
ConcurrencyLevel = Environment.ProcessorCount;
}
try
{
SmtpSizeLimit = int.Parse(ctx.Settings.Single(s => s.Name == "SmtpSizeLimit").Value) * 1024 * 1024;
}
catch
{ }
try
{
LiteViewerAuthorisedPCs = ctx.Settings.Single(s => s.Name == "LiteViewerAuthorisedPCs").Value;
}
catch
{
LiteViewerAuthorisedPCs = null;
}
if (SmtpSizeLimit <= 0)
{
SmtpSizeLimit = int.MaxValue;
}
var setting = ctx.Settings.SingleOrDefault(s => s.Name == "DefaultStoragePath");
DefaultStoragePath = (setting == null) ? null : setting.Value;
FileSystemHelper.CreateDirectory(DefaultStoragePath);
setting = ctx.Settings.SingleOrDefault(s => s.Name == "DefaultEscalationEmail");
DefaultEscalationEmail = (setting == null) ? null : setting.Value;
if (String.IsNullOrWhiteSpace(DefaultEscalationEmail))
{
throw new Exception("DefaultEscalationEmail setting must contain a valid email address.");
}
}
#endregion
#region Mailboxes
MailboxProfiles = new Dictionary <Guid, MailboxProfile>();
LoadMailboxProfiles();
#endregion
}
