protected BaseWorker(MailBoxManager mailBoxManager, MailBox mailBox, TasksConfig tasksConfig,
CancellationToken cancelToken, ILogger log = null)
{
this.mailBoxManager = mailBoxManager;
_mailBox = mailBox;
this.log = log ?? new NullLogger();
_lastTimeItemChecked = DateTime.UtcNow;
_maxMessagesPerSession = tasksConfig.MaxMessagesPerSession;
this.tasksConfig = tasksConfig;
_signalrServiceClient = new SignalrServiceClient();
_needSignal = false;
this.cancelToken = cancelToken;
if (tasksConfig.ShowActiveUpLogs)
{
Logger.Log4NetLogger = log4net.LogManager.GetLogger(string.Format("Task_{0}->ActiveUp", Task.CurrentId));
Logger.Disabled = false;
}
else
{
Logger.Log4NetLogger = null;
Logger.Disabled = true;
}
}
public QueueManager(MailBoxManager mailBoxManager, TasksConfig tasksConfig, ILogger log = null)
{
_manager = mailBoxManager;
_maxItemsLimit = tasksConfig.MaxTasksAtOnce;
_mailBoxQueue = new Queue<MailBox>();
_lockedMailBoxList = new List<MailBox>();
_tasksConfig = tasksConfig;
_log = log ?? new NullLogger();
_loadQueueTime = DateTime.UtcNow;
_tenantMemCache = new MemoryCache("QueueManagerTenantCache");
CancelHandler = new ManualResetEvent(false);
}
public AggregatorService(IEnumerable <string> workOnThisUsersOnly = null)
{
ServiceName = AscMailCollectionServiceName;
EventLog.Log = "Application";
// These Flags set whether or not to handle that specific
// type of event. Set to true if you need it, false otherwise.
CanHandlePowerEvent = false;
CanHandleSessionChangeEvent = false;
CanPauseAndContinue = false;
CanShutdown = true;
CanStop = true;
try
{
XmlConfigurator.Configure();
_log = LoggerFactory.GetLogger(LoggerFactory.LoggerType.Log4Net, "MainThread");
Environment.SetEnvironmentVariable("MONO_TLS_SESSION_CACHE_TIMEOUT", "0");
_queueSettings = MailQueueSettings.FromConfig;
if (workOnThisUsersOnly != null)
{
_queueSettings.WorkOnUsersOnly = workOnThisUsersOnly.ToList();
}
else
{
var userToWorkOn = ConfigurationManager.AppSettings["mail.OneUserMode"];
if (!string.IsNullOrEmpty(userToWorkOn))
{
_queueSettings.WorkOnUsersOnly.Add(userToWorkOn);
}
}
var authErrorWarningTimeout =
ConfigurationManager.AppSettings["mail.auth-error-warning-in-minutes"] != null
? TimeSpan.FromMinutes(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.auth-error-warning-in-minutes"]))
: TimeSpan.FromHours(1);
_log.Info("Auth login error warning timeout is {0}.", authErrorWarningTimeout.ToString());
var authErrorDisableMailboxTimeout =
ConfigurationManager.AppSettings["mail.auth-error-disable-mailbox-in-minutes"] != null
? TimeSpan.FromMinutes(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.auth-error-disable-mailbox-in-minutes"]))
: TimeSpan.FromDays(3);
_log.Info("Auth login error disable mailbox timeout is {0}.", authErrorDisableMailboxTimeout.ToString());
_log.Info("MailWorkerQueue: ConcurrentThreadCount = {0} and CheckInterval = {1} CheckPOP3_UIDL_Chunck = {2}",
_queueSettings.ConcurrentThreadCount, _queueSettings.CheckInterval, _queueSettings.CheckPop3UidlChunk);
var configBuilder = new TasksConfig.Builder();
if (_queueSettings.WorkOnUsersOnly != null && _queueSettings.WorkOnUsersOnly.Any())
{
var i = 0;
var users = string.Empty;
_queueSettings.WorkOnUsersOnly.ForEach(user => users += string.Format("\r\n\t\t\t\t{0}. \"{1}\"", ++i, user));
_log.Info("Aggregator will get tasks for this users only:" + users);
}
var queueLifetime = ConfigurationManager.AppSettings["mail.queue-lifetime-seconds"] != null
? TimeSpan.FromSeconds(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.queue-lifetime-seconds"]))
: TimeSpan.FromSeconds(30);
var showActiveUpLogs = ConfigurationManager.AppSettings["mail.show-activeup-logs"] != null &&
Convert.ToBoolean(
ConfigurationManager.AppSettings["mail.show-activeup-logs"]);
_tasksConfig = configBuilder.SetUsersToWorkOn(_queueSettings.WorkOnUsersOnly)
.SetOnlyTeamlabTasks(_queueSettings.OnlyTeamlabTasks)
.SetActiveInterval(_queueSettings.ActivityTimeout)
.SetChunkOfPop3CheckUidLinDb(_queueSettings.CheckPop3UidlChunk)
.SetEnableSignalr(_queueSettings.EnableSignalr)
.SetMaxMessagesPerSession(_queueSettings.MaxMessagesPerSession)
.SetMaxTasksAtOnce(_queueSettings.ConcurrentThreadCount)
.SetQueueLifetime(queueLifetime)
.SetTenantCachingPeriod(_queueSettings.TenantCachingPeriod)
.SetShowActiveUpLogs(showActiveUpLogs)
.SetInactiveMailboxesRatio(_queueSettings.InactiveMailboxesRatio)
.SetAuthErrorWarningTimeout(authErrorWarningTimeout)
.SetAuthErrorDisableMailboxTimeout(authErrorDisableMailboxTimeout)
.Build();
_tsInterval = _queueSettings.CheckInterval;
_manager = new MailBoxManager(_log)
{
AuthErrorWarningTimeout = _tasksConfig.AuthErrorWarningTimeout,
AuthErrorDisableTimeout = _tasksConfig.AuthErrorDisableMailboxTimeout
};
_queueManager = new QueueManager(_manager, _tasksConfig, _log);
_resetEvent = new ManualResetEvent(false);
_cancelTokenSource = new CancellationTokenSource();
_lcts = new LimitedConcurrencyLevelTaskScheduler(_tasksConfig.MaxTasksAtOnce);
_taskFactory = new TaskFactory(_lcts);
_tsTaskStateCheckInterval = ConfigurationManager.AppSettings["mail.task-check-state-seconds"] != null
? TimeSpan.FromSeconds(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.task-check-state-seconds"]))
: TimeSpan.FromSeconds(30);
if (ConfigurationManager.AppSettings["mail.default-api-scheme"] != null)
{
var defaultApiScheme = ConfigurationManager.AppSettings["mail.default-api-scheme"];
ApiHelper.SetupScheme(defaultApiScheme);
}
_log.Info("Service is ready.");
}
catch (Exception ex)
{
_log.Fatal("CollectorService error under construct: {0}", ex.ToString());
}
}
private void AggregatePop3(TasksConfig tasksConfig, int limitMessages = -1)
{
if (!Pop.IsAuthenticated)
{
LoginPop3();
}
var mailFolder = new MailFolder(MailFolder.Ids.inbox, "INBOX");
var loaded = 0;
var newMessages = GetPop3NewMessagesIDs(tasksConfig);
if (newMessages.Count == 0)
{
Log.Debug("New messages not found.\r\n");
return;
}
Log.Debug("Found {0} new messages.\r\n", newMessages.Count);
try
{
var fstMailHeaders = Pop.GetMessageHeaders(newMessages.First().Key, CancelToken).ToList();
var lstMailHeaders = Pop.GetMessageHeaders(newMessages.Last().Key, CancelToken).ToList();
var fstDateHeader =
fstMailHeaders.FirstOrDefault(
h => h.Field.Equals("Date", StringComparison.InvariantCultureIgnoreCase));
var lstDateHeader =
lstMailHeaders.FirstOrDefault(
h => h.Field.Equals("Date", StringComparison.InvariantCultureIgnoreCase));
var found = false;
if (fstDateHeader != null && lstDateHeader != null)
{
DateTime fstDate;
DateTime lstDate;
if (DateTime.TryParse(fstDateHeader.Value, out fstDate) &&
DateTime.TryParse(lstDateHeader.Value, out lstDate))
{
found = true;
if (fstDate < lstDate)
{
Log.Debug("Account '{0}' order is DESC", Account.EMail.Address);
newMessages = newMessages
.OrderByDescending(item => item.Key)
// This is to ensure that the newest message would be handled primarily.
.ToDictionary(id => id.Key, id => id.Value);
}
}
}
if (!found)
{
Log.Debug("Account '{0}' order is ASC", Account.EMail.Address);
}
}
catch (Exception)
{
Log.Warn("Calculating order skipped! Account '{0}' order is ASC", Account.EMail.Address);
}
var skipOnDate = Account.BeginDate != MailBoxManager.MinBeginDate;
var skipBreakOnDate = tasksConfig.PopUnorderedDomains.Contains(Account.Server.ToLowerInvariant());
foreach (var newMessage in newMessages)
{
if (!Pop.IsConnected || CancelToken.IsCancellationRequested)
{
break;
}
Log.Debug("Processing new message\tUID: {0}\tUIDL: {1}\t",
newMessage.Key,
newMessage.Value);
try
{
var message = Pop.GetMessage(newMessage.Key, CancelToken);
message.FixDateIssues(logger: Log);
if (message.Date < Account.BeginDate && skipOnDate)
{
if (!skipBreakOnDate)
{
Log.Info("Skip other messages older then {0}.", Account.BeginDate);
break;
}
Log.Debug("Skip message (Date = {0}) on BeginDate = {1}", message.Date,
Account.BeginDate);
continue;
}
message.FixEncodingIssues();
OnGetMessage(message, newMessage.Value, true, mailFolder);
loaded++;
if (limitMessages <= 0 || loaded < limitMessages)
{
continue;
}
Log.Debug("Limit of max messages per session is exceeded!");
break;
}
catch (OperationCanceledException)
{
break;
}
catch (Exception e)
{
Log.Error(
"ProcessMessages() Tenant={0} User='******' Account='{2}', MailboxId={3}, MessageIndex={4}, UIDL='{5}' Exception:\r\n{6}\r\n",
Account.TenantId, Account.UserId, Account.EMail.Address, Account.MailBoxId,
newMessage.Key, newMessage.Value, e);
if (e is IOException)
{
break; // stop checking other mailboxes
}
}
}
}
private MailFolder DetectFolder(TasksConfig tasksConfig, IMailFolder folder)
{
var folderName = folder.Name.ToLowerInvariant();
if (tasksConfig.SkipImapFlags != null &&
tasksConfig.SkipImapFlags.Any() &&
tasksConfig.SkipImapFlags.Contains(folderName))
{
return(null);
}
int folderId;
if ((folder.Attributes & FolderAttributes.Inbox) != 0)
{
return(new MailFolder(MailFolder.Ids.inbox, folder.Name));
}
if ((folder.Attributes & FolderAttributes.Sent) != 0)
{
return(new MailFolder(MailFolder.Ids.sent, folder.Name));
}
if ((folder.Attributes & FolderAttributes.Junk) != 0)
{
return(new MailFolder(MailFolder.Ids.spam, folder.Name));
}
if ((folder.Attributes &
(FolderAttributes.All |
FolderAttributes.NoSelect |
FolderAttributes.NoInferiors |
FolderAttributes.NonExistent |
FolderAttributes.Unmarked |
FolderAttributes.Trash |
FolderAttributes.Archive |
FolderAttributes.Drafts |
FolderAttributes.Flagged |
FolderAttributes.Marked)) != 0)
{
return(null); // Skip folders
}
if (tasksConfig.ImapFlags != null &&
tasksConfig.ImapFlags.Any() &&
tasksConfig.ImapFlags.ContainsKey(folderName))
{
folderId = tasksConfig.ImapFlags[folderName];
return(new MailFolder(folderId, folder.Name));
}
if (tasksConfig.SpecialDomainFolders.Any() &&
tasksConfig.SpecialDomainFolders.ContainsKey(Account.Server))
{
var domainSpecialFolders = tasksConfig.SpecialDomainFolders[Account.Server];
if (domainSpecialFolders.Any() &&
domainSpecialFolders.ContainsKey(folderName))
{
var info = domainSpecialFolders[folderName];
return(info.skip ? null : new MailFolder(info.folder_id, folder.Name));
}
}
if (tasksConfig.DefaultFolders == null || !tasksConfig.DefaultFolders.ContainsKey(folderName))
{
return(new MailFolder(MailFolder.Ids.inbox, folder.Name, new[] { folder.FullName }));
}
folderId = tasksConfig.DefaultFolders[folderName];
return(new MailFolder(folderId, folder.Name));
}
private Dictionary <int, string> GetPop3NewMessagesIDs(TasksConfig tasksConfig)
{
var newMessages = new Dictionary <int, string>();
var j = 0;
var uidls =
Pop.GetMessageUids(CancelToken)
.Select(uidl => new KeyValuePair <int, string>(j++, uidl))
.ToDictionary(t => t.Key, t => t.Value);
if (!uidls.Any() || uidls.Count == Account.MessagesCount)
{
return(newMessages);
}
var i = 0;
var chunk = tasksConfig.ChunkOfPop3Uidl;
var chunkUidls = uidls.Skip(i).Take(chunk).ToList();
var manager = new MailBoxManager();
do
{
var checkList = chunkUidls.Select(u => u.Value).Distinct().ToList();
var existingUidls = manager.CheckUidlExistance(Account.MailBoxId, checkList);
if (!existingUidls.Any())
{
var messages = newMessages;
foreach (var item in
chunkUidls.Select(uidl => new KeyValuePair <int, string>(uidl.Key, uidl.Value))
.Where(item => !messages.Contains(item)))
{
newMessages.Add(item.Key, item.Value);
}
}
else if (existingUidls.Count != chunkUidls.Count)
{
var messages = newMessages;
foreach (var item in (from uidl in chunkUidls
where !existingUidls.Contains(uidl.Value)
select new KeyValuePair <int, string>(uidl.Key, uidl.Value)).Where(
item => !messages.Contains(item)))
{
newMessages.Add(item.Key, item.Value);
}
}
i += chunk;
chunkUidls = uidls.Skip(i).Take(chunk).ToList();
} while (chunkUidls.Any());
newMessages = newMessages
.OrderByDescending(item => item.Key)
// This is to ensure that the newest message would be handled primarily.
.ToDictionary(id => id.Key, id => id.Value);
return(newMessages);
}
public AggregatorService(Options options)
{
ServiceName = ASC_MAIL_COLLECTION_SERVICE_NAME;
EventLog.Log = "Application";
// These Flags set whether or not to handle that specific
// type of event. Set to true if you need it, false otherwise.
CanHandlePowerEvent = false;
CanHandleSessionChangeEvent = false;
CanPauseAndContinue = false;
CanShutdown = true;
CanStop = true;
try
{
_log = LoggerFactory.GetLogger(LoggerFactory.LoggerType.Log4Net, "MainThread");
_logStat = LogManager.GetLogger("ASC.MAIL.STAT");
_tasksConfig = TasksConfig.FromConfig;
_tasksConfig.DefaultFolders = MailQueueItemSettings.DefaultFolders;
_tasksConfig.ImapFlags = MailQueueItemSettings.ImapFlags;
_tasksConfig.PopUnorderedDomains = MailQueueItemSettings.PopUnorderedDomains;
_tasksConfig.SkipImapFlags = MailQueueItemSettings.SkipImapFlags;
_tasksConfig.SpecialDomainFolders = MailQueueItemSettings.SpecialDomainFolders;
if (options.OnlyUsers != null)
{
_tasksConfig.WorkOnUsersOnly.AddRange(options.OnlyUsers.ToList());
}
if (options.NoMessagesLimit)
{
_tasksConfig.MaxMessagesPerSession = -1;
}
_taskSecondsLifetime =
TimeSpan.FromSeconds(ConfigurationManager.AppSettings["mail.task-process-lifetime-seconds"] != null
? Convert.ToInt32(ConfigurationManager.AppSettings["mail.task-process-lifetime-seconds"])
: 300);
_queueManager = new QueueManager(_tasksConfig, _log);
_resetEvent = new ManualResetEvent(false);
_cancelTokenSource = new CancellationTokenSource();
_taskFactory = new TaskFactory();
_tsTaskStateCheckInterval = ConfigurationManager.AppSettings["mail.task-check-state-seconds"] != null
? TimeSpan.FromSeconds(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.task-check-state-seconds"]))
: TimeSpan.FromSeconds(30);
if (_tasksConfig.EnableSignalr)
{
_signalrWorker = new SignalrWorker();
}
_workTimer = new Timer(workTimer_Elapsed, _cancelTokenSource.Token, Timeout.Infinite, Timeout.Infinite);
_log.Info("Service is ready.");
}
catch (Exception ex)
{
_log.Fatal("CollectorService error under construct: {0}", ex.ToString());
}
}
public AggregatorService(Options options)
{
ServiceName = AscMailCollectionServiceName;
EventLog.Log = "Application";
// These Flags set whether or not to handle that specific
// type of event. Set to true if you need it, false otherwise.
CanHandlePowerEvent = false;
CanHandleSessionChangeEvent = false;
CanPauseAndContinue = false;
CanShutdown = true;
CanStop = true;
try
{
_log = LoggerFactory.GetLogger(LoggerFactory.LoggerType.Log4Net, "MainThread");
Environment.SetEnvironmentVariable("MONO_TLS_SESSION_CACHE_TIMEOUT", "0");
_tasksConfig = TasksConfig.FromConfig;
if (options.OnlyUsers != null)
{
_tasksConfig.WorkOnUsersOnly.AddRange(options.OnlyUsers.ToList());
}
if (options.NoMessagesLimit)
{
_tasksConfig.MaxMessagesPerSession = -1;
}
_taskSecondsLifetime =
TimeSpan.FromSeconds(ConfigurationManager.AppSettings["mail.task-process-lifetime-seconds"] != null
? Convert.ToInt32(ConfigurationManager.AppSettings["mail.task-process-lifetime-seconds"])
: 300);
_queueManager = new QueueManager(_tasksConfig, _log);
_resetEvent = new ManualResetEvent(false);
_cancelTokenSource = new CancellationTokenSource();
_taskFactory = new TaskFactory();
_tsTaskStateCheckInterval = ConfigurationManager.AppSettings["mail.task-check-state-seconds"] != null
? TimeSpan.FromSeconds(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.task-check-state-seconds"]))
: TimeSpan.FromSeconds(30);
if (_tasksConfig.EnableSignalr)
{
_signalrServiceClient = new SignalrServiceClient();
}
_workTimer = new Timer(workTimer_Elapsed, _cancelTokenSource.Token, Timeout.Infinite, Timeout.Infinite);
_gcCleanerTimer = new Timer(gcCleanerTimer_Elapsed, _cancelTokenSource.Token, _gcCleanTimerInterval, _gcCleanTimerInterval);
_log.Info("Service is ready.");
}
catch (Exception ex)
{
_log.Fatal("CollectorService error under construct: {0}", ex.ToString());
}
}
private void AggregateImap(TasksConfig tasksConfig, int limitMessages = -1)
{
if (!Imap.IsAuthenticated)
{
LoginImap();
}
try
{
var loaded = 0;
var folders = GetImapFolders();
foreach (var folder in folders)
{
if (!Imap.IsConnected || CancelToken.IsCancellationRequested)
{
return;
}
var mailFolder = DetectFolder(tasksConfig, folder);
if (mailFolder == null)
{
Log.Info("[folder] x '{0}' (skipped)", folder.Name);
continue;
}
Log.Info("[folder] >> '{0}' (fId={1}) {2}", folder.Name, mailFolder.FolderId,
mailFolder.Tags.Any() ? string.Format("tag='{0}'", mailFolder.Tags.FirstOrDefault()) : "");
try
{
folder.Open(FolderAccess.ReadOnly, CancelToken);
}
catch (OperationCanceledException)
{
// Skip log error
continue;
}
catch (Exception e)
{
Log.Error("Open faild: {0} Exception: {1}", folder.Name, e.Message);
continue;
}
loaded += LoadFolderMessages(folder, mailFolder, limitMessages);
if (limitMessages <= 0 || loaded < limitMessages)
{
continue;
}
Log.Debug("Limit of maximum number messages per session is exceeded!");
break;
}
}
catch (AggregateException aggEx)
{
if (aggEx.InnerException != null)
{
throw aggEx.InnerException;
}
throw new Exception("AggregateImap failed", aggEx);
}
}
public Pop3Worker(MailBoxManager mailBoxManager, MailBox mailBox, TasksConfig tasksConfig, CancellationToken cancelToken, ILogger log = null)
: base(mailBoxManager, mailBox, tasksConfig, cancelToken, log)
{
}
private void AggregatePop3(TasksConfig tasksConfig, int limitMessages = -1)
{
if (!Pop.IsAuthenticated)
{
LoginPop3();
}
var mailFolder = new MailFolder(MailFolder.Ids.inbox, "INBOX");
try
{
var loaded = 0;
var newMessages = GetPop3NewMessagesIDs(tasksConfig);
if (newMessages.Count == 0)
{
Log.Debug("New messages not found.\r\n");
return;
}
Log.Debug("Found {0} new messages.\r\n", newMessages.Count);
newMessages = FixPop3UidsOrder(newMessages);
var skipOnDate = Account.BeginDate != MailBoxManager.MinBeginDate;
var skipBreakOnDate = tasksConfig.PopUnorderedDomains.Contains(Account.Server.ToLowerInvariant());
foreach (var newMessage in newMessages)
{
if (!Pop.IsConnected || CancelToken.IsCancellationRequested)
{
break;
}
Log.Debug("Processing new message\tUID: {0}\tUIDL: {1}\t",
newMessage.Key,
newMessage.Value);
try
{
var message = Pop.GetMessage(newMessage.Key, CancelToken);
message.FixDateIssues(logger: Log);
if (message.Date < Account.BeginDate && skipOnDate)
{
if (!skipBreakOnDate)
{
Log.Info("Skip other messages older then {0}.", Account.BeginDate);
break;
}
Log.Debug("Skip message (Date = {0}) on BeginDate = {1}", message.Date,
Account.BeginDate);
continue;
}
message.FixEncodingIssues();
OnGetMessage(message, newMessage.Value, true, mailFolder);
loaded++;
if (limitMessages <= 0 || loaded < limitMessages)
{
continue;
}
Log.Debug("Limit of max messages per session is exceeded!");
break;
}
catch (OperationCanceledException)
{
// Skip log error
}
catch (Exception e)
{
Log.Error(
"ProcessMessages() Tenant={0} User='******' Account='{2}', MailboxId={3}, MessageIndex={4}, UIDL='{5}' Exception:\r\n{6}\r\n",
Account.TenantId, Account.UserId, Account.EMail.Address, Account.MailBoxId,
newMessage.Key, newMessage.Value, e);
if (e is IOException)
{
break; // stop checking other mailboxes
}
}
}
}
catch (AggregateException aggEx)
{
if (aggEx.InnerException != null)
{
throw aggEx.InnerException;
}
throw new Exception("AggregatePop3 failed", aggEx);
}
}
public AggregatorService(IEnumerable<string> workOnThisUsersOnly = null)
{
ServiceName = AscMailCollectionServiceName;
EventLog.Log = "Application";
// These Flags set whether or not to handle that specific
// type of event. Set to true if you need it, false otherwise.
CanHandlePowerEvent = false;
CanHandleSessionChangeEvent = false;
CanPauseAndContinue = false;
CanShutdown = true;
CanStop = true;
try
{
XmlConfigurator.Configure();
_log = LoggerFactory.GetLogger(LoggerFactory.LoggerType.Log4Net, "MainThread");
Environment.SetEnvironmentVariable("MONO_TLS_SESSION_CACHE_TIMEOUT", "0");
_queueSettings = MailQueueSettings.FromConfig;
if (workOnThisUsersOnly != null)
_queueSettings.WorkOnUsersOnly = workOnThisUsersOnly.ToList();
else
{
var userToWorkOn = ConfigurationManager.AppSettings["mail.OneUserMode"];
if (!string.IsNullOrEmpty(userToWorkOn))
_queueSettings.WorkOnUsersOnly.Add(userToWorkOn);
}
var authErrorWarningTimeout =
ConfigurationManager.AppSettings["mail.auth-error-warning-in-minutes"] != null
? TimeSpan.FromMinutes(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.auth-error-warning-in-minutes"]))
: TimeSpan.FromHours(1);
_log.Info("Auth login error warning timeout is {0}.", authErrorWarningTimeout.ToString());
var authErrorDisableMailboxTimeout =
ConfigurationManager.AppSettings["mail.auth-error-disable-mailbox-in-minutes"] != null
? TimeSpan.FromMinutes(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.auth-error-disable-mailbox-in-minutes"]))
: TimeSpan.FromDays(3);
_log.Info("Auth login error disable mailbox timeout is {0}.", authErrorDisableMailboxTimeout.ToString());
_log.Info("MailWorkerQueue: ConcurrentThreadCount = {0} and CheckInterval = {1} CheckPOP3_UIDL_Chunck = {2}",
_queueSettings.ConcurrentThreadCount, _queueSettings.CheckInterval, _queueSettings.CheckPop3UidlChunk);
var configBuilder = new TasksConfig.Builder();
if (_queueSettings.WorkOnUsersOnly != null && _queueSettings.WorkOnUsersOnly.Any())
{
var i = 0;
var users = string.Empty;
_queueSettings.WorkOnUsersOnly.ForEach(user => users += string.Format("\r\n\t\t\t\t{0}. \"{1}\"", ++i, user));
_log.Info("Aggregator will get tasks for this users only:" + users);
}
var queueLifetime = ConfigurationManager.AppSettings["mail.queue-lifetime-seconds"] != null
? TimeSpan.FromSeconds(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.queue-lifetime-seconds"]))
: TimeSpan.FromSeconds(30);
var showActiveUpLogs = ConfigurationManager.AppSettings["mail.show-activeup-logs"] != null &&
Convert.ToBoolean(
ConfigurationManager.AppSettings["mail.show-activeup-logs"]);
_tasksConfig = configBuilder.SetUsersToWorkOn(_queueSettings.WorkOnUsersOnly)
.SetOnlyTeamlabTasks(_queueSettings.OnlyTeamlabTasks)
.SetActiveInterval(_queueSettings.ActivityTimeout)
.SetChunkOfPop3CheckUidLinDb(_queueSettings.CheckPop3UidlChunk)
.SetEnableSignalr(_queueSettings.EnableSignalr)
.SetMaxMessagesPerSession(_queueSettings.MaxMessagesPerSession)
.SetMaxTasksAtOnce(_queueSettings.ConcurrentThreadCount)
.SetQueueLifetime(queueLifetime)
.SetTenantCachingPeriod(_queueSettings.TenantCachingPeriod)
.SetShowActiveUpLogs(showActiveUpLogs)
.SetInactiveMailboxesRatio(_queueSettings.InactiveMailboxesRatio)
.SetAuthErrorWarningTimeout(authErrorWarningTimeout)
.SetAuthErrorDisableMailboxTimeout(authErrorDisableMailboxTimeout)
.Build();
_tsInterval = _queueSettings.CheckInterval;
_manager = new MailBoxManager(_log)
{
AuthErrorWarningTimeout = _tasksConfig.AuthErrorWarningTimeout,
AuthErrorDisableTimeout = _tasksConfig.AuthErrorDisableMailboxTimeout
};
_queueManager = new QueueManager(_manager, _tasksConfig, _log);
_resetEvent = new ManualResetEvent(false);
_cancelTokenSource = new CancellationTokenSource();
_lcts = new LimitedConcurrencyLevelTaskScheduler(_tasksConfig.MaxTasksAtOnce);
_taskFactory = new TaskFactory(_lcts);
_tsTaskStateCheckInterval = ConfigurationManager.AppSettings["mail.task-check-state-seconds"] != null
? TimeSpan.FromSeconds(
Convert.ToInt32(ConfigurationManager.AppSettings["mail.task-check-state-seconds"]))
: TimeSpan.FromSeconds(30);
if (ConfigurationManager.AppSettings["mail.default-api-scheme"] != null)
{
var defaultApiScheme = ConfigurationManager.AppSettings["mail.default-api-scheme"];
ApiHelper.SetupScheme(defaultApiScheme);
}
_log.Info("Service is ready.");
}
catch (Exception ex)
{
_log.Fatal("CollectorService error under construct: {0}", ex.ToString());
}
}
private MailBox ProcessMailbox(MailBox mailbox, TasksConfig tasksConfig, CancellationToken cancelToken)
{
var taskLogger = LoggerFactory.GetLogger(LoggerFactory.LoggerType.Log4Net, "Task_" + Task.CurrentId);
taskLogger.Info(
"ProcessMailbox(Tenant = {0}, MailboxId = {1} Address = '{2}') Is {3}",
mailbox.TenantId, mailbox.MailBoxId,
mailbox.EMail, mailbox.Active ? "Active" : "Inactive");
try
{
var manager = new MailBoxManager(taskLogger)
{
AuthErrorWarningTimeout = tasksConfig.AuthErrorWarningTimeout,
AuthErrorDisableTimeout = tasksConfig.AuthErrorDisableMailboxTimeout
};
if (mailbox.Imap)
{
using (var worker = new Imap4Worker(manager, mailbox, tasksConfig, cancelToken, taskLogger))
{
worker.Aggregate();
}
}
else
{
using (var worker = new Pop3Worker(manager, mailbox, tasksConfig, cancelToken, taskLogger))
{
worker.Aggregate();
}
}
taskLogger.Info("Mailbox '{0}' has been processed.", mailbox.EMail);
}
catch (OperationCanceledException)
{
taskLogger.Info("Task canceled.");
throw;
}
catch (Exception ex)
{
taskLogger.Error(
"ProcessMailbox(Tenant = {0}, MailboxId = {1}, Address = '{2}')\r\nException: {3}\r\n",
mailbox.TenantId, mailbox.MailBoxId, mailbox.EMail, ex.ToString());
}
return mailbox;
}
public Imap4Worker(MailBoxManager mailBoxManager, MailBox mailBox, TasksConfig tasksConfig, CancellationToken cancelToken, ILogger log = null)
: base(mailBoxManager, mailBox, tasksConfig, cancelToken, log)
{
}
public Pop3Worker(MailBoxManager mailBoxManager, MailBox mailBox, TasksConfig tasksConfig, CancellationToken cancelToken, bool loadOriginalEml, ILogger log = null)
: base(mailBoxManager, mailBox, tasksConfig, cancelToken, log)
{
LoadOriginalEmlData = loadOriginalEml;
}
private void ProcessMailbox(MailClient client, TasksConfig tasksConfig)
{
var mailbox = client.Account;
Stopwatch watch = null;
if (_tasksConfig.CollectStatistics)
{
watch = new Stopwatch();
watch.Start();
}
var failed = false;
var taskLogger = LoggerFactory.GetLogger(LoggerFactory.LoggerType.Log4Net,
string.Format("Mbox_{0} Task_{1}", mailbox.MailBoxId, Task.CurrentId));
taskLogger.Info(
"ProcessMailbox(Tenant = {0}, MailboxId = {1} Address = '{2}') Is {3}",
mailbox.TenantId, mailbox.MailBoxId,
mailbox.EMail, mailbox.Active ? "Active" : "Inactive");
try
{
client.Log = taskLogger;
client.GetMessage += ClientOnGetMessage;
client.Aggregate(tasksConfig, tasksConfig.MaxMessagesPerSession);
}
catch (OperationCanceledException)
{
taskLogger.Info(
"[CANCEL] ProcessMailbox(Tenant = {0}, MailboxId = {1}, Address = '{2}')",
mailbox.TenantId, mailbox.MailBoxId, mailbox.EMail);
NotifySignalrIfNeed(mailbox, taskLogger);
}
catch (Exception ex)
{
taskLogger.Error(
"ProcessMailbox(Tenant = {0}, MailboxId = {1}, Address = '{2}')\r\nException: {3}\r\n",
mailbox.TenantId, mailbox.MailBoxId, mailbox.EMail,
ex is ImapProtocolException || ex is Pop3ProtocolException ? ex.Message : ex.ToString());
failed = true;
}
finally
{
CloseMailClient(client, mailbox, _log);
if (_tasksConfig.CollectStatistics && watch != null)
{
watch.Stop();
LogStat(PROCESS_MAILBOX, mailbox, watch.Elapsed, failed);
}
}
var manager = new MailBoxManager(taskLogger);
var state = GetMailboxState(manager, mailbox, taskLogger);
switch (state)
{
case MailboxState.NoChanges:
taskLogger.Info("MailBox with id={0} not changed.", mailbox.MailBoxId);
break;
case MailboxState.Disabled:
taskLogger.Info("MailBox with id={0} is deactivated.", mailbox.MailBoxId);
break;
case MailboxState.Deleted:
taskLogger.Info("MailBox with id={0} is removed.", mailbox.MailBoxId);
try
{
taskLogger.Info("RemoveMailBox(id={0}) >> Try clear new data from removed mailbox", mailbox.MailBoxId);
manager.RemoveMailBox(mailbox);
}
catch (Exception exRem)
{
taskLogger.Info(
"[REMOVE] ProcessMailbox->RemoveMailBox(Tenant = {0}, MailboxId = {1}, Address = '{2}') Exception: {3}",
mailbox.TenantId, mailbox.MailBoxId, mailbox.EMail, exRem.Message);
}
break;
case MailboxState.DateChanged:
taskLogger.Info("MailBox with id={0}: beginDate was changed.", mailbox.MailBoxId);
break;
default:
throw new ArgumentOutOfRangeException();
}
taskLogger.Info("Mailbox '{0}' has been processed.", mailbox.EMail);
}
public bool ReleaseMaibox(MailBoxData account, TasksConfig tasksConfig)
{
var disableMailbox = false;
var engineFactory = new EngineFactory(account.TenantId, account.UserId);
if (account.AuthErrorDate.HasValue)
{
var difference = DateTime.UtcNow - account.AuthErrorDate.Value;
if (difference > tasksConfig.AuthErrorDisableMailboxTimeout)
{
disableMailbox = true;
engineFactory.AlertEngine.CreateAuthErrorDisableAlert(account.TenantId, account.UserId,
account.MailBoxId);
}
else if (difference > tasksConfig.AuthErrorWarningTimeout)
{
engineFactory.AlertEngine.CreateAuthErrorWarningAlert(account.TenantId, account.UserId,
account.MailBoxId);
}
}
if (account.QuotaErrorChanged)
{
if (account.QuotaError)
{
engineFactory.AlertEngine.CreateQuotaErrorWarningAlert(account.TenantId, account.UserId);
}
else
{
engineFactory.AlertEngine.DeleteAlert(MailAlertTypes.QuotaError);
}
}
using (var daoFactory = new DaoFactory())
{
var daoMailbox = daoFactory.CreateMailboxDao();
var mailbox =
daoMailbox.GetMailBox(new СoncreteUserMailboxExp(account.MailBoxId, account.TenantId,
account.UserId));
if (mailbox == null) // Mailbox has been removed
{
return(true);
}
bool? enabled = null;
int? messageCount = null;
long? size = null;
bool? quotaError = null;
string oAuthToken = null;
string imapIntervalsJson = null;
bool? resetImapIntervals = null;
if (account.AuthErrorDate.HasValue)
{
if (disableMailbox)
{
enabled = false;
}
}
if (mailbox.MsgCountLast != account.MessagesCount)
{
messageCount = account.MessagesCount;
}
if (mailbox.SizeLast != account.Size)
{
size = account.Size;
}
if (account.QuotaErrorChanged)
{
quotaError = account.QuotaError;
}
if (account.AccessTokenRefreshed)
{
oAuthToken = account.OAuthToken;
}
if (account.Imap && account.ImapFolderChanged)
{
if (account.BeginDateChanged)
{
resetImapIntervals = true;
}
else
{
imapIntervalsJson = account.ImapIntervalsJson;
}
}
return(daoMailbox.SetMailboxProcessed(mailbox, account.ServerLoginDelay, enabled, messageCount, size,
quotaError, oAuthToken, imapIntervalsJson, resetImapIntervals));
}
}
// GET: api/tasks/2
public ArrayList Get(int id)
{
TasksConfig tasks = new TasksConfig();
return(tasks.getTasks(id));
}
