// Returns: maxMessagesPerSession minus count of downloaded messages or zero if limit excided private int ProcessMessages( Mailbox mb, int folder_id, int last_uid, int max_messages_per_session, WaitHandle stop_event, string[] tags_names) { UpdateTimeCheckedIfNeeded(); int[] uids_collection; try { uids_collection = mb.UidSearch("UID " + (0 != last_uid ? last_uid : 1) + ":*") .Where(uid => uid != last_uid) .ToArray(); if (!uids_collection.Any()) throw new Exception("Empty folder"); } catch (Exception) { _log.Info("New messages not found."); return max_messages_per_session; } var stored_uid_list = new Dictionary<int, string>(); var stored_md5_list = new Dictionary<int, string>(); InvokeGetStoredMessagesUIDL_MD5(stored_uid_list, stored_md5_list); var tags_hash = tags_names.Aggregate(string.Empty, (current, name) => current + name).GetMD5Hash(); var new_messages = uids_collection .Select( item_uid => new { uid = item_uid, uidl = tags_names.Any() ? string.Format("{0}-{1}-{2}-{3}", item_uid, folder_id, mb.UidValidity, tags_hash) : string.Format("{0}-{1}-{2}", item_uid, folder_id, mb.UidValidity) }) .Where(msg => !stored_uid_list.Values.Contains(msg.uidl)) .OrderByDescending(msg => msg.uid) .ToDictionary(msg => msg.uid, id => id.uidl); var quota_error_flag = false; var update_folder_uid_flag = new_messages.Any(); int[] tags_ids = null; var message_ids_for_tag_update = new List<int>(); var tags_retrieved = false; foreach (var new_message in new_messages) { int last_founded_message_id = -1; try { if (stop_event.WaitOne(0)) { _log.Debug("Stop event occure."); break; } if (max_messages_per_session == 0) { _log.Debug("Limit of max messages per session is exceeded!"); update_folder_uid_flag = false; break; } // flags should be retrieved before message fetch - because mail server // could add seen flag right after message was retrieved by us var flags = mb.Fetch.UidFlags(new_message.Key); //Peek method didn't set \Seen flag on mail var message = mb.Fetch.UidMessageObjectPeek(new_message.Key); UpdateTimeCheckedIfNeeded(); if (message.Date < Account.BeginDate) { _log.Debug("Skip message (Date = {0}) on BeginDate = {1}", message.Date, Account.BeginDate); break; } var unique_identifier = string.Format("{0}|{1}|{2}|{3}", message.From.Email, message.Subject, message.DateString, message.MessageId); var header_md5 = unique_identifier.GetMD5(); //Get tags ids for folder before message proccessing only once if (!tags_retrieved) { tags_ids = tags_names.Any() ? InvokeOnGetOrCreateTags(tags_names) : null; tags_retrieved = true; } if (folder_id == MailFolder.Ids.inbox || !message.To.Exists(email => email.Email.ToLowerInvariant() .Equals(message.From.Email.ToLowerInvariant()) ) ) { var found_message_id = stored_md5_list .Where(el => el.Value == header_md5) .Select(el => el.Key) .FirstOrDefault(); if (found_message_id > 0) { InvokeOnUpdateUidl(found_message_id, new_message.Value); last_founded_message_id = found_message_id; message_ids_for_tag_update.Add(found_message_id); //Todo: Remove id if exception happened continue; // Skip saving founded message } } var unread = null == flags["seen"]; InvokeOnRetrieve(message, folder_id, new_message.Value, header_md5, unread, tags_ids); } catch (IOException io_ex) { if (io_ex.Message.StartsWith("Unable to write data to the transport connection") || io_ex.Message.StartsWith("Unable to read data from the transport connection")) { _log.Error("ProcessMessages() Account='{0}': {1}", Account.EMail.Address, io_ex.ToString()); message_ids_for_tag_update.Remove(last_founded_message_id); update_folder_uid_flag = false; max_messages_per_session = 0; // stop checking other mailboxes break; } } catch (MailBoxOutException ex) { _log.Info("ProcessMessages() Account='{0}': {1}", Account.EMail.Address, ex.Message); message_ids_for_tag_update.Remove(last_founded_message_id); update_folder_uid_flag = false; max_messages_per_session = 0; // stop checking other mailboxes break; } catch (TenantQuotaException qex) { _log.Info("Tenant {0} quota exception: {1}", Account.TenantId, qex.Message); message_ids_for_tag_update.Remove(last_founded_message_id); quota_error_flag = true; update_folder_uid_flag = false; } catch (Exception e) { _log.Error("ProcessMessages() Account='{0}', MessageId={1} Exception:\r\n{2}\r\n", Account.EMail.Address, new_message, e.ToString()); update_folder_uid_flag = false; message_ids_for_tag_update.Remove(last_founded_message_id); } UpdateTimeCheckedIfNeeded(); max_messages_per_session--; } if (tags_ids != null && tags_ids.Length > 0 && message_ids_for_tag_update.Count > 0) { InvokeOnUpdateMessagesTags(Account.TenantId, Account.UserId, tags_ids, message_ids_for_tag_update.ToArray()); } if (update_folder_uid_flag) { var max_uid = new_messages.Keys.Max(); if(Account.ImapFolders.Keys.Contains(mb.Name)) Account.ImapFolders[mb.Name] = max_uid; else Account.ImapFolders.Add(mb.Name, max_uid); Account.ImapFolderChanged = true; } InvokeOnDone(quota_error_flag); return max_messages_per_session; }
private int LoadMailboxMessage(Mailbox mb, int folderId, int maxMessagesPerSession, string[] tagsNames, out bool quotaErrorFlag) { UpdateTimeCheckedIfNeeded(); ImapFolderUids folderUids; if (!Account.ImapIntervals.TryGetValue(mb.Name, out folderUids)) folderUids = new ImapFolderUids(); // by default - mailbox never was processed before var imapIntervals = new ImapIntervals(folderUids.UnhandledUidIntervals); var beginDateUid = folderUids.BeginDateUid; quotaErrorFlag = false; int[] tagsIds = null; var tagsRetrieved = false; var allUids = mb.UidSearch("1:*").OrderBy(i => i).SkipWhile(i => i < beginDateUid).ToList(); foreach (var uidsInterval in imapIntervals.GetUnhandledIntervalsCopy()) { cancelToken.ThrowIfCancellationRequested(); if (!mb.SourceClient.IsConnected) break; if (maxMessagesPerSession == 0) break; var interval = uidsInterval; var uidsCollection = allUids.Select(u => u) .Where(u => u <= interval.To && u >= interval.From) .OrderByDescending(x => x) .ToList(); if (!uidsCollection.Any()) { if (!uidsInterval.IsToUidMax()) imapIntervals.AddHandledInterval(uidsInterval); continue; } var toUid = uidsInterval.IsToUidMax() ? uidsCollection.First() : Math.Max(uidsInterval.To, uidsCollection.First()); #region Loading messages foreach (var uid in uidsCollection) { var hasParseError = false; cancelToken.ThrowIfCancellationRequested(); try { if (!mb.SourceClient.IsConnected) { log.Warn("Imap4-server is disconnected. Skip another messages."); break; } log.Debug("Processing new message\tUID: {0}", uid); // flags should be retrieved before message fetch - because mail server // could add seen flag right after message was retrieved by us var flags = mb.Fetch.UidFlags(uid); //Peek method didn't set \Seen flag on mail var message = mb.Fetch.UidMessageObjectPeek(uid); if (message.HasParseError) { log.Error("ActiveUp: message parsed with some errors. MailboxId = {0} Message UID = {1}", Account.MailBoxId, uid); hasParseError = true; } UpdateTimeCheckedIfNeeded(); if (message.Date < Account.BeginDate) { log.Debug("Skip message (Date = {0}) on BeginDate = {1}", message.Date, Account.BeginDate); imapIntervals.SetBeginIndex(toUid); beginDateUid = toUid; break; } var uniqueIdentifier = string.Format("{0}|{1}|{2}|{3}", message.From.Email, message.Subject, message.DateString, message.MessageId); var headerMd5 = uniqueIdentifier.GetMd5(); //Get tags ids for folder before message proccessing only once if (!tagsRetrieved) { tagsIds = tagsNames.Any() ? InvokeOnGetOrCreateTags(tagsNames) : null; tagsRetrieved = true; } var unread = null == flags["seen"]; var uidl = string.Format("{0}-{1}", uid, folderId); RetrieveMessage(message, folderId, uidl, headerMd5, hasParseError, unread, tagsIds); } catch (Exception e) { log.Error( "ProcessMessages() Tenant={0} User='******' Account='{2}', MailboxId={3}, UID={4} Exception:\r\n{5}\r\n", Account.TenantId, Account.UserId, Account.EMail.Address, Account.MailBoxId, uid, e); if (e is IOException || e is MailBoxOutException) { maxMessagesPerSession = 0; // stop checking other mailboxes } else if (e is TenantQuotaException) { quotaErrorFlag = true; } if (uid != uidsCollection.First() && uid != toUid) { imapIntervals.AddHandledInterval(new UidInterval(uid + 1, toUid)); } toUid = uid - 1; if (maxMessagesPerSession == 0) break; continue; } // after successfully message saving - lets update imap intervals state imapIntervals.AddHandledInterval( new UidInterval( uid == uidsCollection.Last() && uidsInterval.IsFromUidMin() ? uidsInterval.From : uid, toUid)); toUid = uid - 1; UpdateTimeCheckedIfNeeded(); maxMessagesPerSession--; if (maxMessagesPerSession != 0) continue; log.Info("Limit of max messages per session is exceeded!"); break; } #endregion } var updatedImapFolderUids = new ImapFolderUids(imapIntervals.ToIndexes(), beginDateUid); if (!Account.ImapIntervals.Keys.Contains(mb.Name)) { Account.ImapFolderChanged = true; Account.ImapIntervals.Add(mb.Name, updatedImapFolderUids); } else if (Account.ImapIntervals[mb.Name] != updatedImapFolderUids) { Account.ImapFolderChanged = true; Account.ImapIntervals[mb.Name] = updatedImapFolderUids; } return maxMessagesPerSession; }