public override void Initialise(List <MailFolder> folderList)
{
Status = MessageProcessorStatus.Initialising;
/*
* try
* {
* _filestream = new StreamReader(_mboxPath);
* }
* catch (Exception ex)
* {
* Logger.Error("Opening mbox file [" + _mboxPath + "] failed : " + ex.Message, ex);
* Status = MessageProcessorStatus.UnknownError;
* throw ex;
* }
*/
// Build the folder list
foreach (var folderMap in _folderMap)
{
_mainFolderList.Add(new MailFolder()
{
DestinationFolder = folderMap.Value,
SourceFolder = folderMap.Value,
});
}
NextReader.Initialise(_mainFolderList);
Status = MessageProcessorStatus.Initialised;
}
public override void Close()
{
if (!Closed)
{
Closed = true;
NextReader.Close();
}
}
public override void Initialise(List <MailFolder> folderList)
{
Status = MessageProcessorStatus.Initialising;
service = ExchangeHelper.ExchangeConnect(_hostname, _username, _password);
folders = new List <ExchangeFolder>();
// Use Exchange Helper to get all the folders for this account
ExchangeHelper.GetAllSubFolders(service,
new ExchangeFolder()
{
Folder = Folder.Bind(service, WellKnownFolderName.MsgFolderRoot)
}, folders,
false);
if (IncludePublicFolders)
{
Logger.Debug("Including Public Folders");
ExchangeHelper.GetAllSubFolders(service,
new ExchangeFolder()
{
Folder = Folder.Bind(service, WellKnownFolderName.PublicFoldersRoot), IsPublicFolder = true
}, folders,
false);
}
// Are we limited folders to a specific list?
if (_limitFolderList != null)
{
var newFolders = new List <ExchangeFolder>();
foreach (var mailbox in _limitFolderList)
{
var mailboxMatch = mailbox.ToLower().Replace('/', '\\');;
newFolders.AddRange(folders.Where(folder => folder.FolderPath.ToLower().Equals(mailboxMatch)));
}
folders = newFolders;
}
// Scan the folders to get message counts
ExchangeHelper.GetFolderSummary(service, folders, _startDate, _endDate);
folders.ForEach(folder => TotalMessages += !TestOnly ? folder.MessageCount : (folder.MessageCount > 20 ? 20 : folder.MessageCount));
Logger.Debug("Found " + folders.Count + " folders and " + TotalMessages + " messages.");
// Now build the folder list that we pass on to the next folders.
foreach (var exchangeFolder in folders)
{
var folder = new MailFolder()
{
SourceFolder = exchangeFolder.FolderPath,
DestinationFolder = exchangeFolder.MappedDestination,
MessageCount = exchangeFolder.MessageCount,
};
_mainFolderList.Add(folder);
}
// Now initialise the next read, I am not going to start reading unless I know the pipeline is groovy
NextReader.Initialise(_mainFolderList);
Status = MessageProcessorStatus.Initialised;
Logger.Info("ExchangeExporter Initialised");
}
public override void Close()
{
if (!Closed)
{
Closed = true;
if (_imapClient != null)
{
try
{
_imapClient.Logout();
}
catch (Exception ex)
{
Logger.Error("Failed to logout : " + ex.Message, ex);
}
}
ThreadPool.QueueUserWorkItem(state => NextReader.Close());
}
}
private void Run()
{
try
{
Status = MessageProcessorStatus.Started;
var fullPropertySet = new PropertySet(PropertySet.FirstClassProperties)
{
EmailMessageSchema.IsRead,
EmailMessageSchema.IsReadReceiptRequested,
EmailMessageSchema.IsDeliveryReceiptRequested,
ItemSchema.DateTimeSent,
ItemSchema.DateTimeReceived,
ItemSchema.DateTimeCreated,
ItemSchema.ItemClass,
ItemSchema.MimeContent,
ItemSchema.Categories,
ItemSchema.Importance,
ItemSchema.InReplyTo,
ItemSchema.IsFromMe,
ItemSchema.IsReminderSet,
ItemSchema.IsResend,
ItemSchema.IsDraft,
ItemSchema.ReminderDueBy,
ItemSchema.Sensitivity,
ItemSchema.Subject,
ItemSchema.Id,
ExchangeHelper.MsgPropertyContentType,
ExchangeHelper.PidTagFollowupIcon,
};
if (service.RequestedServerVersion != ExchangeVersion.Exchange2007_SP1)
{
fullPropertySet.Add(ItemSchema.ConversationId);
fullPropertySet.Add(ItemSchema.IsAssociated);
}
if (service.RequestedServerVersion == ExchangeVersion.Exchange2013)
{
fullPropertySet.Add(ItemSchema.ArchiveTag);
fullPropertySet.Add(ItemSchema.Flag);
fullPropertySet.Add(ItemSchema.IconIndex);
}
SearchFilter.SearchFilterCollection filter = new SearchFilter.SearchFilterCollection();
filter.LogicalOperator = LogicalOperator.And;
if (_startDate != null)
{
Logger.Debug("Getting mails from " + _startDate);
filter.Add(new SearchFilter.IsGreaterThanOrEqualTo(ItemSchema.DateTimeReceived, _startDate));
}
if (_endDate != null)
{
Logger.Debug("Getting mails up until " + _endDate);
filter.Add(new SearchFilter.IsLessThanOrEqualTo(ItemSchema.DateTimeReceived, _endDate));
}
foreach (var exchangeFolder in folders)
{
ItemView view = new ItemView(_pageSize, 0, OffsetBasePoint.Beginning);
view.PropertySet = PropertySet.IdOnly;
List <EmailMessage> emails = new List <EmailMessage>();
Boolean more = true;
while (more)
{
try
{
more = FindExchangeMessages(exchangeFolder, filter, view, emails, fullPropertySet);
if (emails.Count > 0)
{
try
{
foreach (var emailMessage in emails)
{
try
{
String subject;
if (!emailMessage.TryGetProperty(ItemSchema.Subject, out subject) ||
subject == null)
{
Logger.Warn("Item " + emailMessage.Id.UniqueId + " has no subject assigned, unable to determine subject.");
}
Logger.Debug("Exporting " + emailMessage.Id.UniqueId + " from " + exchangeFolder.FolderPath + " : " + subject);
var flags = new Collection <MessageFlags>();
Boolean flag;
if (emailMessage.TryGetProperty(EmailMessageSchema.IsRead, out flag) &&
!flag)
{
flags.Add(MessageFlags.Unread);
}
if (emailMessage.TryGetProperty(ItemSchema.IsDraft, out flag) && flag)
{
flags.Add(MessageFlags.Draft);
}
if (
emailMessage.TryGetProperty(EmailMessageSchema.IsReadReceiptRequested,
out flag) && flag)
{
flags.Add(MessageFlags.ReadReceiptRequested);
}
if (
emailMessage.TryGetProperty(
EmailMessageSchema.IsDeliveryReceiptRequested, out flag) && flag)
{
flags.Add(MessageFlags.DeliveryReceiptRequested);
}
if (emailMessage.TryGetProperty(ItemSchema.IsReminderSet, out flag) && flag)
{
flags.Add(MessageFlags.ReminderSet);
}
if (emailMessage.TryGetProperty(ItemSchema.IsAssociated, out flag) && flag)
{
flags.Add(MessageFlags.Associated);
}
if (emailMessage.TryGetProperty(ItemSchema.IsFromMe, out flag) && flag)
{
flags.Add(MessageFlags.FromMe);
}
if (emailMessage.TryGetProperty(ItemSchema.IsResend, out flag) && flag)
{
flags.Add(MessageFlags.Resend);
}
var message = new RawMessageDescriptor
{
SourceId = emailMessage.Id.UniqueId,
Subject = subject,
Flags = flags,
RawMessage = "",
SourceFolder = exchangeFolder.FolderPath,
DestinationFolder = exchangeFolder.MappedDestination,
IsPublicFolder = exchangeFolder.IsPublicFolder,
};
Object result;
if (emailMessage.TryGetProperty(ItemSchema.MimeContent, out result) &&
result != null)
{
message.RawMessage = Encoding.UTF8.GetString(emailMessage.MimeContent.Content);
}
if (emailMessage.TryGetProperty(ItemSchema.ItemClass, out result) &&
result != null)
{
message.ItemClass = emailMessage.ItemClass;
}
if (emailMessage.TryGetProperty(ItemSchema.IconIndex, out result) &&
result != null)
{
message.IconIndex = (int)emailMessage.IconIndex;
}
if (emailMessage.TryGetProperty(ItemSchema.Importance, out result) &&
result != null)
{
message.Importance = (int)emailMessage.Importance;
}
if (emailMessage.TryGetProperty(ItemSchema.Sensitivity, out result) &&
result != null)
{
message.Sensitivity = (int)emailMessage.Sensitivity;
}
if (emailMessage.TryGetProperty(ItemSchema.InReplyTo, out result) &&
result != null)
{
message.InReplyTo = emailMessage.InReplyTo;
}
if (emailMessage.TryGetProperty(ItemSchema.ConversationId, out result) &&
result != null)
{
message.ConversationId = emailMessage.ConversationId.ChangeKey;
}
if (emailMessage.TryGetProperty(ItemSchema.ReminderDueBy, out result) &&
result != null)
{
message.ReminderDueBy = emailMessage.ReminderDueBy;
}
if (
emailMessage.TryGetProperty(ExchangeHelper.PidTagFollowupIcon,
out result) && result != null)
{
message.FlagIcon = ExchangeHelper.ConvertFlagIcon((int)result);
}
if (emailMessage.TryGetProperty(ItemSchema.DateTimeReceived, out result) &&
result != null)
{
message.ReceivedDateTime = emailMessage.DateTimeReceived;
}
if (emailMessage.TryGetProperty(ItemSchema.DateTimeSent, out result) &&
result != null)
{
message.SentDateTime = emailMessage.DateTimeSent;
}
if (emailMessage.TryGetProperty(ItemSchema.Flag, out result) &&
result != null)
{
message.FollowUpFlag = new FollowUpFlag()
{
StartDateTime = ((Flag)result).StartDate,
DueDateTime = ((Flag)result).DueDate,
CompleteDateTime = ((Flag)result).CompleteDate,
Status =
ExchangeHelper.ConvertFlagStatus(((Flag)result).FlagStatus),
}
}
;
if (emailMessage.TryGetProperty(ItemSchema.Categories, out result) &&
result != null && emailMessage.Categories.Count > 0)
{
foreach (var category in emailMessage.Categories)
{
message.Categories.Add(category);
}
}
if (emailMessage.ExtendedProperties != null)
{
foreach (var extendedProperty in emailMessage.ExtendedProperties)
{
if (
extendedProperty.PropertyDefinition.Equals(
ExchangeHelper.MsgPropertyContentType))
{
if (extendedProperty.Value.ToString().Contains("signed-data"))
{
message.IsEncrypted = true;
}
}
}
}
NextReader.Process(message);
SucceededMessageCount++;
}
catch (Exception e)
{
Logger.Error("Failed to load properties for message " + emailMessage.Id.UniqueId, e);
FailedMessageCount++;
}
}
}
catch (Exception e)
{
Logger.Error("Failed to load properties for messages in " + exchangeFolder.FolderPath, e);
FailedMessageCount += emails.Count;
}
ProcessedMessageCount += emails.Count;
}
if (more)
{
view.Offset += _pageSize;
}
}
catch (Exception e)
{
Logger.Error("Failed to find results against folder " + exchangeFolder.FolderPath, e);
more = false;
}
}
}
}
catch (Exception e)
{
Logger.Error("Failed to run exporter", e);
}
finally
{
Close();
}
}
public override void Initialise(List <MailFolder> folderList)
{
Status = MessageProcessorStatus.Initialising;
_folders = new List <ImapFolder>();
try
{
_imapClient = new ImapClient(_server, _useSsl ? 993 : 143, _username, _password, AuthMethod.Login, _useSsl,
delegate { return(true); });
Logger.Debug("Logged into " + _server + " as " + _username);
var folders = _imapClient.ListMailboxes();
if (_startDate != null || _endDate != null)
{
// ok we need to do a search
if (_startDate != null)
{
_searchCondition = SearchCondition.Since((DateTime)_startDate);
}
if (_endDate != null)
{
_searchCondition = _searchCondition == null
? SearchCondition.Before((DateTime)_endDate)
: _searchCondition.And(SearchCondition.Before((DateTime)_endDate));
}
Logger.Debug("Only getting messages " + _searchCondition);
}
// Are we limiting the folder list?
if (_limitFolderList != null)
{
var newFolders = new List <String>();
foreach (var mailbox in _limitFolderList)
{
var mailboxMatch = mailbox.ToLower().Replace('\\', '/');;
newFolders.AddRange(folders.Where(folder => folder.ToLower().Equals(mailboxMatch)));
}
folders = newFolders;
}
foreach (var folderPath in folders)
{
bool isPublicFolder = false;
var destinationFolder = FolderMapping.ApplyMappings(folderPath, Provider);
if (IncludePublicFolders && (String.Equals(destinationFolder, PublicFolderRoot) || destinationFolder.StartsWith(PublicFolderRoot + @"\")))
{
isPublicFolder = true;
var start = PublicFolderRoot.Length + (destinationFolder.StartsWith(PublicFolderRoot + @"\") ? 1 : 0);
destinationFolder = destinationFolder.Substring(start, destinationFolder.Length - start);
}
if (!String.IsNullOrWhiteSpace(destinationFolder))
{
try
{
var folder = _imapClient.GetMailboxInfo(folderPath);
if (folder.Messages == 0)
{
Logger.Debug("Skipping folder " + folderPath + ", no messages at all.");
continue;
}
int messageCount = 0;
if (_searchCondition != null)
{
var uids = _imapClient.Search(_searchCondition, folderPath);
messageCount = uids.Count();
}
else
{
messageCount = folder.Messages;
}
// Add it to our main folder list
_mainFolderList.Add(new MailFolder()
{
DestinationFolder = destinationFolder,
MessageCount = FailedMessageCount,
SourceFolder = folderPath
});
if (messageCount == 0)
{
Logger.Debug("Skipping folder " + folderPath + ", no messages within criteria.");
continue;
}
_folders.Add(new ImapFolder()
{
MappedDestination = destinationFolder,
FolderPath = folder,
IsPublicFolder = isPublicFolder
});
TotalMessages += !_testOnly ? messageCount : (messageCount > 20 ? 20 : messageCount);
Logger.Debug("Will process " + folderPath + " => " + (isPublicFolder ? "[PUBLIC FOLDER]/" : "") + destinationFolder + ", " + messageCount + " messages, " + TotalMessages + " messages total so far.");
}
catch (Exception ex)
{
Logger.Error("Failed to get Mailbox " + folderPath + ", skipping.", ex);
}
}
else
{
Logger.Info("Ignoring folder " + folderPath + ", no destination specified.");
}
}
}
catch (InvalidCredentialsException ex)
{
Logger.Error("Imap Runner for " + _username + " [********] to " + _server + " failed : " + ex.Message, ex);
throw new MessageProcessorException("Imap Runner for " + _username + " [********] to " + _server + " failed : " + ex.Message)
{
Status = MessageProcessorStatus.SourceAuthFailure
};
}
catch (SocketException ex)
{
Logger.Error("Imap Runner for " + _username + " [********] to " + _server + " failed : " + ex.Message, ex);
throw new MessageProcessorException("Imap Runner for " + _username + " [********] to " + _server + " failed : " + ex.Message)
{
Status = MessageProcessorStatus.ConnectionError
};
}
NextReader.Initialise(_mainFolderList);
Status = MessageProcessorStatus.Initialised;
Logger.Info("ExchangeExporter Initialised");
}
private void RunImap()
{
try
{
Status = MessageProcessorStatus.Started;
foreach (var mailbox in _folders)
{
if (Closed)
{
Logger.Warn("Cancellation requested");
break;
}
var folder = mailbox.FolderPath;
var folderPath = folder.Name;
var destinationFolder = mailbox.MappedDestination;
try
{
var uids = _imapClient.Search(_searchCondition ?? SearchCondition.All(), folderPath);
Logger.Info("Processing folder Name=" + folderPath + " Destination=" + destinationFolder +
", Count=" + folder.Messages + ", Unread=" + folder.Unread);
var count = folder.Messages;
var countQueued = 0;
foreach (var uid in uids.Reverse())
{
if (Closed)
{
Logger.Warn("Cancellation requested");
break;
}
if (_testOnly && count >= 20)
{
Logger.Debug("Testing only, hit 20 limit");
break;
}
try
{
var flags = _imapClient.GetMessageFlags(uid, folderPath).ToList();
if (flags.Contains(MessageFlag.Deleted))
{
Logger.Debug("Skipping message " + folderPath + "/" + uid + ", its marked for deletion");
IgnoredMessageCount++;
continue;
}
var message = GetImapMessage(_imapClient, folder, uid);
message.Subject = Regex.Match(message.RawMessage, @"[\r\n]Subject:\s*(.*?)[\r\n]").Groups[1].Value;
//Logger.Debug(folder + "/" + uid + "/" + subject + " " + String.Join(", ",flags));
Logger.Debug("Exporting " + uid + " from " + folder + " : " + message.Subject);
if (!flags.Contains(MessageFlag.Seen))
{
message.Flags.Add(MessageFlags.Unread);
}
if (flags.Contains(MessageFlag.Flagged))
{
message.Flags.Add(MessageFlags.FollowUp);
}
message.SourceFolder = folderPath;
message.DestinationFolder = destinationFolder;
message.IsPublicFolder = mailbox.IsPublicFolder;
NextReader.Process(message);
SucceededMessageCount++;
countQueued++;
}
catch (Exception ex)
{
Logger.Error("Failed to get and enqueue Imap message [" + uid + "]", ex);
FailedMessageCount++;
}
ProcessedMessageCount++;
}
Logger.Info("Processed folder " + folderPath + ", read=" + count + ", queued=" + countQueued);
}
catch (Exception ex)
{
Logger.Error("Failed to select " + folderPath, ex);
}
}
}
catch (Exception e)
{
Logger.Error("Failed to run exporter");
}
finally
{
Close();
}
}
