public override void Initialise(List <MailFolder> folderList)
{
Status = MessageProcessorStatus.Initialising;
_mainFolderList = folderList;
service = ExchangeHelper.ExchangeConnect(_hostname, _username, _password);
//service.TraceEnabled = true;
//service.TraceFlags = TraceFlags.All;
folders = new List <ExchangeFolder>();
ExchangeHelper.GetAllSubFolders(service, new ExchangeFolder {
Folder = Folder.Bind(service, WellKnownFolderName.MsgFolderRoot)
}, folders, false);
if (_createAllFolders)
{
// we need to create all folders which don't exist
foreach (var mailFolder in _mainFolderList)
{
GetCreateFolder(mailFolder.DestinationFolder);
}
}
_lastState = new ImportState();
_queue = new PCQueue <RawMessageDescriptor, ImportState>(Name + "-exchangeTarget")
{
ProduceMethod = ProcessMessage,
InitialiseProducer = () => _lastState,
ShutdownProducer = ShutdownQueue
};
_queue.Start();
Status = MessageProcessorStatus.Initialised;
}
public override void Initialise(List <MailFolder> folderList)
{
Status = MessageProcessorStatus.Initialising;
service = ExchangeHelper.ExchangeConnect(_hostname, _username, _password);
folders = new List <ExchangeFolder>();
// Use Exchange Helper to get all the folders for this account
ExchangeHelper.GetAllSubFolders(service,
new ExchangeFolder()
{
Folder = Folder.Bind(service, WellKnownFolderName.MsgFolderRoot)
}, folders,
false);
if (IncludePublicFolders)
{
Logger.Debug("Including Public Folders");
ExchangeHelper.GetAllSubFolders(service,
new ExchangeFolder()
{
Folder = Folder.Bind(service, WellKnownFolderName.PublicFoldersRoot), IsPublicFolder = true
}, folders,
false);
}
// Are we limited folders to a specific list?
if (_limitFolderList != null)
{
var newFolders = new List <ExchangeFolder>();
foreach (var mailbox in _limitFolderList)
{
var mailboxMatch = mailbox.ToLower().Replace('/', '\\');;
newFolders.AddRange(folders.Where(folder => folder.FolderPath.ToLower().Equals(mailboxMatch)));
}
folders = newFolders;
}
// Scan the folders to get message counts
ExchangeHelper.GetFolderSummary(service, folders, _startDate, _endDate);
folders.ForEach(folder => TotalMessages += !TestOnly ? folder.MessageCount : (folder.MessageCount > 20 ? 20 : folder.MessageCount));
Logger.Debug("Found " + folders.Count + " folders and " + TotalMessages + " messages.");
// Now build the folder list that we pass on to the next folders.
foreach (var exchangeFolder in folders)
{
var folder = new MailFolder()
{
SourceFolder = exchangeFolder.FolderPath,
DestinationFolder = exchangeFolder.MappedDestination,
MessageCount = exchangeFolder.MessageCount,
};
_mainFolderList.Add(folder);
}
// Now initialise the next read, I am not going to start reading unless I know the pipeline is groovy
NextReader.Initialise(_mainFolderList);
Status = MessageProcessorStatus.Initialised;
Logger.Info("ExchangeExporter Initialised");
}
private void Run()
{
try
{
Status = MessageProcessorStatus.Started;
var fullPropertySet = new PropertySet(PropertySet.FirstClassProperties)
{
EmailMessageSchema.IsRead,
EmailMessageSchema.IsReadReceiptRequested,
EmailMessageSchema.IsDeliveryReceiptRequested,
ItemSchema.DateTimeSent,
ItemSchema.DateTimeReceived,
ItemSchema.DateTimeCreated,
ItemSchema.ItemClass,
ItemSchema.MimeContent,
ItemSchema.Categories,
ItemSchema.Importance,
ItemSchema.InReplyTo,
ItemSchema.IsFromMe,
ItemSchema.IsReminderSet,
ItemSchema.IsResend,
ItemSchema.IsDraft,
ItemSchema.ReminderDueBy,
ItemSchema.Sensitivity,
ItemSchema.Subject,
ItemSchema.Id,
ExchangeHelper.MsgPropertyContentType,
ExchangeHelper.PidTagFollowupIcon,
};
if (service.RequestedServerVersion != ExchangeVersion.Exchange2007_SP1)
{
fullPropertySet.Add(ItemSchema.ConversationId);
fullPropertySet.Add(ItemSchema.IsAssociated);
}
if (service.RequestedServerVersion == ExchangeVersion.Exchange2013)
{
fullPropertySet.Add(ItemSchema.ArchiveTag);
fullPropertySet.Add(ItemSchema.Flag);
fullPropertySet.Add(ItemSchema.IconIndex);
}
SearchFilter.SearchFilterCollection filter = new SearchFilter.SearchFilterCollection();
filter.LogicalOperator = LogicalOperator.And;
if (_startDate != null)
{
Logger.Debug("Getting mails from " + _startDate);
filter.Add(new SearchFilter.IsGreaterThanOrEqualTo(ItemSchema.DateTimeReceived, _startDate));
}
if (_endDate != null)
{
Logger.Debug("Getting mails up until " + _endDate);
filter.Add(new SearchFilter.IsLessThanOrEqualTo(ItemSchema.DateTimeReceived, _endDate));
}
foreach (var exchangeFolder in folders)
{
ItemView view = new ItemView(_pageSize, 0, OffsetBasePoint.Beginning);
view.PropertySet = PropertySet.IdOnly;
List <EmailMessage> emails = new List <EmailMessage>();
Boolean more = true;
while (more)
{
try
{
more = FindExchangeMessages(exchangeFolder, filter, view, emails, fullPropertySet);
if (emails.Count > 0)
{
try
{
foreach (var emailMessage in emails)
{
try
{
String subject;
if (!emailMessage.TryGetProperty(ItemSchema.Subject, out subject) ||
subject == null)
{
Logger.Warn("Item " + emailMessage.Id.UniqueId + " has no subject assigned, unable to determine subject.");
}
Logger.Debug("Exporting " + emailMessage.Id.UniqueId + " from " + exchangeFolder.FolderPath + " : " + subject);
var flags = new Collection <MessageFlags>();
Boolean flag;
if (emailMessage.TryGetProperty(EmailMessageSchema.IsRead, out flag) &&
!flag)
{
flags.Add(MessageFlags.Unread);
}
if (emailMessage.TryGetProperty(ItemSchema.IsDraft, out flag) && flag)
{
flags.Add(MessageFlags.Draft);
}
if (
emailMessage.TryGetProperty(EmailMessageSchema.IsReadReceiptRequested,
out flag) && flag)
{
flags.Add(MessageFlags.ReadReceiptRequested);
}
if (
emailMessage.TryGetProperty(
EmailMessageSchema.IsDeliveryReceiptRequested, out flag) && flag)
{
flags.Add(MessageFlags.DeliveryReceiptRequested);
}
if (emailMessage.TryGetProperty(ItemSchema.IsReminderSet, out flag) && flag)
{
flags.Add(MessageFlags.ReminderSet);
}
if (emailMessage.TryGetProperty(ItemSchema.IsAssociated, out flag) && flag)
{
flags.Add(MessageFlags.Associated);
}
if (emailMessage.TryGetProperty(ItemSchema.IsFromMe, out flag) && flag)
{
flags.Add(MessageFlags.FromMe);
}
if (emailMessage.TryGetProperty(ItemSchema.IsResend, out flag) && flag)
{
flags.Add(MessageFlags.Resend);
}
var message = new RawMessageDescriptor
{
SourceId = emailMessage.Id.UniqueId,
Subject = subject,
Flags = flags,
RawMessage = "",
SourceFolder = exchangeFolder.FolderPath,
DestinationFolder = exchangeFolder.MappedDestination,
IsPublicFolder = exchangeFolder.IsPublicFolder,
};
Object result;
if (emailMessage.TryGetProperty(ItemSchema.MimeContent, out result) &&
result != null)
{
message.RawMessage = Encoding.UTF8.GetString(emailMessage.MimeContent.Content);
}
if (emailMessage.TryGetProperty(ItemSchema.ItemClass, out result) &&
result != null)
{
message.ItemClass = emailMessage.ItemClass;
}
if (emailMessage.TryGetProperty(ItemSchema.IconIndex, out result) &&
result != null)
{
message.IconIndex = (int)emailMessage.IconIndex;
}
if (emailMessage.TryGetProperty(ItemSchema.Importance, out result) &&
result != null)
{
message.Importance = (int)emailMessage.Importance;
}
if (emailMessage.TryGetProperty(ItemSchema.Sensitivity, out result) &&
result != null)
{
message.Sensitivity = (int)emailMessage.Sensitivity;
}
if (emailMessage.TryGetProperty(ItemSchema.InReplyTo, out result) &&
result != null)
{
message.InReplyTo = emailMessage.InReplyTo;
}
if (emailMessage.TryGetProperty(ItemSchema.ConversationId, out result) &&
result != null)
{
message.ConversationId = emailMessage.ConversationId.ChangeKey;
}
if (emailMessage.TryGetProperty(ItemSchema.ReminderDueBy, out result) &&
result != null)
{
message.ReminderDueBy = emailMessage.ReminderDueBy;
}
if (
emailMessage.TryGetProperty(ExchangeHelper.PidTagFollowupIcon,
out result) && result != null)
{
message.FlagIcon = ExchangeHelper.ConvertFlagIcon((int)result);
}
if (emailMessage.TryGetProperty(ItemSchema.DateTimeReceived, out result) &&
result != null)
{
message.ReceivedDateTime = emailMessage.DateTimeReceived;
}
if (emailMessage.TryGetProperty(ItemSchema.DateTimeSent, out result) &&
result != null)
{
message.SentDateTime = emailMessage.DateTimeSent;
}
if (emailMessage.TryGetProperty(ItemSchema.Flag, out result) &&
result != null)
{
message.FollowUpFlag = new FollowUpFlag()
{
StartDateTime = ((Flag)result).StartDate,
DueDateTime = ((Flag)result).DueDate,
CompleteDateTime = ((Flag)result).CompleteDate,
Status =
ExchangeHelper.ConvertFlagStatus(((Flag)result).FlagStatus),
}
}
;
if (emailMessage.TryGetProperty(ItemSchema.Categories, out result) &&
result != null && emailMessage.Categories.Count > 0)
{
foreach (var category in emailMessage.Categories)
{
message.Categories.Add(category);
}
}
if (emailMessage.ExtendedProperties != null)
{
foreach (var extendedProperty in emailMessage.ExtendedProperties)
{
if (
extendedProperty.PropertyDefinition.Equals(
ExchangeHelper.MsgPropertyContentType))
{
if (extendedProperty.Value.ToString().Contains("signed-data"))
{
message.IsEncrypted = true;
}
}
}
}
NextReader.Process(message);
SucceededMessageCount++;
}
catch (Exception e)
{
Logger.Error("Failed to load properties for message " + emailMessage.Id.UniqueId, e);
FailedMessageCount++;
}
}
}
catch (Exception e)
{
Logger.Error("Failed to load properties for messages in " + exchangeFolder.FolderPath, e);
FailedMessageCount += emails.Count;
}
ProcessedMessageCount += emails.Count;
}
if (more)
{
view.Offset += _pageSize;
}
}
catch (Exception e)
{
Logger.Error("Failed to find results against folder " + exchangeFolder.FolderPath, e);
more = false;
}
}
}
}
catch (Exception e)
{
Logger.Error("Failed to run exporter", e);
}
finally
{
Close();
}
}
private FolderId GetCreateFolder(string destinationFolder, bool secondAttempt = false)
{
lock (_folderCreationLock)
{
if (!folders.Any(folder => folder.FolderPath.Equals(destinationFolder)))
{
// folder doesn't exist
// getCreate its parent
var parentPath = Regex.Replace(destinationFolder, @"\\[^\\]+$", "");
FolderId parentFolderId = null;
if (parentPath.Equals(destinationFolder))
{
// we are at the root
parentPath = "";
parentFolderId = WellKnownFolderName.MsgFolderRoot;
}
else
{
parentFolderId = GetCreateFolder(parentPath);
}
Logger.Debug("Folder " + destinationFolder + " doesn't exist, creating.");
var destinationFolderName = Regex.Replace(destinationFolder, @"^.*\\", "");
Folder folder = new Folder(service)
{
DisplayName = destinationFolderName
};
try
{
folder.Save(parentFolderId);
}
catch (Exception e)
{
// If the folder exists, we need to refresh and have another crack
if (e.Message.Equals("A folder with the specified name already exists.") && !secondAttempt)
{
Logger.Warn("Looks like the folder " + destinationFolder + " was created under our feet, refreshing the folder list. We will only attempt this once per folder.");
// Oops, the folders have been updated on the server, we need a refresh
var newFolders = new List <ExchangeFolder>();
ExchangeHelper.GetAllSubFolders(service, new ExchangeFolder {
Folder = Folder.Bind(service, WellKnownFolderName.MsgFolderRoot)
}, newFolders, false);
folders = newFolders;
// lets try again
return(GetCreateFolder(destinationFolder, true));
}
throw e;
}
folders.Add(new ExchangeFolder()
{
Folder = folder,
FolderId = folder.Id,
FolderPath =
String.IsNullOrEmpty(parentPath)
? destinationFolderName
: parentPath + @"\" + destinationFolderName,
});
return(folder.Id);
}
else
{
return(folders.First(folder => folder.FolderPath.Equals(destinationFolder)).FolderId);
}
}
}
private EmailMessage PrepareEWSItem(RawMessageDescriptor msg)
{
var headersString = msg.RawMessage;
var headerLimit = msg.RawMessage.IndexOf("\n\n");
if (headerLimit < 0)
{
headerLimit = msg.RawMessage.IndexOf("\r\n\r\n");
}
if (headerLimit > 0)
{
headersString = msg.RawMessage.Substring(0, headerLimit);
}
// sometimes there is no received header, we will have to modify this so Outlook sorts this correctly
if (!headersString.Contains("Received: ") && !msg.DestinationFolder.Equals("Sent Items"))
{
var match = Regex.Match(headersString, @"Date: (.*)");
if (match.Success)
{
Logger.Debug("Missing Received header, adding dummy header");
msg.RawMessage = "Received: from zinkuba.export (127.0.0.1) by\r\n" +
" zinkuba.import (127.0.0.1) with Zinkuba id 0.0.0.0;\r\n" +
" " + match.Groups[1] +
"\r\n" +
msg.RawMessage;
}
}
EmailMessage item = new EmailMessage(service)
{
MimeContent = new MimeContent("UTF-8", Encoding.UTF8.GetBytes(msg.RawMessage))
};
if (msg.DestinationFolder.Equals("Sent Items"))
{
// we need to set a sent date property otherwise dates don't show up properly in exchange
DateTime?sentDate = msg.SentDateTime;
if (sentDate == null)
{
var match = Regex.Match(headersString, @"Date: (.*)");
if (match.Success)
{
try
{
sentDate = DateTime.Parse(match.Groups[1].Value);
}
catch (Exception e)
{
Logger.Error("Failed to parse header date " + match.Groups[1] + " to a date for sending.");
}
}
}
if (sentDate == null)
{
Logger.Error("Failed to set sent date on " + msg.Subject);
}
else
{
try
{
item.SetExtendedProperty(ExchangeHelper.MsgPropertyDateTimeSent,
sentDate.Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"));
item.SetExtendedProperty(ExchangeHelper.MsgPropertyDateTimeReceived,
sentDate.Value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ"));
}
catch (Exception e)
{
Logger.Error("Failed to set sent item date on " + msg.Subject + " to " + sentDate, e);
}
}
}
String folder = msg.DestinationFolder;
// This is required to be set one way or another, otherwise the message is marked as new (not delivered)
item.IsRead = !msg.Flags.Contains(MessageFlags.Unread);
// Set the defaults to no receipts, to be corrected later by flags.
item.IsReadReceiptRequested = false;
item.IsDeliveryReceiptRequested = false;
if (!String.IsNullOrEmpty(msg.ItemClass))
{
item.ItemClass = msg.ItemClass;
}
else
{
// default Item.Class
item.ItemClass = "IPM.Note";
// we need to detect the item class
if (headersString.Contains("Return-Path: <>"))
{
// Looks like it may be a bounce
if (headersString.Contains("X-MS-Exchange-Message-Is-Ndr:") ||
headersString.Contains("Content-Type: multipart/report; report-type=delivery-status;") ||
headersString.Contains("X-Failed-Recipients:")
)
{
item.ItemClass = "REPORT.IPM.NOTE.NDR";
}
}
}
foreach (var messageFlag in msg.Flags)
{
try
{
switch (messageFlag)
{
case MessageFlags.Associated:
{
item.IsAssociated = true;
break;
}
case MessageFlags.FollowUp:
{
item.Flag = new Flag()
{
FlagStatus = ItemFlagStatus.Flagged
};
break;
}
case MessageFlags.ReminderSet:
{
item.IsReminderSet = true;
break;
}
case MessageFlags.ReadReceiptRequested:
{
// If the item is read already, we don't set a read receipt request as it will send one on save.
if (!item.IsRead)
{
item.IsReadReceiptRequested = true;
}
break;
}
case MessageFlags.DeliveryReceiptRequested:
{
// this causes spam
//item.IsDeliveryReceiptRequested = true;
break;
}
}
}
catch (Exception e)
{
Logger.Warn("Failed to set flag on " + folder + @"\" + msg.Subject + ", ignoring flag.", e);
}
}
if (msg.FlagIcon != FlagIcon.None)
{
item.SetExtendedProperty(ExchangeHelper.PidTagFlagStatus, 2);
item.SetExtendedProperty(ExchangeHelper.PidTagFollowupIcon, ExchangeHelper.ConvertFlagIcon(msg.FlagIcon));
}
if (service.RequestedServerVersion == ExchangeVersion.Exchange2013 && msg.FollowUpFlag != null)
{
item.Flag = new Flag()
{
DueDate = msg.FollowUpFlag.DueDateTime,
StartDate = msg.FollowUpFlag.StartDateTime,
CompleteDate = msg.FollowUpFlag.CompleteDateTime,
FlagStatus = ExchangeHelper.ConvertFlagStatus(msg.FollowUpFlag.Status),
};
}
try
{
if (!msg.Flags.Contains(MessageFlags.Draft))
{
item.SetExtendedProperty(ExchangeHelper.MsgFlagRead, 1);
}
if (msg.Importance != null)
{
item.Importance = (Importance)msg.Importance;
}
if (msg.Sensitivity != null)
{
item.Sensitivity = (Sensitivity)msg.Sensitivity;
}
if (msg.ReminderDueBy != null)
{
item.ReminderDueBy = (DateTime)msg.ReminderDueBy;
}
item.Categories = new StringList(msg.Categories);
}
catch (Exception e)
{
Logger.Warn(
"Failed to set metadata on " + folder + @"\" + msg.Subject + ", ignoring metadata.", e);
}
return(item);
}
