private static long DownloadMails(DataContext dc, PcapCapture pcap)
{
List <string> log = new List <string>();
List <string> logError = new List <string>()
{
"", ""
};
DateTime dtStart = DateTime.UtcNow;
log.Add("");
log.Add("LOGIN PARAMS");
log.Add("Host name: " + dc.HostName + " (resolved ip = " + System.Net.Dns.GetHostEntry(dc.HostName).AddressList.FirstOrDefault() + ") ");
log.Add("Port: " + dc.Port);
log.Add("UseSSL: " + dc.UseSSL.ToString());
log.Add("User name: " + dc.UserName);
log.Add("User password (Base64): " + Convert.ToBase64String(Encoding.Default.GetBytes(dc.UserPassword)));
log.Add("");
if (dc.MergeFolders == false)
{
if (File.Exists(dc.DestinationFolder))
{
try
{
File.Delete(dc.DestinationFolder);
}
catch (Exception)
{
File.Move(dc.DestinationFolder, dc.DestinationFolder + "_old");
}
}
}
else
{
}
long totalMessagesDownloaded = 0;
// clone the list
TotalMails = dc.EmailFolders.Where(o => o.Selected).Sum(o => o.Messages);
string lastDirName = dc.DestinationFolder.Split('\\', '/').Where(o => !string.IsNullOrEmpty(o)).Last();
string superDir = dc.DestinationFolder.Substring(0, dc.DestinationFolder.Length - lastDirName.Length - 1);
string logFileName = Path.Combine(superDir, lastDirName + ".log");
if (!Directory.Exists(superDir))
{
Directory.CreateDirectory(superDir);
}
if (!dc.MergeFolders)
{
// Delete previous log file
if (File.Exists(logFileName))
{
File.Delete(logFileName);
}
// Delete previous pcap file
if (File.Exists(pcap.OutputFile))
{
try
{
File.Delete(pcap.OutputFile);
}
catch { }
}
}
pcap.StartCapture();
bool downloadFail = false;
Exception internalEx = null;
object writeEntryBlock = new object();
using (FileStream zipToOpen = new FileStream(dc.DestinationFolder, FileMode.OpenOrCreate))
{
ZipArchiveMode openMode = ZipArchiveMode.Create;
if (File.Exists(dc.DestinationFolder) && dc.MergeFolders)
{
openMode = ZipArchiveMode.Update;
}
using (ZipArchive archive = new ZipArchive(zipToOpen, openMode))
{
try
{
Parallel.ForEach(dc.EmailFolders, new ParallelOptions()
{
MaxDegreeOfParallelism = dc.ConcurrentThreads
}, (folder) =>
{
if (folder.Selected == false)
{
return;
}
// The default port for IMAP over SSL is 993.
using (ImapClient client = new ImapClient())
{
client.ServerCertificateValidationCallback = (s, c, h, ee) => true;
try
{
client.Connect(dc.HostName, dc.Port, dc.UseSSL);
}
catch (ImapProtocolException)
{
// try twice
System.Threading.Thread.Sleep(100);
client.Connect(dc.HostName, dc.Port, dc.UseSSL);
}
// wait 10 seconds if error is too many connection
var intWaitTime = 10 * 1000;
while (true)
{
try
{
client.Authenticate(dc.UserName, dc.UserPassword);
break;
}
catch (MailKit.Security.AuthenticationException ex)
{
System.Threading.Thread.Sleep(intWaitTime *= 2);
}
if (intWaitTime > 15 * 60 * 1000)
{
// is waiting time is greather than 15 min I assume the download fails
throw new Exception("Multiple connetion to host fails");
}
}
ImapFolder imapFodler = (ImapFolder)client.GetFolder(folder.Folder);
folder.IsDownloading = true;
folder.DownloadedItems = 0;
string destZipFolder = folder.Folder.Replace(imapFodler.DirectorySeparator, '\\');
// remove wrong chars
var illegalChars = Path.GetInvalidFileNameChars().ToList();
// remove folder separator
illegalChars.Remove('\\');
destZipFolder = string.Join("_", destZipFolder.Split(illegalChars.ToArray()));
string messageIdSafeName = "";
try
{
imapFodler.Open(FolderAccess.ReadOnly);
}
catch (Exception)
{
logError.Add("Error: can't select imap folder '" + folder.Folder + "'");
return;
}
//IList<IMessageSummary> items = imapFodler.Fetch(0, -1, MessageSummaryItems.UniqueId | MessageSummaryItems.Size);
IList <IMessageSummary> items = imapFodler.Fetch(0, -1, MessageSummaryItems.UniqueId | MessageSummaryItems.Size | MessageSummaryItems.InternalDate | MessageSummaryItems.Flags);
DateTime dt = DateTime.Now;
long fileSize = 0;
MimeMessage msg;
long folderSize = items.Sum(o => o.Size ?? 0);
List <string> AlreadyExistingEntries = new List <string>();
if (dc.MergeFolders)
{
AlreadyExistingEntries = archive.Entries.Select(o => o.FullName).OrderBy(o => o).ToList();
}
foreach (var item in items)
{
if (dc.MergeFolders)
{
// search entry before start downloading
if (AlreadyExistingEntries.Any(o => o.StartsWith(destZipFolder + "\\" + item.UniqueId + "_")))
{
logError.Add("Log: message id " + item.UniqueId + " already downloaded from folder '" + folder.Folder + "'");
totalMessagesDownloaded++;
folder.DownloadedItems++;
continue;
}
else
{
}
}
dt = DateTime.Now;
fileSize = 0;
try
{
msg = imapFodler.GetMessage(item.UniqueId);
}
catch
{
// Second attempt
try
{
msg = imapFodler.GetMessage(item.UniqueId);
}
catch (Exception ex)
{
// in the meanwhile a message has been deleted.. sometimes happens
logError.Add("Error: can't download message id " + item.UniqueId + " from folder '" + folder.Folder + "'");
continue;
}
}
ProgressMails++;
if (folder.Selected == false)
{
continue;
}
// msg not exsist
if (msg.From == null)
{
log.Add("Error: can't save message id " + item.UniqueId + " from folder '" + folder.Folder + "' because has no From field");
continue;
}
totalMessagesDownloaded++;
folder.DownloadedItems++;
messageIdSafeName = System.Text.RegularExpressions.Regex.Replace(msg.Headers["Message-ID"] + "", "[<>\\/:]", "");
string msgPrefix = item.UniqueId + "";
if (item.Flags != null && !item.Flags.Value.HasFlag(MessageFlags.Seen))
{
msgPrefix += "_N";
}
if (string.IsNullOrEmpty(messageIdSafeName))
{
messageIdSafeName = Guid.NewGuid().ToString();
}
var destFileName = destZipFolder + "\\" + msgPrefix + "_" + messageIdSafeName + ".eml";
lock (writeEntryBlock)
{
var entry = archive.CreateEntry(destFileName);
entry.LastWriteTime = item.InternalDate.Value;
using (Stream s = entry.Open())
{
msg.WriteTo(s);
fileSize = s.Position;
s.Close();
}
}
DownloadSpeed.Add(new Tuple <DateTime, double, long>(dt, DateTime.Now.Subtract(dt).TotalMilliseconds, fileSize));
if (totalMessagesDownloaded % 1024 == 0)
{
GCCollectUtils.CheckAndFreeMemory();
zipToOpen.FlushAsync();
}
}
folder.IsDownloading = false;
try
{
imapFodler.Close();
}
catch (MailKit.ServiceNotConnectedException)
{
}
log.Add("Folder: " + folder.Folder + "\t\t" + folder.DownloadedItems + " emails");
}
});
}
catch (Exception ex)
{
// someting worong but the zip file is safe
internalEx = ex;
downloadFail = true;
}
if (pcap != null && pcap.IsCapturing)
{
// Add pcap file to archive
pcap.StopCapture();
var pcapName = pcap.OutputFile.Split('\\').Last();
if (File.Exists(pcap.OutputFile))
{
using (FileStream fileStream = new FileStream(pcap.OutputFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
{
var entry = archive.CreateEntry(pcapName);
using (Stream s = entry.Open())
{
fileStream.CopyTo(s);
s.Close();
}
}
}
log.Add("Pcap: " + pcapName + "");
}
else
{
log.Add("No Pcap file.");
}
}
}
if (downloadFail)
{
throw internalEx;
}
log.Add("");
log.Add("Total emails: " + totalMessagesDownloaded);
DateTime dtEnd = DateTime.UtcNow;
log.Add("");
log.Add("Startd at " + dtStart.ToUniversalTime() + " UTC");
log.Add("End at " + dtEnd.ToUniversalTime() + " UTC");
log.Add("");
dc.PartialPercent = 100;
log.Add("Export file : " + dc.DestinationFolder);
dc.Speed30sec = "";
dc.SpeedTotal = "Calculating hash..";
string md5 = CalculateMD5(dc.DestinationFolder).Replace("-", "");
string sha1 = CalculateSHA1(dc.DestinationFolder).Replace("-", "");
log.Add("MD5 : " + md5);
log.Add("SHA1 : " + sha1);
if (File.Exists(logFileName))
{
File.Delete(logFileName);
}
File.WriteAllLines(logFileName, log.Union(logError));
dc.PartialPercent = 100;
dc.Speed30sec = "DONE! ";
dc.SpeedTotal = string.Format(" It took {0} ", DateTime.UtcNow.Subtract(dtStart));
return(totalMessagesDownloaded);
}
private static long DownloadMails(DataContext dc)
{
List <string> log = new List <string>();
List <string> logError = new List <string>()
{
"", ""
};
DateTime dtStart = DateTime.UtcNow;
log.Add("");
log.Add("LOGIN PARAMS");
log.Add("Host name: " + dc.HostName + " (resolved ip = " + System.Net.Dns.GetHostEntry(dc.HostName).AddressList.FirstOrDefault() + ") ");
log.Add("Port: " + dc.Port);
log.Add("UseSSL: " + dc.UseSSL.ToString());
log.Add("User name: " + dc.UserName);
log.Add("User password: "******"");
long totalMessagesDownloaded = 0;
// clone the list
TotalMails = dc.EmailFolders.Where(o => o.Selected).Sum(o => o.Messages);
Parallel.ForEach(dc.EmailFolders, new ParallelOptions()
{
MaxDegreeOfParallelism = dc.ConcurrentThreads
}, (folder) =>
{
if (folder.Selected == false)
{
return;
}
// The default port for IMAP over SSL is 993.
using (ImapClient client = new ImapClient())
{
client.ServerCertificateValidationCallback = (s, c, h, ee) => true;
try
{
client.Connect(dc.HostName, dc.Port, dc.UseSSL);
}
catch (ImapProtocolException)
{
// try twice
System.Threading.Thread.Sleep(100);
client.Connect(dc.HostName, dc.Port, dc.UseSSL);
}
// Note: since we don't have an OAuth2 token, disable
// the XOAUTH2 authentication mechanism.
client.AuthenticationMechanisms.Remove("XOAUTH2");
client.Authenticate(dc.UserName, dc.UserPassword);
ImapFolder imapFodler = (ImapFolder)client.GetFolder(folder.Folder);
folder.IsDownloading = true;
folder.DownloadedItems = 0;
string destFolder = Path.Combine(dc.DestinationFolder, folder.Folder.Replace(imapFodler.DirectorySeparator, '\\'));
// If the folder already exsist I have do delete it
if (!dc.MergeFolders)
{
if (Directory.Exists(destFolder))
{
Directory.Delete(destFolder, true);
}
}
if (!Directory.Exists(destFolder))
{
Directory.CreateDirectory(destFolder);
}
string messageIdSafeName = "";
int downloadedEmails = 0;
try
{
imapFodler.Open(FolderAccess.ReadOnly);
}
catch (Exception)
{
logError.Add("Error: can't select imap folder '" + folder.Folder + "'");
return;
}
//IList<IMessageSummary> items = imapFodler.Fetch(0, -1, MessageSummaryItems.UniqueId | MessageSummaryItems.Size);
IList <IMessageSummary> items = imapFodler.Fetch(0, -1, MessageSummaryItems.UniqueId | MessageSummaryItems.Size);
DateTime dt = DateTime.Now;
long fileSize = 0;
MimeMessage msg;
long folderSize = items.Sum(o => o.Size ?? 0);
foreach (var item in items)
{
if (dc.MergeFolders)
{
var file = Directory.GetFiles(destFolder, item.UniqueId + "_*.eml");
if (file.Count() == 1)
{
logError.Add("Log: message id " + item.UniqueId + " already downloaded from folder '" + folder.Folder + "'");
downloadedEmails++;
folder.DownloadedItems++;
continue;
}
else
{
}
}
dt = DateTime.Now;
fileSize = 0;
try
{
msg = imapFodler.GetMessage(item.UniqueId);
}
catch
{
// Second attempt
try
{
msg = imapFodler.GetMessage(item.UniqueId);
}
catch (Exception ex)
{
// in the meanwhile a message has been deleted.. sometimes happens
logError.Add("Error: can't download message id " + item.UniqueId + " from folder '" + folder.Folder + "'");
continue;
}
}
ProgressMails++;
if (folder.Selected == false)
{
continue;
}
// msg not exsist
if (msg.From == null)
{
log.Add("Error: can't save message id " + item.UniqueId + " from folder '" + folder.Folder + "' because has no From field");
continue;
}
downloadedEmails++;
folder.DownloadedItems++;
messageIdSafeName = System.Text.RegularExpressions.Regex.Replace(msg.Headers["Message-ID"] + "", "[<>\\/]", "");
if (string.IsNullOrEmpty(messageIdSafeName))
{
messageIdSafeName = Guid.NewGuid().ToString();
}
else if (messageIdSafeName.Length > 250)
{
// i'll take the lst 250 characters
messageIdSafeName = messageIdSafeName.Substring(messageIdSafeName.Length - 250);
}
try
{
using (var fs = new FileStream(Path.Combine(destFolder, item.UniqueId + "_" + messageIdSafeName + ".eml"), FileMode.Create))
{
msg.WriteTo(fs);
fileSize = fs.Length;
}
}
catch (PathTooLongException)
{
logError.Add("Warning: message id " + item.UniqueId + " from folder '" + folder.Folder + "' will be saved with name '" + item.UniqueId + ".eml' because '" + item.UniqueId + "_" + messageIdSafeName + ".eml' is too long");
using (var fs = new FileStream(Path.Combine(destFolder, item.UniqueId + ".eml"), FileMode.Create))
{
msg.WriteTo(fs);
fileSize = fs.Length;
}
}
DownloadSpeed.Add(new Tuple <DateTime, double, long>(dt, DateTime.Now.Subtract(dt).TotalMilliseconds, fileSize));
}
folder.IsDownloading = false;
try
{
imapFodler.Close();
}
catch (MailKit.ServiceNotConnectedException)
{
}
log.Add("Folder: " + folder.Folder + "\t\t" + downloadedEmails + " emails");
totalMessagesDownloaded += downloadedEmails;
}
});
log.Add("");
log.Add("Total emails: " + totalMessagesDownloaded);
DateTime dtEnd = DateTime.UtcNow;
log.Add("");
log.Add("Startd at " + dtStart.ToUniversalTime() + " UTC");
log.Add("End at " + dtEnd.ToUniversalTime() + " UTC");
log.Add("");
dc.PartialPercent = 100;
string lastDirName = dc.DestinationFolder.Split('\\', '/').Where(o => !string.IsNullOrEmpty(o)).Last();
string superDir = dc.DestinationFolder.Substring(0, dc.DestinationFolder.Length - lastDirName.Length - 1);
string zipFileName = Path.Combine(superDir, lastDirName + ".zip");
if (File.Exists(zipFileName))
{
File.Delete(zipFileName);
}
dc.PartialPercent = 0;
dc.Speed30sec = "";
dc.SpeedTotal = "Creating archive..";
ZipFile.CreateFromDirectory(dc.DestinationFolder, zipFileName, CompressionLevel.Fastest, true);
log.Add("Export file : " + zipFileName);
dc.Speed30sec = "";
dc.SpeedTotal = "Calculating hash..";
string md5 = CalculateMD5(zipFileName).Replace("-", "");
string sha1 = CalculateSHA1(zipFileName).Replace("-", "");
log.Add("MD5 : " + md5);
log.Add("SHA1 : " + sha1);
string logFileName = Path.Combine(superDir, lastDirName + ".log");
if (File.Exists(logFileName))
{
File.Delete(logFileName);
}
File.WriteAllLines(logFileName, log.Union(logError));
Directory.Delete(dc.DestinationFolder, true);
dc.PartialPercent = 100;
dc.Speed30sec = "DONE! ";
dc.SpeedTotal = string.Format(" It took {0} ", DateTime.UtcNow.Subtract(dtStart));
return(totalMessagesDownloaded);
}