async Task <List <Mailbox> > GetMailboxesList(ImapCmd cmd, StatusCb status, ProgressCb progress)
{
List <Mailbox> mailboxes = new List <Mailbox>();
if (await cmd.Run(new ListCommand(async delegate(String mailbox, String ctx)
{ await Task.Yield(); mailboxes.Add(new Mailbox(mailbox)); })) != ReturnCode.Ok)
{
throw new FailedException();
}
mailboxes.Sort((Mailbox m1, Mailbox m2) => {
bool g1 = GmailSpecial(m1.name);
bool g2 = GmailSpecial(m2.name);
if (g1 && g2)
{
return(String.Compare(m2.name, m1.name)); // reverse
}
else if (g1)
{
return(1);
}
else if (g2)
{
return(-1);
}
else
{
return(String.Compare(m1.name, m2.name));
}
});
for (int i = 0; i < mailboxes.Count; i++)
{
await cmd.Run(new StatusCommand(mailboxes[i].name, async delegate(String cnt, String ctx)
{
await Task.Yield();
status("{0} - {1}", mailboxes[i].name, cnt);
mailboxes[i].cnt = int.Parse(cnt);
messagesInAccount += int.Parse(cnt);
}));
}
status("Total messages: {0}", messagesInAccount);
Dictionary <String, int> downloaded = await CheckResume(status, progress);
foreach (String mbox in downloaded.Keys)
{
mailboxes.ForEach((Mailbox m) => {
if (m.name == mbox)
{
m.start = downloaded[mbox] > 0 ? downloaded[mbox] : 1;
}
});
}
return(mailboxes);
}
async Task <Dictionary <String, int> > CheckResume(StatusCb status, ProgressCb progress)
{
Dictionary <String, int> downloaded = new Dictionary <string, int>();
String index = Path.Combine(Path.GetDirectoryName(dldFile), "email_proc1596.index");
if (File.Exists(index))
{
try {
Regex rx = new Regex("^((\"[^\"]+\")|([^ ]+))(.*)$");
int read = 0;
FileInfo info = new FileInfo(index);
long length = info.Length;
bool first = true;
using (StreamReader reader = new StreamReader(info.OpenRead()))
{
String line = await reader.ReadLineAsync(); // host user download file name
while ((line = await reader.ReadLineAsync()) != null)
{
if (first)
{
first = false;
status("Resuming download");
status("Calculating resume point...");
}
Match m = rx.Match(line);
progress((100.0 * read) / length);
if (m.Success)
{
if (downloaded.ContainsKey(m.Groups[1].Value) == false)
{
downloaded[m.Groups[1].Value] = 0;
}
downloaded[m.Groups[1].Value]++;
if (m.Groups[4].Success && m.Groups[4].Value != "")
{
messageid[m.Groups[4].Value] = "";
}
}
read += line.Length + 1;
}
if (downloaded.Count > 0)
{
progress(100);
}
reader.Close();
}
} catch { }
}
return(downloaded);
}
private void PopulateStatusCb()
{
Dictionary <string, decimal> cbData = new Dictionary <string, decimal>();
var taskStatuses = context.TASK_STATUS.ToList();
foreach (var stat in taskStatuses)
{
cbData.Add(stat.STATUS_NAME, stat.STATUS_ID);
}
this.StatusCb.DataSource = new BindingSource(cbData, null);
this.StatusCb.DisplayMember = "Key";
this.StatusCb.ValueMember = "Value";
StatusCb.Refresh();
cbData.Clear();
}
public async Task Start(StatusCb status, DataCb data, ProgressCb progress, bool compression = false)
{
try
{
status("Connecting...");
if (await connect.Connect() != ConnectStatus.Ok)
{
throw new ConnectionFailedException();
}
ImapCmd cmd = null;
if (await connect.ConnectTls() == ConnectStatus.Failed)
{
cmd = CreateImap(connect.stream);
if (await cmd.Run(new StarttlsCommand()) != ReturnCode.Ok)
{
throw new SslFailedException();
}
}
cmd = CreateImap(connect.stream);
// initial Capability
await cmd.Run(new NullCommand());
// login to the server
if (await cmd.Run(new LoginCommand(user, pswd)) != ReturnCode.Ok)
{
throw new FailedLoginException();
}
status("Logged in to the email server");
// check if compression is supported
if (compression && await cmd.Run(new CapabilityCommand(new String[] { "compress=deflate" })) == ReturnCode.Ok &&
await cmd.Run(new DeflateCommand()) == ReturnCode.Ok)
{
StreamReader reader = new StreamReader(new DeflateStream(connect.stream, CompressionMode.Decompress, true));
cmd = new ImapCmd(token, reader, connect.stream, true);
status("Compression enabled");
}
// get the list of mailboxes
status("Fetching mailbox list...");
mailboxes = await GetMailboxesList(cmd, status, progress);
status("Downloading ...");
progress(0);
int processed = 0;
Regex rx_msgid = new Regex("^message-id: ([^ \r\n]+)", RegexOptions.IgnoreCase | RegexOptions.Multiline);
Func <String, String> getMsgId = (String message) =>
{
int len = message.Length > 5000 ? 5000 : message.Length;
Match m = rx_msgid.Match(message, 0, len);
return(m.Success ? m.Groups[1].Value : "");
};
foreach (Mailbox mailbox in mailboxes)
{
if (token.IsCancellationRequested)
{
return;
}
status("{0} - {1} messages: ", mailbox.name, mailbox.cnt);
if (mailbox.start < mailbox.cnt)
{
if (await cmd.Run(new SelectCommand(mailbox.name)) != ReturnCode.Ok)
{
throw new FailedException();
}
if (!GmailSpecial(mailbox.name))
{
if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body,
async delegate(String message, String msgn)
{
mailbox.start = int.Parse(msgn);
String msgid = "";
if (host == "imap.gmail.com")
{
msgid = getMsgId(message);
messageid[msgid] = "";
}
progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount);
await data(mailbox.name, message, msgid, msgn);
}, mailbox.start)) != ReturnCode.Ok)
{
status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded");
}
}
else
{
List <String> unique = new List <string>();
progress(0);
if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.MessageID,
async delegate(String message, String msgn)
{
mailbox.start = int.Parse(msgn);
String msgid = getMsgId(message);
if (msgid != "" && messageid.ContainsKey(msgid))
{
return;
}
else
{
messageid[msgid] = "";
unique.Add(msgn);
}
progress(100.0 * int.Parse(msgn) / mailbox.cnt);
await Task.Yield();
})) != ReturnCode.Ok)
{
progress(0);
status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded, failed to get the list of unique messages");
}
else
{
progress(0);
status("Downloading {0} out of {1}", unique.Count, mailbox.cnt);
status("{0} - {1} messages: ", mailbox.name, unique.Count);
int downloaded = 0;
foreach (String n in unique)
{
int num = int.Parse(n);
if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body,
async delegate(String message, String msgn)
{
progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount);
await data(mailbox.name, message, getMsgId(message), msgn);
downloaded++;
}, num, num)) != ReturnCode.Ok)
{
status("failed to download {0}", n);
continue;
}
}
if (token.IsCancellationRequested)
{
status("Download cancelled");
}
if (unique.Count != downloaded)
{
status("{0} out of {1} failed to download", unique.Count - downloaded, unique.Count);
}
}
}
}
processed += mailbox.cnt;
mailbox.start = mailbox.cnt;
}
status("Download complete");
}
finally
{
connect.Close();
}
}
public async Task Start(String dir, String file, StatusCb status, ProgressCb pcb)
{
StreamWriter filew = null;
try {
status(false, "", "Started statistics processing.");
MessageReader reader = new MessageReader(file);
long size = reader.BaseStream.Length;
double progress = .0;
StringBuilder sb = new StringBuilder();
sb = new StringBuilder();
sb.AppendFormat(@"{0}\stats{1}.out", dir, DateTime.Now.ToFileTime());
file = sb.ToString();
filew = new StreamWriter(file);
DateTime start_time = DateTime.Now;
await WriteStatsLine(filew, "archive size: {0}\n", size);
int count = 1;
await EmailParser.ParseMessages(token, reader, async delegate (Message message, Exception reason)
{
status(true, "^message:", "message: {0}", count++);
try
{
if (null == message)
{
status(false, "", "message parsing failed: " + (null != reason ? reason.Message : ""));
await WriteStatsLine(filew, "--> start");
await WriteStatsLine(filew, "<-- end failed to process: {0}", (null != reason) ? reason.Message : "");
return;
}
// display progress
progress += message.size;
double pct = (100.0 * progress / (double)size);
pcb(pct);
// get required headers
Dictionary<String, String> headers = message.email.headers.GetDictionary(new Dictionary<string, string>()
{ {"from","" }, { "cc", "" }, {"subject","" }, {"date","" },
{ "to",""}, {"bcc", "" }, { "in-reply-to","" }, {"reply-to","" }, {"content-type","" }, {"message-id","" }, { "x-gmail-labels",""}});
String msgid = headers["message-id"];
// get unique messages
if (msgid != null && msgid != "" && MessageidUnique(msgid) == false)
{
return;
}
await WriteStatsLine(filew, "--> start");
int csize = await CompressedSize(message.GetBytes());
await WriteStatsLine(filew, "Full Message: {0} {1}", message.size, csize);
await WriteStatsLine(filew, "Hdrs");
await WriteStatsLine(filew, "from: {0}", Sha1(EmailAddr(headers["from"])));
await WriteStatsLine(filew, "to: {0}", GetAddrList(headers["to"]));
await WriteStatsLine(filew, "cc: {0}", GetAddrList(headers["cc"]));
await WriteStatsLine(filew, "bcc: {0}", GetAddrList(headers["bcc"]));
await WriteStatsLine(filew, "date: {0}", headers["date"]);
await WriteStatsLine(filew, "subject: {0}", GetSubject(headers["subject"]));
await WriteStatsLine(filew, "mailbox: {0}", GetMailbox(headers["x-gmail-labels"]));
await WriteStatsLine(filew, "messageid: {0}", GetMessageId(headers["message-id"]));
await WriteStatsLine(filew, "inreplyto: {0}", GetInReplyTo(headers["in-reply-to"]));
await WriteStatsLine(filew, "replyto: {0}", GetAddrList(headers["reply-to"]));
await WriteStatsLine(filew, "Parts:");
await TraverseEmail(filew, 0, 0, message.email);
await WriteStatsLine(filew, "<-- end");
}
catch (Exception ex)
{
await WriteStatsLine(filew, "<-- end failed to process: {0}, {1}", ex.Message, ex.StackTrace);
}
});
status(false, "", "Statistics is generated in file {0}", file);
TimeSpan span = DateTime.Now - start_time;
status(false, "", "Processing time: {0} seconds", span.TotalSeconds);
}
catch (Exception ex)
{
status(false, "", "Statistics failed: {0}", ex.Message);
}
finally
{
if (filew != null)
filew.Close();
}
}
public async Task Start(String dir, String file, StatusCb status, ProgressCb pcb)
{
StreamWriter filew = null;
try {
status(false, "", "Started statistics processing.");
MessageReader reader = new MessageReader(file);
long size = reader.BaseStream.Length;
double progress = .0;
StringBuilder sb = new StringBuilder();
sb = new StringBuilder();
sb.AppendFormat(@"{0}\stats{1}.out", dir, DateTime.Now.ToFileTime());
file = sb.ToString();
filew = new StreamWriter(file);
DateTime start_time = DateTime.Now;
await WriteStatsLine(filew, "archive size: {0}\n", size);
int count = 1;
await EmailParser.ParseMessages(token, reader, async delegate(Message message, Exception reason)
{
status(true, "^message:", "message: {0}", count++);
try
{
if (null == message)
{
status(false, "", "message parsing failed: " + (null != reason ? reason.Message : ""));
await WriteStatsLine(filew, "--> start");
await WriteStatsLine(filew, "<-- end failed to process: {0}", (null != reason) ? reason.Message : "");
return;
}
// display progress
progress += message.size;
double pct = (100.0 * progress / (double)size);
pcb(pct);
// get required headers
Dictionary <String, String> headers = message.email.headers.GetDictionary(new Dictionary <string, string>()
{
{ "from", "" }, { "cc", "" }, { "subject", "" }, { "date", "" },
{ "to", "" }, { "bcc", "" }, { "in-reply-to", "" }, { "reply-to", "" }, { "content-type", "" }, { "message-id", "" }, { "x-gmail-labels", "" }
});
String msgid = headers["message-id"];
// get unique messages
if (msgid != null && msgid != "" && MessageidUnique(msgid) == false)
{
return;
}
await WriteStatsLine(filew, "--> start");
int csize = await CompressedSize(message.GetBytes());
await WriteStatsLine(filew, "Full Message: {0} {1}", message.size, csize);
await WriteStatsLine(filew, "Hdrs");
await WriteStatsLine(filew, "from: {0}", Sha1(EmailAddr(headers["from"])));
await WriteStatsLine(filew, "to: {0}", GetAddrList(headers["to"]));
await WriteStatsLine(filew, "cc: {0}", GetAddrList(headers["cc"]));
await WriteStatsLine(filew, "bcc: {0}", GetAddrList(headers["bcc"]));
await WriteStatsLine(filew, "date: {0}", headers["date"]);
await WriteStatsLine(filew, "subject: {0}", GetSubject(headers["subject"]));
await WriteStatsLine(filew, "mailbox: {0}", GetMailbox(headers["x-gmail-labels"]));
await WriteStatsLine(filew, "messageid: {0}", GetMessageId(headers["message-id"]));
await WriteStatsLine(filew, "inreplyto: {0}", GetInReplyTo(headers["in-reply-to"]));
await WriteStatsLine(filew, "replyto: {0}", GetAddrList(headers["reply-to"]));
await WriteStatsLine(filew, "Parts:");
await TraverseEmail(filew, 0, 0, message.email);
await WriteStatsLine(filew, "<-- end");
}
catch (Exception ex)
{
await WriteStatsLine(filew, "<-- end failed to process: {0}, {1}", ex.Message, ex.StackTrace);
}
});
status(false, "", "Statistics is generated in file {0}", file);
TimeSpan span = DateTime.Now - start_time;
status(false, "", "Processing time: {0} seconds", span.TotalSeconds);
}
catch (Exception ex)
{
status(false, "", "Statistics failed: {0}", ex.Message);
}
finally
{
if (filew != null)
{
filew.Close();
}
}
}
async Task<Dictionary<String,int>> CheckResume(StatusCb status, ProgressCb progress)
{
Dictionary<String, int> downloaded = new Dictionary<string, int>();
String index = Path.Combine(Path.GetDirectoryName(dldFile), "email_proc1596.index");
if (File.Exists(index))
{
try {
Regex rx = new Regex("^((\"[^\"]+\")|([^ ]+))(.*)$");
int read = 0;
FileInfo info = new FileInfo(index);
long length = info.Length;
bool first = true;
using (StreamReader reader = new StreamReader(info.OpenRead()))
{
String line = await reader.ReadLineAsync(); // host user download file name
while ((line = await reader.ReadLineAsync()) != null)
{
if (first)
{
first = false;
status("Resuming download");
status("Calculating resume point...");
}
Match m = rx.Match(line);
progress((100.0 * read) / length);
if (m.Success)
{
if (downloaded.ContainsKey(m.Groups[1].Value) == false)
downloaded[m.Groups[1].Value] = 0;
downloaded[m.Groups[1].Value]++;
if (m.Groups[4].Success && m.Groups[4].Value != "")
messageid[m.Groups[4].Value] = "";
}
read += line.Length + 1;
}
if (downloaded.Count > 0)
progress(100);
reader.Close();
}
} catch { }
}
return downloaded;
}
public async Task Start(StatusCb status, DataCb data, ProgressCb progress, bool compression=false)
{
try
{
status("Connecting...");
if (await connect.Connect() != ConnectStatus.Ok)
throw new ConnectionFailedException();
ImapCmd cmd = null;
if (await connect.ConnectTls() == ConnectStatus.Failed)
{
cmd = CreateImap(connect.stream);
if (await cmd.Run(new StarttlsCommand()) != ReturnCode.Ok)
throw new SslFailedException();
}
cmd = CreateImap(connect.stream);
// initial Capability
await cmd.Run(new NullCommand());
// login to the server
if (await cmd.Run(new LoginCommand(user, pswd)) != ReturnCode.Ok)
throw new FailedLoginException();
status("Logged in to the email server");
// check if compression is supported
if (compression && await cmd.Run(new CapabilityCommand(new String[] { "compress=deflate" })) == ReturnCode.Ok &&
await cmd.Run(new DeflateCommand()) == ReturnCode.Ok)
{
StreamReader reader = new StreamReader(new DeflateStream(connect.stream, CompressionMode.Decompress, true));
cmd = new ImapCmd(token, reader, connect.stream, true);
status("Compression enabled");
}
// get the list of mailboxes
status("Fetching mailbox list...");
mailboxes = await GetMailboxesList(cmd,status,progress);
status("Downloading ...");
progress(0);
int processed = 0;
Regex rx_msgid = new Regex("^message-id: ([^ \r\n]+)", RegexOptions.IgnoreCase | RegexOptions.Multiline);
Func<String, String> getMsgId = (String message) =>
{
int len = message.Length > 5000 ? 5000 : message.Length;
Match m = rx_msgid.Match(message, 0, len);
return (m.Success ? m.Groups[1].Value : "");
};
foreach (Mailbox mailbox in mailboxes)
{
if (token.IsCancellationRequested)
return;
status("{0} - {1} messages: ", mailbox.name, mailbox.cnt);
if (mailbox.start < mailbox.cnt)
{
if (await cmd.Run(new SelectCommand(mailbox.name)) != ReturnCode.Ok)
throw new FailedException();
if (!GmailSpecial(mailbox.name))
{
if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body,
async delegate (String message, String msgn)
{
mailbox.start = int.Parse(msgn);
String msgid = "";
if (host == "imap.gmail.com")
{
msgid = getMsgId(message);
messageid[msgid] = "";
}
progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount);
await data(mailbox.name, message, msgid, msgn);
}, mailbox.start)) != ReturnCode.Ok)
{
status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded");
}
}
else
{
List<String> unique = new List<string>();
progress(0);
if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.MessageID,
async delegate (String message, String msgn)
{
mailbox.start = int.Parse(msgn);
String msgid = getMsgId(message);
if (msgid != "" && messageid.ContainsKey(msgid))
return;
else
{
messageid[msgid] = "";
unique.Add(msgn);
}
progress(100.0 * int.Parse(msgn) / mailbox.cnt);
await Task.Yield();
})) != ReturnCode.Ok)
{
progress(0);
status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded, failed to get the list of unique messages");
}
else
{
progress(0);
status("Downloading {0} out of {1}", unique.Count, mailbox.cnt);
status("{0} - {1} messages: ", mailbox.name, unique.Count);
int downloaded = 0;
foreach (String n in unique)
{
int num = int.Parse(n);
if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body,
async delegate (String message, String msgn)
{
progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount);
await data(mailbox.name, message, getMsgId(message), msgn);
downloaded++;
}, num, num)) != ReturnCode.Ok)
{
status("failed to download {0}", n);
continue;
}
}
if (token.IsCancellationRequested)
status("Download cancelled");
if (unique.Count != downloaded)
status("{0} out of {1} failed to download", unique.Count-downloaded, unique.Count);
}
}
}
processed += mailbox.cnt;
mailbox.start = mailbox.cnt;
}
status("Download complete");
}
finally
{
connect.Close();
}
}
async Task<List<Mailbox>> GetMailboxesList(ImapCmd cmd, StatusCb status, ProgressCb progress)
{
List<Mailbox> mailboxes = new List<Mailbox>();
if (await cmd.Run(new ListCommand(async delegate (String mailbox, String ctx)
{ await Task.Yield(); mailboxes.Add(new Mailbox(mailbox)); })) != ReturnCode.Ok)
throw new FailedException();
mailboxes.Sort((Mailbox m1, Mailbox m2) => {
bool g1 = GmailSpecial(m1.name);
bool g2 = GmailSpecial(m2.name);
if (g1 && g2)
return String.Compare(m2.name, m1.name); // reverse
else if (g1)
return 1;
else if (g2)
return -1;
else
return String.Compare(m1.name, m2.name);
});
for (int i = 0; i < mailboxes.Count; i++)
{
await cmd.Run(new StatusCommand(mailboxes[i].name, async delegate (String cnt, String ctx)
{
await Task.Yield();
status("{0} - {1}", mailboxes[i].name, cnt);
mailboxes[i].cnt = int.Parse(cnt);
messagesInAccount += int.Parse(cnt);
}));
}
status("Total messages: {0}", messagesInAccount);
Dictionary<String, int> downloaded = await CheckResume(status, progress);
foreach (String mbox in downloaded.Keys)
{
mailboxes.ForEach((Mailbox m) => {
if (m.name == mbox)
m.start = downloaded[mbox] > 0 ? downloaded[mbox] : 1;
});
}
return mailboxes;
}
public async static Task ParseToResume(CancellationToken token, String addr, String user, String file, StatusCb status, ProgressCb progress)
{
String index = Path.Combine(Path.GetDirectoryName(file), "email_proc1596.index");
FileInfo info = new FileInfo(file);
long length = info.Length;
FileStream stream = info.OpenRead();
status("Generating resume file...");
using (StreamReader reader = new StreamReader(stream))
{
using (StreamWriter writer = new StreamWriter(index))
{
await writer.WriteLineAsync(addr + " " + user + " " + file);
String line = "";
String mailbox = "";
String messageid = "";
Regex re_mailbox = new Regex("^X-Mailbox: (.+)$");
Regex re_messageid = new Regex("^Message-ID: ([^ ]+)", RegexOptions.IgnoreCase);
while ((line = await reader.ReadLineAsync()) != null)
{
if (token.IsCancellationRequested)
return;
progress(100.0 * stream.Position / length);
Match m = re_mailbox.Match(line);
if (m.Success)
{
if (mailbox != "")
{
if (messageid != "")
messageid = " " + messageid;
await writer.WriteLineAsync(mailbox + messageid);
}
mailbox = m.Groups[1].Value;
messageid = "";
}
else if (messageid == "")
{
m = re_messageid.Match(line);
if (m.Success)
messageid = m.Groups[1].Value;
}
}
if (mailbox != "")
{
if (messageid != "")
messageid = " " + messageid;
await writer.WriteLineAsync(mailbox + messageid);
}
}
}
}
public async static Task ParseToResume(CancellationToken token, String addr, String user, String file, StatusCb status, ProgressCb progress)
{
String index = Path.Combine(Path.GetDirectoryName(file), "email_proc1596.index");
FileInfo info = new FileInfo(file);
long length = info.Length;
FileStream stream = info.OpenRead();
status("Generating resume file...");
using (StreamReader reader = new StreamReader(stream))
{
using (StreamWriter writer = new StreamWriter(index))
{
await writer.WriteLineAsync(addr + " " + user + " " + file);
String line = "";
String mailbox = "";
String messageid = "";
Regex re_mailbox = new Regex("^X-Mailbox: (.+)$");
Regex re_messageid = new Regex("^Message-ID: ([^ ]+)", RegexOptions.IgnoreCase);
while ((line = await reader.ReadLineAsync()) != null)
{
if (token.IsCancellationRequested)
{
return;
}
progress(100.0 * stream.Position / length);
Match m = re_mailbox.Match(line);
if (m.Success)
{
if (mailbox != "")
{
if (messageid != "")
{
messageid = " " + messageid;
}
await writer.WriteLineAsync(mailbox + messageid);
}
mailbox = m.Groups[1].Value;
messageid = "";
}
else if (messageid == "")
{
m = re_messageid.Match(line);
if (m.Success)
{
messageid = m.Groups[1].Value;
}
}
}
if (mailbox != "")
{
if (messageid != "")
{
messageid = " " + messageid;
}
await writer.WriteLineAsync(mailbox + messageid);
}
}
}
}
