async Task <List <Mailbox> > GetMailboxesList(ImapCmd cmd, StatusCb status, ProgressCb progress) { List <Mailbox> mailboxes = new List <Mailbox>(); if (await cmd.Run(new ListCommand(async delegate(String mailbox, String ctx) { await Task.Yield(); mailboxes.Add(new Mailbox(mailbox)); })) != ReturnCode.Ok) { throw new FailedException(); } mailboxes.Sort((Mailbox m1, Mailbox m2) => { bool g1 = GmailSpecial(m1.name); bool g2 = GmailSpecial(m2.name); if (g1 && g2) { return(String.Compare(m2.name, m1.name)); // reverse } else if (g1) { return(1); } else if (g2) { return(-1); } else { return(String.Compare(m1.name, m2.name)); } }); for (int i = 0; i < mailboxes.Count; i++) { await cmd.Run(new StatusCommand(mailboxes[i].name, async delegate(String cnt, String ctx) { await Task.Yield(); status("{0} - {1}", mailboxes[i].name, cnt); mailboxes[i].cnt = int.Parse(cnt); messagesInAccount += int.Parse(cnt); })); } status("Total messages: {0}", messagesInAccount); Dictionary <String, int> downloaded = await CheckResume(status, progress); foreach (String mbox in downloaded.Keys) { mailboxes.ForEach((Mailbox m) => { if (m.name == mbox) { m.start = downloaded[mbox] > 0 ? downloaded[mbox] : 1; } }); } return(mailboxes); }
async Task <Dictionary <String, int> > CheckResume(StatusCb status, ProgressCb progress) { Dictionary <String, int> downloaded = new Dictionary <string, int>(); String index = Path.Combine(Path.GetDirectoryName(dldFile), "email_proc1596.index"); if (File.Exists(index)) { try { Regex rx = new Regex("^((\"[^\"]+\")|([^ ]+))(.*)$"); int read = 0; FileInfo info = new FileInfo(index); long length = info.Length; bool first = true; using (StreamReader reader = new StreamReader(info.OpenRead())) { String line = await reader.ReadLineAsync(); // host user download file name while ((line = await reader.ReadLineAsync()) != null) { if (first) { first = false; status("Resuming download"); status("Calculating resume point..."); } Match m = rx.Match(line); progress((100.0 * read) / length); if (m.Success) { if (downloaded.ContainsKey(m.Groups[1].Value) == false) { downloaded[m.Groups[1].Value] = 0; } downloaded[m.Groups[1].Value]++; if (m.Groups[4].Success && m.Groups[4].Value != "") { messageid[m.Groups[4].Value] = ""; } } read += line.Length + 1; } if (downloaded.Count > 0) { progress(100); } reader.Close(); } } catch { } } return(downloaded); }
private void PopulateStatusCb() { Dictionary <string, decimal> cbData = new Dictionary <string, decimal>(); var taskStatuses = context.TASK_STATUS.ToList(); foreach (var stat in taskStatuses) { cbData.Add(stat.STATUS_NAME, stat.STATUS_ID); } this.StatusCb.DataSource = new BindingSource(cbData, null); this.StatusCb.DisplayMember = "Key"; this.StatusCb.ValueMember = "Value"; StatusCb.Refresh(); cbData.Clear(); }
public async Task Start(StatusCb status, DataCb data, ProgressCb progress, bool compression = false) { try { status("Connecting..."); if (await connect.Connect() != ConnectStatus.Ok) { throw new ConnectionFailedException(); } ImapCmd cmd = null; if (await connect.ConnectTls() == ConnectStatus.Failed) { cmd = CreateImap(connect.stream); if (await cmd.Run(new StarttlsCommand()) != ReturnCode.Ok) { throw new SslFailedException(); } } cmd = CreateImap(connect.stream); // initial Capability await cmd.Run(new NullCommand()); // login to the server if (await cmd.Run(new LoginCommand(user, pswd)) != ReturnCode.Ok) { throw new FailedLoginException(); } status("Logged in to the email server"); // check if compression is supported if (compression && await cmd.Run(new CapabilityCommand(new String[] { "compress=deflate" })) == ReturnCode.Ok && await cmd.Run(new DeflateCommand()) == ReturnCode.Ok) { StreamReader reader = new StreamReader(new DeflateStream(connect.stream, CompressionMode.Decompress, true)); cmd = new ImapCmd(token, reader, connect.stream, true); status("Compression enabled"); } // get the list of mailboxes status("Fetching mailbox list..."); mailboxes = await GetMailboxesList(cmd, status, progress); status("Downloading ..."); progress(0); int processed = 0; Regex rx_msgid = new Regex("^message-id: ([^ \r\n]+)", RegexOptions.IgnoreCase | RegexOptions.Multiline); Func <String, String> getMsgId = (String message) => { int len = message.Length > 5000 ? 5000 : message.Length; Match m = rx_msgid.Match(message, 0, len); return(m.Success ? m.Groups[1].Value : ""); }; foreach (Mailbox mailbox in mailboxes) { if (token.IsCancellationRequested) { return; } status("{0} - {1} messages: ", mailbox.name, mailbox.cnt); if (mailbox.start < mailbox.cnt) { if (await cmd.Run(new SelectCommand(mailbox.name)) != ReturnCode.Ok) { throw new FailedException(); } if (!GmailSpecial(mailbox.name)) { if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body, async delegate(String message, String msgn) { mailbox.start = int.Parse(msgn); String msgid = ""; if (host == "imap.gmail.com") { msgid = getMsgId(message); messageid[msgid] = ""; } progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount); await data(mailbox.name, message, msgid, msgn); }, mailbox.start)) != ReturnCode.Ok) { status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded"); } } else { List <String> unique = new List <string>(); progress(0); if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.MessageID, async delegate(String message, String msgn) { mailbox.start = int.Parse(msgn); String msgid = getMsgId(message); if (msgid != "" && messageid.ContainsKey(msgid)) { return; } else { messageid[msgid] = ""; unique.Add(msgn); } progress(100.0 * int.Parse(msgn) / mailbox.cnt); await Task.Yield(); })) != ReturnCode.Ok) { progress(0); status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded, failed to get the list of unique messages"); } else { progress(0); status("Downloading {0} out of {1}", unique.Count, mailbox.cnt); status("{0} - {1} messages: ", mailbox.name, unique.Count); int downloaded = 0; foreach (String n in unique) { int num = int.Parse(n); if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body, async delegate(String message, String msgn) { progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount); await data(mailbox.name, message, getMsgId(message), msgn); downloaded++; }, num, num)) != ReturnCode.Ok) { status("failed to download {0}", n); continue; } } if (token.IsCancellationRequested) { status("Download cancelled"); } if (unique.Count != downloaded) { status("{0} out of {1} failed to download", unique.Count - downloaded, unique.Count); } } } } processed += mailbox.cnt; mailbox.start = mailbox.cnt; } status("Download complete"); } finally { connect.Close(); } }
public async Task Start(String dir, String file, StatusCb status, ProgressCb pcb) { StreamWriter filew = null; try { status(false, "", "Started statistics processing."); MessageReader reader = new MessageReader(file); long size = reader.BaseStream.Length; double progress = .0; StringBuilder sb = new StringBuilder(); sb = new StringBuilder(); sb.AppendFormat(@"{0}\stats{1}.out", dir, DateTime.Now.ToFileTime()); file = sb.ToString(); filew = new StreamWriter(file); DateTime start_time = DateTime.Now; await WriteStatsLine(filew, "archive size: {0}\n", size); int count = 1; await EmailParser.ParseMessages(token, reader, async delegate (Message message, Exception reason) { status(true, "^message:", "message: {0}", count++); try { if (null == message) { status(false, "", "message parsing failed: " + (null != reason ? reason.Message : "")); await WriteStatsLine(filew, "--> start"); await WriteStatsLine(filew, "<-- end failed to process: {0}", (null != reason) ? reason.Message : ""); return; } // display progress progress += message.size; double pct = (100.0 * progress / (double)size); pcb(pct); // get required headers Dictionary<String, String> headers = message.email.headers.GetDictionary(new Dictionary<string, string>() { {"from","" }, { "cc", "" }, {"subject","" }, {"date","" }, { "to",""}, {"bcc", "" }, { "in-reply-to","" }, {"reply-to","" }, {"content-type","" }, {"message-id","" }, { "x-gmail-labels",""}}); String msgid = headers["message-id"]; // get unique messages if (msgid != null && msgid != "" && MessageidUnique(msgid) == false) { return; } await WriteStatsLine(filew, "--> start"); int csize = await CompressedSize(message.GetBytes()); await WriteStatsLine(filew, "Full Message: {0} {1}", message.size, csize); await WriteStatsLine(filew, "Hdrs"); await WriteStatsLine(filew, "from: {0}", Sha1(EmailAddr(headers["from"]))); await WriteStatsLine(filew, "to: {0}", GetAddrList(headers["to"])); await WriteStatsLine(filew, "cc: {0}", GetAddrList(headers["cc"])); await WriteStatsLine(filew, "bcc: {0}", GetAddrList(headers["bcc"])); await WriteStatsLine(filew, "date: {0}", headers["date"]); await WriteStatsLine(filew, "subject: {0}", GetSubject(headers["subject"])); await WriteStatsLine(filew, "mailbox: {0}", GetMailbox(headers["x-gmail-labels"])); await WriteStatsLine(filew, "messageid: {0}", GetMessageId(headers["message-id"])); await WriteStatsLine(filew, "inreplyto: {0}", GetInReplyTo(headers["in-reply-to"])); await WriteStatsLine(filew, "replyto: {0}", GetAddrList(headers["reply-to"])); await WriteStatsLine(filew, "Parts:"); await TraverseEmail(filew, 0, 0, message.email); await WriteStatsLine(filew, "<-- end"); } catch (Exception ex) { await WriteStatsLine(filew, "<-- end failed to process: {0}, {1}", ex.Message, ex.StackTrace); } }); status(false, "", "Statistics is generated in file {0}", file); TimeSpan span = DateTime.Now - start_time; status(false, "", "Processing time: {0} seconds", span.TotalSeconds); } catch (Exception ex) { status(false, "", "Statistics failed: {0}", ex.Message); } finally { if (filew != null) filew.Close(); } }
public async Task Start(String dir, String file, StatusCb status, ProgressCb pcb) { StreamWriter filew = null; try { status(false, "", "Started statistics processing."); MessageReader reader = new MessageReader(file); long size = reader.BaseStream.Length; double progress = .0; StringBuilder sb = new StringBuilder(); sb = new StringBuilder(); sb.AppendFormat(@"{0}\stats{1}.out", dir, DateTime.Now.ToFileTime()); file = sb.ToString(); filew = new StreamWriter(file); DateTime start_time = DateTime.Now; await WriteStatsLine(filew, "archive size: {0}\n", size); int count = 1; await EmailParser.ParseMessages(token, reader, async delegate(Message message, Exception reason) { status(true, "^message:", "message: {0}", count++); try { if (null == message) { status(false, "", "message parsing failed: " + (null != reason ? reason.Message : "")); await WriteStatsLine(filew, "--> start"); await WriteStatsLine(filew, "<-- end failed to process: {0}", (null != reason) ? reason.Message : ""); return; } // display progress progress += message.size; double pct = (100.0 * progress / (double)size); pcb(pct); // get required headers Dictionary <String, String> headers = message.email.headers.GetDictionary(new Dictionary <string, string>() { { "from", "" }, { "cc", "" }, { "subject", "" }, { "date", "" }, { "to", "" }, { "bcc", "" }, { "in-reply-to", "" }, { "reply-to", "" }, { "content-type", "" }, { "message-id", "" }, { "x-gmail-labels", "" } }); String msgid = headers["message-id"]; // get unique messages if (msgid != null && msgid != "" && MessageidUnique(msgid) == false) { return; } await WriteStatsLine(filew, "--> start"); int csize = await CompressedSize(message.GetBytes()); await WriteStatsLine(filew, "Full Message: {0} {1}", message.size, csize); await WriteStatsLine(filew, "Hdrs"); await WriteStatsLine(filew, "from: {0}", Sha1(EmailAddr(headers["from"]))); await WriteStatsLine(filew, "to: {0}", GetAddrList(headers["to"])); await WriteStatsLine(filew, "cc: {0}", GetAddrList(headers["cc"])); await WriteStatsLine(filew, "bcc: {0}", GetAddrList(headers["bcc"])); await WriteStatsLine(filew, "date: {0}", headers["date"]); await WriteStatsLine(filew, "subject: {0}", GetSubject(headers["subject"])); await WriteStatsLine(filew, "mailbox: {0}", GetMailbox(headers["x-gmail-labels"])); await WriteStatsLine(filew, "messageid: {0}", GetMessageId(headers["message-id"])); await WriteStatsLine(filew, "inreplyto: {0}", GetInReplyTo(headers["in-reply-to"])); await WriteStatsLine(filew, "replyto: {0}", GetAddrList(headers["reply-to"])); await WriteStatsLine(filew, "Parts:"); await TraverseEmail(filew, 0, 0, message.email); await WriteStatsLine(filew, "<-- end"); } catch (Exception ex) { await WriteStatsLine(filew, "<-- end failed to process: {0}, {1}", ex.Message, ex.StackTrace); } }); status(false, "", "Statistics is generated in file {0}", file); TimeSpan span = DateTime.Now - start_time; status(false, "", "Processing time: {0} seconds", span.TotalSeconds); } catch (Exception ex) { status(false, "", "Statistics failed: {0}", ex.Message); } finally { if (filew != null) { filew.Close(); } } }
async Task<Dictionary<String,int>> CheckResume(StatusCb status, ProgressCb progress) { Dictionary<String, int> downloaded = new Dictionary<string, int>(); String index = Path.Combine(Path.GetDirectoryName(dldFile), "email_proc1596.index"); if (File.Exists(index)) { try { Regex rx = new Regex("^((\"[^\"]+\")|([^ ]+))(.*)$"); int read = 0; FileInfo info = new FileInfo(index); long length = info.Length; bool first = true; using (StreamReader reader = new StreamReader(info.OpenRead())) { String line = await reader.ReadLineAsync(); // host user download file name while ((line = await reader.ReadLineAsync()) != null) { if (first) { first = false; status("Resuming download"); status("Calculating resume point..."); } Match m = rx.Match(line); progress((100.0 * read) / length); if (m.Success) { if (downloaded.ContainsKey(m.Groups[1].Value) == false) downloaded[m.Groups[1].Value] = 0; downloaded[m.Groups[1].Value]++; if (m.Groups[4].Success && m.Groups[4].Value != "") messageid[m.Groups[4].Value] = ""; } read += line.Length + 1; } if (downloaded.Count > 0) progress(100); reader.Close(); } } catch { } } return downloaded; }
public async Task Start(StatusCb status, DataCb data, ProgressCb progress, bool compression=false) { try { status("Connecting..."); if (await connect.Connect() != ConnectStatus.Ok) throw new ConnectionFailedException(); ImapCmd cmd = null; if (await connect.ConnectTls() == ConnectStatus.Failed) { cmd = CreateImap(connect.stream); if (await cmd.Run(new StarttlsCommand()) != ReturnCode.Ok) throw new SslFailedException(); } cmd = CreateImap(connect.stream); // initial Capability await cmd.Run(new NullCommand()); // login to the server if (await cmd.Run(new LoginCommand(user, pswd)) != ReturnCode.Ok) throw new FailedLoginException(); status("Logged in to the email server"); // check if compression is supported if (compression && await cmd.Run(new CapabilityCommand(new String[] { "compress=deflate" })) == ReturnCode.Ok && await cmd.Run(new DeflateCommand()) == ReturnCode.Ok) { StreamReader reader = new StreamReader(new DeflateStream(connect.stream, CompressionMode.Decompress, true)); cmd = new ImapCmd(token, reader, connect.stream, true); status("Compression enabled"); } // get the list of mailboxes status("Fetching mailbox list..."); mailboxes = await GetMailboxesList(cmd,status,progress); status("Downloading ..."); progress(0); int processed = 0; Regex rx_msgid = new Regex("^message-id: ([^ \r\n]+)", RegexOptions.IgnoreCase | RegexOptions.Multiline); Func<String, String> getMsgId = (String message) => { int len = message.Length > 5000 ? 5000 : message.Length; Match m = rx_msgid.Match(message, 0, len); return (m.Success ? m.Groups[1].Value : ""); }; foreach (Mailbox mailbox in mailboxes) { if (token.IsCancellationRequested) return; status("{0} - {1} messages: ", mailbox.name, mailbox.cnt); if (mailbox.start < mailbox.cnt) { if (await cmd.Run(new SelectCommand(mailbox.name)) != ReturnCode.Ok) throw new FailedException(); if (!GmailSpecial(mailbox.name)) { if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body, async delegate (String message, String msgn) { mailbox.start = int.Parse(msgn); String msgid = ""; if (host == "imap.gmail.com") { msgid = getMsgId(message); messageid[msgid] = ""; } progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount); await data(mailbox.name, message, msgid, msgn); }, mailbox.start)) != ReturnCode.Ok) { status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded"); } } else { List<String> unique = new List<string>(); progress(0); if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.MessageID, async delegate (String message, String msgn) { mailbox.start = int.Parse(msgn); String msgid = getMsgId(message); if (msgid != "" && messageid.ContainsKey(msgid)) return; else { messageid[msgid] = ""; unique.Add(msgn); } progress(100.0 * int.Parse(msgn) / mailbox.cnt); await Task.Yield(); })) != ReturnCode.Ok) { progress(0); status(token.IsCancellationRequested ? "Download cancelled" : "No messages downloaded, failed to get the list of unique messages"); } else { progress(0); status("Downloading {0} out of {1}", unique.Count, mailbox.cnt); status("{0} - {1} messages: ", mailbox.name, unique.Count); int downloaded = 0; foreach (String n in unique) { int num = int.Parse(n); if (await cmd.Run(new FetchCommand(FetchCommand.Fetch.Body, async delegate (String message, String msgn) { progress((100.0 * (processed + int.Parse(msgn))) / messagesInAccount); await data(mailbox.name, message, getMsgId(message), msgn); downloaded++; }, num, num)) != ReturnCode.Ok) { status("failed to download {0}", n); continue; } } if (token.IsCancellationRequested) status("Download cancelled"); if (unique.Count != downloaded) status("{0} out of {1} failed to download", unique.Count-downloaded, unique.Count); } } } processed += mailbox.cnt; mailbox.start = mailbox.cnt; } status("Download complete"); } finally { connect.Close(); } }
async Task<List<Mailbox>> GetMailboxesList(ImapCmd cmd, StatusCb status, ProgressCb progress) { List<Mailbox> mailboxes = new List<Mailbox>(); if (await cmd.Run(new ListCommand(async delegate (String mailbox, String ctx) { await Task.Yield(); mailboxes.Add(new Mailbox(mailbox)); })) != ReturnCode.Ok) throw new FailedException(); mailboxes.Sort((Mailbox m1, Mailbox m2) => { bool g1 = GmailSpecial(m1.name); bool g2 = GmailSpecial(m2.name); if (g1 && g2) return String.Compare(m2.name, m1.name); // reverse else if (g1) return 1; else if (g2) return -1; else return String.Compare(m1.name, m2.name); }); for (int i = 0; i < mailboxes.Count; i++) { await cmd.Run(new StatusCommand(mailboxes[i].name, async delegate (String cnt, String ctx) { await Task.Yield(); status("{0} - {1}", mailboxes[i].name, cnt); mailboxes[i].cnt = int.Parse(cnt); messagesInAccount += int.Parse(cnt); })); } status("Total messages: {0}", messagesInAccount); Dictionary<String, int> downloaded = await CheckResume(status, progress); foreach (String mbox in downloaded.Keys) { mailboxes.ForEach((Mailbox m) => { if (m.name == mbox) m.start = downloaded[mbox] > 0 ? downloaded[mbox] : 1; }); } return mailboxes; }
public async static Task ParseToResume(CancellationToken token, String addr, String user, String file, StatusCb status, ProgressCb progress) { String index = Path.Combine(Path.GetDirectoryName(file), "email_proc1596.index"); FileInfo info = new FileInfo(file); long length = info.Length; FileStream stream = info.OpenRead(); status("Generating resume file..."); using (StreamReader reader = new StreamReader(stream)) { using (StreamWriter writer = new StreamWriter(index)) { await writer.WriteLineAsync(addr + " " + user + " " + file); String line = ""; String mailbox = ""; String messageid = ""; Regex re_mailbox = new Regex("^X-Mailbox: (.+)$"); Regex re_messageid = new Regex("^Message-ID: ([^ ]+)", RegexOptions.IgnoreCase); while ((line = await reader.ReadLineAsync()) != null) { if (token.IsCancellationRequested) return; progress(100.0 * stream.Position / length); Match m = re_mailbox.Match(line); if (m.Success) { if (mailbox != "") { if (messageid != "") messageid = " " + messageid; await writer.WriteLineAsync(mailbox + messageid); } mailbox = m.Groups[1].Value; messageid = ""; } else if (messageid == "") { m = re_messageid.Match(line); if (m.Success) messageid = m.Groups[1].Value; } } if (mailbox != "") { if (messageid != "") messageid = " " + messageid; await writer.WriteLineAsync(mailbox + messageid); } } } }
public async static Task ParseToResume(CancellationToken token, String addr, String user, String file, StatusCb status, ProgressCb progress) { String index = Path.Combine(Path.GetDirectoryName(file), "email_proc1596.index"); FileInfo info = new FileInfo(file); long length = info.Length; FileStream stream = info.OpenRead(); status("Generating resume file..."); using (StreamReader reader = new StreamReader(stream)) { using (StreamWriter writer = new StreamWriter(index)) { await writer.WriteLineAsync(addr + " " + user + " " + file); String line = ""; String mailbox = ""; String messageid = ""; Regex re_mailbox = new Regex("^X-Mailbox: (.+)$"); Regex re_messageid = new Regex("^Message-ID: ([^ ]+)", RegexOptions.IgnoreCase); while ((line = await reader.ReadLineAsync()) != null) { if (token.IsCancellationRequested) { return; } progress(100.0 * stream.Position / length); Match m = re_mailbox.Match(line); if (m.Success) { if (mailbox != "") { if (messageid != "") { messageid = " " + messageid; } await writer.WriteLineAsync(mailbox + messageid); } mailbox = m.Groups[1].Value; messageid = ""; } else if (messageid == "") { m = re_messageid.Match(line); if (m.Success) { messageid = m.Groups[1].Value; } } } if (mailbox != "") { if (messageid != "") { messageid = " " + messageid; } await writer.WriteLineAsync(mailbox + messageid); } } } }