public void PerformSend(object stateInfo) { bool flag = true; foreach (KeyValuePair <SmtpItem, Tuple <DateTime, Queue <IScrapeResult> > > current in this.tsks) { if (current.Value.Value.Count > 0) { SmtpItem key = current.Key; SmtpClient client = this.GetClient(key); client.SendCompleted += new SendCompletedEventHandler(this.SendCompleted); this.pendingCount++; IScrapeResult scrapeResult = null; Queue <IScrapeResult> value; Monitor.Enter(value = current.Value.Value); try { scrapeResult = current.Value.Value.Dequeue(); } finally { Monitor.Exit(value); } string email = scrapeResult.GetEmail(); MailMessage mailMessage = this.GetMailMessage(key.EmailAddress, email); flag = false; try { if (this.ErrorCallback != null) { this.ErrorCallback.Log(string.Format("Sending message to {0} from {1}", email, key.EmailAddress)); } client.SendAsync(mailMessage, new KeyValuePair <IScrapeResult, SmtpItem>(scrapeResult, key)); } catch (Exception arg_EE_0) { Exception exception = arg_EE_0; if (this.ErrorCallback != null) { this.ErrorCallback.Log(exception.Message); } this.ProcessSendCompleted(new KeyValuePair <IScrapeResult, SmtpItem>(scrapeResult, key), false); } } } if (flag && this.pendingCount <= 0) { this.Stop(); } }
protected override bool CanAddRes(List <IScrapeResult> result, IScrapeResult res) { int num = 0; Monitor.Enter(result); try { num = result.Count((IScrapeResult it) => (it as MapScrapeResult).Address == (res as MapScrapeResult).Address); } finally { Monitor.Exit(result); } return(num <= 0 && base.CanAddRes(result, res)); }
public void ProcessSendCompleted(KeyValuePair <IScrapeResult, SmtpItem> kvp, bool success) { if (this.Callback != null) { this.Callback.EmailSent(kvp); } if (success) { this.recipients.Remove(kvp.Key); if (this.SearchCampaign != null) { IScrapeResult key = kvp.Key; if (key != null) { key.IsEmailSent = true; key.IsQueued = false; this.SearchCampaign.SentEmails.Add(new Campaign.SentEmail(kvp.Key.GetEmail(), DateTime.Now)); } } if (this.ErrorCallback != null) { this.ErrorCallback.Log(string.Format("Message sent: from {0} to {1}", kvp.Value.EmailAddress, kvp.Key)); } } else { int min = this.tsks.Min((KeyValuePair <SmtpItem, Tuple <DateTime, Queue <IScrapeResult> > > t) => t.Value.Value.Count); KeyValuePair <SmtpItem, Tuple <DateTime, Queue <IScrapeResult> > > keyValuePair = this.tsks.FirstOrDefault((KeyValuePair <SmtpItem, Tuple <DateTime, Queue <IScrapeResult> > > t) => t.Value.Value.Count == min); IScrapeResult key2 = kvp.Key; Queue <IScrapeResult> value; Monitor.Enter(value = keyValuePair.Value.Value); try { keyValuePair.Value.Value.Enqueue(key2); } finally { Monitor.Exit(value); } keyValuePair.Value.Key = keyValuePair.Value.Key.AddMilliseconds((double)(3600000 / this.perHour)); if (this.Callback != null) { this.Callback.AddTask(kvp, "Error occured - resend", keyValuePair.Value.Key); } } this.pendingCount--; }
protected void ThreadedWalk(object o) { string url = ((object[])o)[0] as string; IScrapeResult scrapeResult = ((object[])o)[1] as IScrapeResult; if (this.terminated) { return; } string text; if (this.downloader.DownloadUrl(url, out text)) { if (this.terminated) { return; } List <string> list = this.ExtractEmails(text); if (list.Count > 0) { IScrapeResult obj; Monitor.Enter(obj = scrapeResult); try { foreach (string current in list) { if (!scrapeResult.Emails.Contains(current)) { scrapeResult.Emails.Add(current); } } if (string.IsNullOrEmpty(scrapeResult.Email)) { scrapeResult.Email = scrapeResult.Emails.FirstOrDefault <string>(); } scrapeResult.Emails.Remove(scrapeResult.Email); } finally { Monitor.Exit(obj); } } } }
protected virtual bool CanAddRes(List <IScrapeResult> result, IScrapeResult res) { Monitor.Enter(result); bool result2; try { switch (CommonData.Instance.Settings.DuplicateProcess) { case ConsultingLeadsProSettings.Duplicates.Email: { result2 = (result.Count((IScrapeResult it) => (!string.IsNullOrEmpty(it.Email) && it.Email == res.Email) || it.Emails.Contains(res.Email) || res.Emails.Contains(it.Email) || it.Emails.Intersect(res.Emails).Count <string>() > 0) <= 0); break; } case ConsultingLeadsProSettings.Duplicates.Headline: { result2 = (result.Count((IScrapeResult it) => !string.IsNullOrEmpty(it.Headline) && it.Headline == res.Headline) <= 0); break; } case ConsultingLeadsProSettings.Duplicates.EmailHeadline: { result2 = (result.Count((IScrapeResult it) => !string.IsNullOrEmpty(it.Headline) && it.Headline == res.Headline && ((!string.IsNullOrEmpty(it.Email) && it.Email == res.Email) || it.Emails.Contains(res.Email) || res.Emails.Contains(it.Email) || it.Emails.Intersect(res.Emails).Count <string>() > 0)) <= 0); break; } default: { result2 = true; break; } } } finally { Monitor.Exit(result); } return(result2); }
public override void Process(IScrapeResult scr) { MapScrapeResult mapScrapeResult = scr as MapScrapeResult; if (mapScrapeResult == null) { return; } if (this.listView != null) { if (this.listView.InvokeRequired) { ScrapeResultCallback.ProcessDelegate method = new ScrapeResultCallback.ProcessDelegate(this.Process); try { this.listView.Invoke(method, new object[] { mapScrapeResult }); return; } catch { return; } } StringBuilder stringBuilder = new StringBuilder(); for (int i = 0; i < mapScrapeResult.Emails.Count; i++) { stringBuilder.Append(mapScrapeResult.Emails[i]); if (i < mapScrapeResult.Emails.Count - 1) { stringBuilder.Append(", "); } } ListViewItem listViewItem = new ListViewItem(new string[] { mapScrapeResult.Category, mapScrapeResult.Radius, mapScrapeResult.Headline, mapScrapeResult.Address, mapScrapeResult.City, mapScrapeResult.Region, mapScrapeResult.ZipCode, mapScrapeResult.Phone, mapScrapeResult.Email, stringBuilder.ToString(), mapScrapeResult.Website, mapScrapeResult.Latitude, mapScrapeResult.Longitude, mapScrapeResult.Map, mapScrapeResult.AdUrl }); listViewItem.Tag = mapScrapeResult; if (mapScrapeResult.IsEmailSent) { foreach (ListViewItem.ListViewSubItem listViewSubItem in listViewItem.SubItems) { listViewSubItem.BackColor = Color.GreenYellow; } } listViewItem.Checked = mapScrapeResult.IsSelected; Font font = new Font(listViewItem.SubItems[8].Font, listViewItem.SubItems[8].Font.Style | FontStyle.Underline); this.listView.Items.Add(listViewItem); listViewItem.UseItemStyleForSubItems = false; listViewItem.SubItems[10].Font = (listViewItem.SubItems[13].Font = (listViewItem.SubItems[14].Font = font)); listViewItem.SubItems[10].ForeColor = (listViewItem.SubItems[13].ForeColor = (listViewItem.SubItems[14].ForeColor = Color.FromArgb(0, 0, 0, 255))); this.listView.CausesValidation = !this.listView.CausesValidation; } }
public abstract void Process(IScrapeResult scr);
protected void WalkWebsite(string url, IScrapeResult res, out string resolvedUrl) { resolvedUrl = url; if (CommonData.Instance.Settings.DeepSearch == ConsultingLeadsProSettings.DeepSearchOptions.SearchNever) { return; } if (CommonData.Instance.Settings.DeepSearch == ConsultingLeadsProSettings.DeepSearchOptions.SearchIfNoEmails) { if (!string.IsNullOrEmpty(res.Email)) { return; } if (res.Emails.Count > 0) { return; } } if (this.terminated) { return; } string text; Uri uri; if (this.downloader.DownloadUrl(url, false, out text, out uri)) { resolvedUrl = uri.AbsoluteUri; if (this.terminated) { return; } foreach (string current in this.ExtractEmails(text)) { if (!res.Emails.Contains(current)) { res.Emails.Add(current); } } if (string.IsNullOrEmpty(res.Email)) { res.Email = res.Emails.FirstOrDefault <string>(); } res.Emails.Remove(res.Email); if (this.terminated) { return; } string text2 = string.Format("{0}://{1}", uri.Scheme, uri.Host); StringBuilder stringBuilder = new StringBuilder(text2); for (int i = 0; i < uri.Segments.Length; i++) { if (i != uri.Segments.Length - 1 || uri.Segments[i].EndsWith("/")) { stringBuilder.Append(uri.Segments[i]); } } string text3 = stringBuilder.ToString(); if (this.terminated) { return; } RegexOptions options = RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline; Regex regex = new Regex("<a.*?href=(?<char>\"|')(?<href>.*?)\\k<char>.*?</a>", options); MatchCollection matchCollection = regex.Matches(text); if (this.terminated) { return; } Queue <string> queue = new Queue <string>(); foreach (Match match in matchCollection) { if (this.terminated) { return; } string text4 = match.Groups["href"].Value.Trim().Replace("&", "&"); if (!text4.StartsWith("'") && !text4.StartsWith("#")) { if (text4.StartsWith("./")) { text4 = text4.Remove(0, 1); } if (text4.StartsWith("www.")) { text4 = "http://" + text4; } if (!text4.StartsWith("javascript:") && !text4.StartsWith("mailto:")) { if (!text4.StartsWith("http://") && !text4.StartsWith("https://")) { if (text4.StartsWith("/")) { text4 = text2 + text4; } else { if (text3.EndsWith("/")) { text4 = text3 + text4; } else { text4 = text3 + "/" + text4; } } } if (text4.StartsWith(text2)) { if (!queue.Contains(text4) && string.CompareOrdinal(text4, uri.AbsoluteUri) != 0) { queue.Enqueue(text4); } if (this.terminated) { return; } } } } } Queue <KeyValuePair <Thread, object> > queue2 = new Queue <KeyValuePair <Thread, object> >(); while (queue.Count > 0) { string text5 = queue.Dequeue(); object[] value = new object[] { text5, res }; queue2.Enqueue(new KeyValuePair <Thread, object>(new Thread(new ParameterizedThreadStart(this.ThreadedWalk)) { Name = text5, Priority = ThreadPriority.BelowNormal }, value)); } int num = this.maxThreadCount * 2; List <Thread> list = new List <Thread>(this.maxThreadCount); Thread.CurrentThread.Priority = ThreadPriority.Lowest; while (queue2.Count > 0 || list.Count > 0) { if (this.terminated) { return; } while (list.Count < num && queue2.Count > 0) { KeyValuePair <Thread, object> keyValuePair = queue2.Dequeue(); list.Add(keyValuePair.Key); keyValuePair.Key.Start(keyValuePair.Value); } if (this.terminated) { return; } int j = 0; while (j < list.Count) { if (list[j].ThreadState == ThreadState.Stopped) { list.RemoveAt(j); } else { j++; } } } } }
public override void Process(IScrapeResult scr) { SimpleScrapeResult simpleScrapeResult = scr as SimpleScrapeResult; if (simpleScrapeResult == null) { return; } if (this.listView != null) { if (this.listView.InvokeRequired) { ScrapeResultCallback.ProcessDelegate method = new ScrapeResultCallback.ProcessDelegate(this.Process); try { this.listView.Invoke(method, new object[] { simpleScrapeResult }); return; } catch { return; } } string text = string.Empty; for (int i = 0; i < simpleScrapeResult.Emails.Count; i++) { text += simpleScrapeResult.Emails[i]; if (i < simpleScrapeResult.Emails.Count - 1) { text += ", "; } } string text2 = string.Empty; for (int j = 0; j < simpleScrapeResult.PhonesInBody.Count; j++) { text2 += simpleScrapeResult.PhonesInBody[j]; if (j < simpleScrapeResult.PhonesInBody.Count - 1) { text2 += ", "; } } ListViewItem listViewItem = new ListViewItem(new string[] { simpleScrapeResult.Category, simpleScrapeResult.Location, simpleScrapeResult.Headline, UrlDownloader.SkipHtmlTags(simpleScrapeResult.Description), simpleScrapeResult.Email, text, text2, simpleScrapeResult.DatePosted, simpleScrapeResult.AdUrl }); listViewItem.Tag = simpleScrapeResult; if (simpleScrapeResult.IsEmailSent) { foreach (ListViewItem.ListViewSubItem listViewSubItem in listViewItem.SubItems) { listViewSubItem.BackColor = Color.GreenYellow; } } listViewItem.Checked = simpleScrapeResult.IsSelected; Font font = new Font(listViewItem.SubItems[8].Font, listViewItem.SubItems[8].Font.Style | FontStyle.Underline); this.listView.Items.Add(listViewItem); listViewItem.UseItemStyleForSubItems = false; listViewItem.SubItems[8].Font = font; listViewItem.SubItems[8].ForeColor = Color.FromArgb(0, 0, 0, 255); this.listView.CausesValidation = !this.listView.CausesValidation; } }