private SiteTopic ProcessTopic(SiteTopic topic) { topic.SectionId = this.PostSectionId; topic.IconId = this.PostIconId; if (this.TitleRegex) { foreach (string pattern in this.TitleReplacements) { try { topic.Title = Regex.Replace(topic.Title, pattern.Trim(), String.Empty); } catch { } } } if (this.ContentRegex) { foreach (string pattern in this.ContentReplacements) { try { topic.Content = Regex.Replace(topic.Content, pattern.Trim(), String.Empty); } catch { } } } switch (this.PersonalMessageLocation) { case MessageLocation.Bottom: topic.Content += "\n" + this.PersonalMessage; break; case MessageLocation.Top: topic.Content = this.PersonalMessage + "\n" + topic.Content; break; case MessageLocation.Nowhere: default: break; } return(topic); }
public virtual string Format(SiteTopic topic, Dictionary<string, string> replacements = null) { string result = this.ContentTemplate; if (replacements != null) { foreach (var replacement in replacements) { result = result.Replace(replacement.Key, replacement.Value); } } result = String.Format( result, topic.Title, // {0} topic.Content, // {1} topic.SectionId, // {2} topic.IconId // {3} ); return result; }
public virtual string Format(SiteTopic topic, Dictionary <string, string> replacements = null) { string result = this.ContentTemplate; if (replacements != null) { foreach (var replacement in replacements) { result = result.Replace(replacement.Key, replacement.Value); } } result = String.Format( result, topic.Title, // {0} topic.Content, // {1} topic.SectionId, // {2} topic.IconId // {3} ); return(result); }
protected virtual void Worker() { var sw = new System.Diagnostics.Stopwatch(); int pageStep = 1; int page = (this.StartPage < 1) ? 1 : this.StartPage; int maxPages = (this.MaxPages < 1) ? int.MaxValue : this.MaxPages; string[] urls = new string[0]; while (this.pagesLeeched < maxPages && page > 0) { try { var pageReq = this.Reader.GetPage(this.LeechSectionId, page, this.Reader.TopicsPerPage); pageReq.CookieContainer.Add(this.Reader.Type.User.Cookies); var pageRes = (!this.Reader.Type.AllowRedirects) ? Http.Request(pageReq) : Http.HandleRedirects(Http.Request(pageReq), false); if (pageRes.HasError) { this.OnError(this, new ErrorEventArgs(pageRes.Error)); page = this.PageMove(page, pageStep); continue; } this.OnClientMessage("Reading page #" + page + "."); urls = this.Reader.GetTopicUrls(pageRes.Data); this.OnClientMessage(String.Format("Extracted {0} topics from page #{1}.", urls.Length, page)); } catch (ThreadAbortException) { this.IsRunning = false; this.OnStopped(this, new EventArgs()); return; } catch (Exception error) { this.OnError(this, new ErrorEventArgs(error)); } for (int i = 0; i < urls.Length; i++) { try { sw.Reset(); sw.Start(); this.OnClientMessage(String.Format("Reading topic #{0} from page #{1}.", i + 1, page)); string hash = SiteTopic.GetUrlHash(urls[i]); if (!this._hashes.Contains(hash)) { this.AddHash(hash); var t = this.Reader.GetTopic(urls[i]); this.OnTopicRead(this, new TopicReadEventArgs(t)); if (t != null) { this.OnClientMessage(String.Format("Posting topic: {0}.", t.Title)); var postReq = this.PostSite.CreateTopic(this.ProcessTopic(t)); postReq.CookieContainer.Add(this.PostSite.User.Cookies); var postRes = (!this.Reader.Type.AllowRedirects) ? Http.FastRequest(postReq) : Http.HandleRedirects(Http.FastRequest(postReq), true); if (postRes.HasError) { this.OnError(this, new ErrorEventArgs(postRes.Error)); } } } else { this.OnClientMessage("Skipping duplicate topics..."); Thread.Sleep(300); } sw.Stop(); if (sw.ElapsedMilliseconds < this.Pause) { var p = this.Pause - (int)sw.ElapsedMilliseconds; this.OnClientMessage(String.Format("Pausing for {0:N0} ms.", p)); Thread.Sleep(p); } } catch (ThreadAbortException) { this.IsRunning = false; this.OnStopped(this, new EventArgs()); return; } catch (Exception error) { this.OnError(this, new ErrorEventArgs(error)); } } this.StartPage = page = this.PageMove(page, pageStep); this.pagesLeeched++; } this.IsRunning = false; this.OnClientMessage("Finished."); this.OnStopped(this, new EventArgs()); }
public abstract HttpWebRequest CreateTopic(SiteTopic topic);
public static string GetUrlHash(SiteTopic topic) { return GetUrlHash(topic.Url); }
public TopicPostedEventArgs(SiteTopic topic, HttpResult result) { this.Topic = topic; this.Result = result; }
private SiteTopic ProcessTopic(SiteTopic topic) { topic.SectionId = this.PostSectionId; topic.IconId = this.PostIconId; if (this.TitleRegex) { foreach (string pattern in this.TitleReplacements) { try { topic.Title = Regex.Replace(topic.Title, pattern.Trim(), String.Empty); } catch { } } } if (this.ContentRegex) { foreach (string pattern in this.ContentReplacements) { try { topic.Content = Regex.Replace(topic.Content, pattern.Trim(), String.Empty); } catch { } } } switch (this.PersonalMessageLocation) { case MessageLocation.Bottom: topic.Content += "\n" + this.PersonalMessage; break; case MessageLocation.Top: topic.Content = this.PersonalMessage + "\n" + topic.Content; break; case MessageLocation.Nowhere: default: break; } return topic; }
public static string GetUrlHash(SiteTopic topic) { return(GetUrlHash(topic.Url)); }
public override HttpWebRequest CreateTopic(SiteTopic topic) { if (!this.User.IsLoggedIn || !this.IsReady) return null; topic.Title = HttpUtility.UrlEncode(topic.Title, this.SiteEncoding); topic.Content = HttpUtility.UrlEncode(topic.Content, this.SiteEncoding); string url = this.BaseUrl + "/newthread.php?do=postthread&f=" + topic.SectionId; string data = this.Details.Format(topic, this.TemplateReplacements); byte[] rawData = this.SiteEncoding.GetBytes(data); HttpWebRequest req = Http.Prepare(url); Stream stream; req.Method = "POST"; req.ContentType = Res.vBulletin_300_ContentType; req.ContentLength = rawData.Length; req.Referer = url; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); return req; }
public TopicReadEventArgs(SiteTopic topic) { this.Topic = topic; }
public override HttpWebRequest CreateTopic(SiteTopic topic) { if (!this.User.IsLoggedIn || !this.IsReady) return null; string url = this.BaseUrl + String.Format(this.NewTopicPath, topic.SectionId) + this.GetSessionId(); string data = this.Details.Format(topic, this.TemplateReplacements); byte[] rawData = this.SiteEncoding.GetBytes(data); HttpWebRequest req = Http.Prepare(url); Stream stream; req.Method = "POST"; req.ContentType = Res.phpBB_300_ContentType; req.ContentLength = rawData.Length; req.Referer = url; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); return req; }
public override HttpWebRequest[] CreateTopics(SiteTopic[] topics) { if (!this.User.IsLoggedIn || !this.IsReady) return null; string url, data; byte[] rawData; HttpWebRequest req; Stream stream; var requests = new List<HttpWebRequest>(); foreach (SiteTopic topic in topics) { url = this.BaseUrl + String.Format(this.NewTopicPath, topic.SectionId) + this.GetSessionId(); data = this.Details.Format(topic, this.TemplateReplacements); rawData = this.SiteEncoding.GetBytes(data); req = Http.Prepare(url); req.Method = "POST"; req.ContentType = Res.phpBB_300_ContentType; req.ContentLength = rawData.Length; req.Referer = url; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); requests.Add(req); } return requests.ToArray(); }
public override HttpWebRequest[] CreateTopics(SiteTopic[] topics) { if (!this.User.IsLoggedIn || !this.IsReady) return null; string url, data; byte[] rawData; HttpWebRequest req; Stream stream; var requests = new List<HttpWebRequest>(); foreach (SiteTopic topic in topics) { url = this.BaseUrl + "/index.php"; data = this.Details.Format(topic, this.TemplateReplacements).Replace("\r", ""); rawData = this.SiteEncoding.GetBytes(data); req = Http.Prepare(url); req.Method = "POST"; req.ContentType = Res.IPBoard_200_ContentType; req.ContentLength = rawData.Length; req.Referer = url; //req.AllowAutoRedirect = false; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); requests.Add(req); } return requests.ToArray(); }
public override HttpWebRequest CreateTopic(SiteTopic topic) { if (!this.User.IsLoggedIn || !this.IsReady) return null; topic.Title = HttpUtility.UrlEncode(topic.Title, this.SiteEncoding); topic.Content = HttpUtility.UrlEncode(topic.Content, this.SiteEncoding); string url = this.BaseUrl + "/posting.php"; string data = this.Details.Format(topic, this.TemplateReplacements); byte[] rawData = this.SiteEncoding.GetBytes(data); HttpWebRequest req = Http.Prepare(url); Stream stream; req.Method = "POST"; req.ContentType = Res.phpBB_200_ContentType; req.ContentLength = rawData.Length; req.Referer = url; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); // Empty read topics cookie var cookies = from Cookie c in Http.SessionCookies where c.Name.EndsWith("_t") select c; foreach (Cookie c in cookies) c.Value = String.Empty; return req; }
protected virtual void AsyncWorker() { //var sw = new System.Diagnostics.Stopwatch(); int pageStep = 1; int page = (this.StartPage < 1) ? 1 : this.StartPage; int maxPages = (this.MaxPages < 1) ? int.MaxValue : this.MaxPages; string[] urls = new string[0]; while (this.pagesLeeched < maxPages && page > 0) { try { var pageReq = this.Reader.GetPage(this.LeechSectionId, page, this.Reader.TopicsPerPage); pageReq.CookieContainer.Add(this.Reader.Type.User.Cookies); var pageRes = (!this.Reader.Type.AllowRedirects) ? Http.Request(pageReq) : Http.HandleRedirects(Http.Request(pageReq), false); if (pageRes.HasError) { this.OnError(this, new ErrorEventArgs(pageRes.Error)); page = this.PageMove(page, pageStep); continue; } this.OnClientMessage("Reading page #" + page + "."); urls = this.Reader.GetTopicUrls(pageRes.Data); this.OnClientMessage(String.Format("Extracted {0} topics from page #{1}.", urls.Length, page)); } catch (ThreadAbortException) { this.IsRunning = false; this.OnStopped(this, new EventArgs()); return; } catch (Exception error) { this.OnError(this, new ErrorEventArgs(error)); } ParallelOptions po = new ParallelOptions() { MaxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit }; var plr = Parallel.For(0, urls.Length, po, (i) => { if (!this.IsRunning) { return; } try { this.OnClientMessage(String.Format("Reading topic #{0} from page #{1}.", i + 1, page)); string hash = SiteTopic.GetUrlHash(urls[i]); if (!this._hashes.Contains(hash)) { this.AddHash(hash); var t = this.Reader.GetTopic(urls[i]); this.OnTopicRead(this, new TopicReadEventArgs(t)); if (!this.IsRunning) { return; } if (t != null) { this.OnClientMessage(String.Format("Posting topic: {0}.", t.Title)); var postReq = this.PostSite.CreateTopic(this.ProcessTopic(t)); postReq.CookieContainer.Add(this.PostSite.User.Cookies); var postRes = (!this.Reader.Type.AllowRedirects) ? Http.FastRequest(postReq) : Http.HandleRedirects(Http.FastRequest(postReq), true); if (postRes.HasError) { this.OnError(this, new ErrorEventArgs(postRes.Error)); } } } else { this.OnClientMessage("Skipping duplicate topics..."); } } catch (ThreadAbortException) { this.IsRunning = false; this.OnStopped(this, new EventArgs()); return; } catch (Exception error) { this.OnError(this, new ErrorEventArgs(error)); } }); /*try * { * while (!plr.IsCompleted) Thread.Sleep(10); * } * catch (ThreadAbortException) * { * this.IsRunning = false; * this.OnStopped(this, new EventArgs()); * return; * } * catch (Exception error) * { * this.OnError(this, new ErrorEventArgs(error)); * }*/ this.StartPage = page = this.PageMove(page, pageStep); this.pagesLeeched++; } this.IsRunning = false; this.OnClientMessage("Finished."); this.OnStopped(this, new EventArgs()); }
public abstract HttpWebRequest[] CreateTopics(SiteTopic[] topics);
public override HttpWebRequest[] CreateTopics(SiteTopic[] topics) { if (!this.User.IsLoggedIn || !this.IsReady) return null; string url, data; byte[] rawData; HttpWebRequest req; Stream stream; var requests = new List<HttpWebRequest>(); foreach (SiteTopic topic in topics) { topic.Title = HttpUtility.UrlEncode(topic.Title, this.SiteEncoding); topic.Content = HttpUtility.UrlEncode(topic.Content, this.SiteEncoding); url = this.BaseUrl + "/newthread.php?do=postthread&f=" + topic.SectionId; data = this.Details.Format(topic, this.TemplateReplacements); rawData = this.SiteEncoding.GetBytes(data); req = Http.Prepare(url); req.Method = "POST"; req.ContentType = Res.vBulletin_400_ContentType; req.ContentLength = rawData.Length; req.Referer = url; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); requests.Add(req); } return requests.ToArray(); }
public override HttpWebRequest CreateTopic(SiteTopic topic) { if (!this.User.IsLoggedIn || !this.IsReady) return null; string url = this.BaseUrl + "/index.php"; string data = this.Details.Format(topic, this.TemplateReplacements).Replace("\r", ""); byte[] rawData = this.SiteEncoding.GetBytes(data); HttpWebRequest req = Http.Prepare(url); Stream stream; req.Method = "POST"; req.ContentType = Res.IPBoard_300_ContentType; req.ContentLength = rawData.Length; req.Referer = url; //req.AllowAutoRedirect = false; stream = req.GetRequestStream(); stream.Write(rawData, 0, rawData.Length); stream.Close(); return req; }