private SiteTopic ProcessTopic(SiteTopic topic)
{
topic.SectionId = this.PostSectionId;
topic.IconId = this.PostIconId;
if (this.TitleRegex)
{
foreach (string pattern in this.TitleReplacements)
{
try
{
topic.Title = Regex.Replace(topic.Title, pattern.Trim(), String.Empty);
}
catch { }
}
}
if (this.ContentRegex)
{
foreach (string pattern in this.ContentReplacements)
{
try
{
topic.Content = Regex.Replace(topic.Content, pattern.Trim(), String.Empty);
}
catch { }
}
}
switch (this.PersonalMessageLocation)
{
case MessageLocation.Bottom:
topic.Content += "\n" + this.PersonalMessage;
break;
case MessageLocation.Top:
topic.Content = this.PersonalMessage + "\n" + topic.Content;
break;
case MessageLocation.Nowhere:
default:
break;
}
return(topic);
}
public virtual string Format(SiteTopic topic, Dictionary<string, string> replacements = null)
{
string result = this.ContentTemplate;
if (replacements != null)
{
foreach (var replacement in replacements)
{
result = result.Replace(replacement.Key, replacement.Value);
}
}
result = String.Format(
result,
topic.Title, // {0}
topic.Content, // {1}
topic.SectionId, // {2}
topic.IconId // {3}
);
return result;
}
public virtual string Format(SiteTopic topic, Dictionary <string, string> replacements = null)
{
string result = this.ContentTemplate;
if (replacements != null)
{
foreach (var replacement in replacements)
{
result = result.Replace(replacement.Key, replacement.Value);
}
}
result = String.Format(
result,
topic.Title, // {0}
topic.Content, // {1}
topic.SectionId, // {2}
topic.IconId // {3}
);
return(result);
}
protected virtual void Worker()
{
var sw = new System.Diagnostics.Stopwatch();
int pageStep = 1;
int page = (this.StartPage < 1) ? 1 : this.StartPage;
int maxPages = (this.MaxPages < 1) ? int.MaxValue : this.MaxPages;
string[] urls = new string[0];
while (this.pagesLeeched < maxPages && page > 0)
{
try
{
var pageReq = this.Reader.GetPage(this.LeechSectionId, page, this.Reader.TopicsPerPage);
pageReq.CookieContainer.Add(this.Reader.Type.User.Cookies);
var pageRes = (!this.Reader.Type.AllowRedirects) ?
Http.Request(pageReq) : Http.HandleRedirects(Http.Request(pageReq), false);
if (pageRes.HasError)
{
this.OnError(this, new ErrorEventArgs(pageRes.Error));
page = this.PageMove(page, pageStep);
continue;
}
this.OnClientMessage("Reading page #" + page + ".");
urls = this.Reader.GetTopicUrls(pageRes.Data);
this.OnClientMessage(String.Format("Extracted {0} topics from page #{1}.", urls.Length, page));
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
for (int i = 0; i < urls.Length; i++)
{
try
{
sw.Reset();
sw.Start();
this.OnClientMessage(String.Format("Reading topic #{0} from page #{1}.", i + 1, page));
string hash = SiteTopic.GetUrlHash(urls[i]);
if (!this._hashes.Contains(hash))
{
this.AddHash(hash);
var t = this.Reader.GetTopic(urls[i]);
this.OnTopicRead(this, new TopicReadEventArgs(t));
if (t != null)
{
this.OnClientMessage(String.Format("Posting topic: {0}.", t.Title));
var postReq = this.PostSite.CreateTopic(this.ProcessTopic(t));
postReq.CookieContainer.Add(this.PostSite.User.Cookies);
var postRes = (!this.Reader.Type.AllowRedirects) ?
Http.FastRequest(postReq) : Http.HandleRedirects(Http.FastRequest(postReq), true);
if (postRes.HasError)
{
this.OnError(this, new ErrorEventArgs(postRes.Error));
}
}
}
else
{
this.OnClientMessage("Skipping duplicate topics...");
Thread.Sleep(300);
}
sw.Stop();
if (sw.ElapsedMilliseconds < this.Pause)
{
var p = this.Pause - (int)sw.ElapsedMilliseconds;
this.OnClientMessage(String.Format("Pausing for {0:N0} ms.", p));
Thread.Sleep(p);
}
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
}
this.StartPage = page = this.PageMove(page, pageStep);
this.pagesLeeched++;
}
this.IsRunning = false;
this.OnClientMessage("Finished.");
this.OnStopped(this, new EventArgs());
}
public abstract HttpWebRequest CreateTopic(SiteTopic topic);
public static string GetUrlHash(SiteTopic topic)
{
return GetUrlHash(topic.Url);
}
public TopicPostedEventArgs(SiteTopic topic, HttpResult result)
{
this.Topic = topic;
this.Result = result;
}
private SiteTopic ProcessTopic(SiteTopic topic)
{
topic.SectionId = this.PostSectionId;
topic.IconId = this.PostIconId;
if (this.TitleRegex)
{
foreach (string pattern in this.TitleReplacements)
{
try
{
topic.Title = Regex.Replace(topic.Title, pattern.Trim(), String.Empty);
}
catch { }
}
}
if (this.ContentRegex)
{
foreach (string pattern in this.ContentReplacements)
{
try
{
topic.Content = Regex.Replace(topic.Content, pattern.Trim(), String.Empty);
}
catch { }
}
}
switch (this.PersonalMessageLocation)
{
case MessageLocation.Bottom:
topic.Content += "\n" + this.PersonalMessage;
break;
case MessageLocation.Top:
topic.Content = this.PersonalMessage + "\n" + topic.Content;
break;
case MessageLocation.Nowhere:
default:
break;
}
return topic;
}
public static string GetUrlHash(SiteTopic topic)
{
return(GetUrlHash(topic.Url));
}
public override HttpWebRequest CreateTopic(SiteTopic topic)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
topic.Title = HttpUtility.UrlEncode(topic.Title, this.SiteEncoding);
topic.Content = HttpUtility.UrlEncode(topic.Content, this.SiteEncoding);
string url = this.BaseUrl + "/newthread.php?do=postthread&f=" + topic.SectionId;
string data = this.Details.Format(topic, this.TemplateReplacements);
byte[] rawData = this.SiteEncoding.GetBytes(data);
HttpWebRequest req = Http.Prepare(url);
Stream stream;
req.Method = "POST";
req.ContentType = Res.vBulletin_300_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
return req;
}
public TopicReadEventArgs(SiteTopic topic)
{
this.Topic = topic;
}
public override HttpWebRequest CreateTopic(SiteTopic topic)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
string url = this.BaseUrl + String.Format(this.NewTopicPath, topic.SectionId) + this.GetSessionId();
string data = this.Details.Format(topic, this.TemplateReplacements);
byte[] rawData = this.SiteEncoding.GetBytes(data);
HttpWebRequest req = Http.Prepare(url);
Stream stream;
req.Method = "POST";
req.ContentType = Res.phpBB_300_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
return req;
}
public override HttpWebRequest[] CreateTopics(SiteTopic[] topics)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
string url, data;
byte[] rawData;
HttpWebRequest req;
Stream stream;
var requests = new List<HttpWebRequest>();
foreach (SiteTopic topic in topics)
{
url = this.BaseUrl + String.Format(this.NewTopicPath, topic.SectionId) + this.GetSessionId();
data = this.Details.Format(topic, this.TemplateReplacements);
rawData = this.SiteEncoding.GetBytes(data);
req = Http.Prepare(url);
req.Method = "POST";
req.ContentType = Res.phpBB_300_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
requests.Add(req);
}
return requests.ToArray();
}
public override HttpWebRequest[] CreateTopics(SiteTopic[] topics)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
string url, data;
byte[] rawData;
HttpWebRequest req;
Stream stream;
var requests = new List<HttpWebRequest>();
foreach (SiteTopic topic in topics)
{
url = this.BaseUrl + "/index.php";
data = this.Details.Format(topic, this.TemplateReplacements).Replace("\r", "");
rawData = this.SiteEncoding.GetBytes(data);
req = Http.Prepare(url);
req.Method = "POST";
req.ContentType = Res.IPBoard_200_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
//req.AllowAutoRedirect = false;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
requests.Add(req);
}
return requests.ToArray();
}
public override HttpWebRequest CreateTopic(SiteTopic topic)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
topic.Title = HttpUtility.UrlEncode(topic.Title, this.SiteEncoding);
topic.Content = HttpUtility.UrlEncode(topic.Content, this.SiteEncoding);
string url = this.BaseUrl + "/posting.php";
string data = this.Details.Format(topic, this.TemplateReplacements);
byte[] rawData = this.SiteEncoding.GetBytes(data);
HttpWebRequest req = Http.Prepare(url);
Stream stream;
req.Method = "POST";
req.ContentType = Res.phpBB_200_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
// Empty read topics cookie
var cookies = from Cookie c in Http.SessionCookies
where c.Name.EndsWith("_t")
select c;
foreach (Cookie c in cookies) c.Value = String.Empty;
return req;
}
public TopicReadEventArgs(SiteTopic topic)
{
this.Topic = topic;
}
protected virtual void AsyncWorker()
{
//var sw = new System.Diagnostics.Stopwatch();
int pageStep = 1;
int page = (this.StartPage < 1) ? 1 : this.StartPage;
int maxPages = (this.MaxPages < 1) ? int.MaxValue : this.MaxPages;
string[] urls = new string[0];
while (this.pagesLeeched < maxPages && page > 0)
{
try
{
var pageReq = this.Reader.GetPage(this.LeechSectionId, page, this.Reader.TopicsPerPage);
pageReq.CookieContainer.Add(this.Reader.Type.User.Cookies);
var pageRes = (!this.Reader.Type.AllowRedirects) ?
Http.Request(pageReq) : Http.HandleRedirects(Http.Request(pageReq), false);
if (pageRes.HasError)
{
this.OnError(this, new ErrorEventArgs(pageRes.Error));
page = this.PageMove(page, pageStep);
continue;
}
this.OnClientMessage("Reading page #" + page + ".");
urls = this.Reader.GetTopicUrls(pageRes.Data);
this.OnClientMessage(String.Format("Extracted {0} topics from page #{1}.", urls.Length, page));
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
ParallelOptions po = new ParallelOptions()
{
MaxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit
};
var plr = Parallel.For(0, urls.Length, po, (i) =>
{
if (!this.IsRunning)
{
return;
}
try
{
this.OnClientMessage(String.Format("Reading topic #{0} from page #{1}.", i + 1, page));
string hash = SiteTopic.GetUrlHash(urls[i]);
if (!this._hashes.Contains(hash))
{
this.AddHash(hash);
var t = this.Reader.GetTopic(urls[i]);
this.OnTopicRead(this, new TopicReadEventArgs(t));
if (!this.IsRunning)
{
return;
}
if (t != null)
{
this.OnClientMessage(String.Format("Posting topic: {0}.", t.Title));
var postReq = this.PostSite.CreateTopic(this.ProcessTopic(t));
postReq.CookieContainer.Add(this.PostSite.User.Cookies);
var postRes = (!this.Reader.Type.AllowRedirects) ?
Http.FastRequest(postReq) : Http.HandleRedirects(Http.FastRequest(postReq), true);
if (postRes.HasError)
{
this.OnError(this, new ErrorEventArgs(postRes.Error));
}
}
}
else
{
this.OnClientMessage("Skipping duplicate topics...");
}
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
});
/*try
* {
* while (!plr.IsCompleted) Thread.Sleep(10);
* }
* catch (ThreadAbortException)
* {
* this.IsRunning = false;
* this.OnStopped(this, new EventArgs());
* return;
* }
* catch (Exception error)
* {
* this.OnError(this, new ErrorEventArgs(error));
* }*/
this.StartPage = page = this.PageMove(page, pageStep);
this.pagesLeeched++;
}
this.IsRunning = false;
this.OnClientMessage("Finished.");
this.OnStopped(this, new EventArgs());
}
public abstract HttpWebRequest[] CreateTopics(SiteTopic[] topics);
public override HttpWebRequest[] CreateTopics(SiteTopic[] topics)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
string url, data;
byte[] rawData;
HttpWebRequest req;
Stream stream;
var requests = new List<HttpWebRequest>();
foreach (SiteTopic topic in topics)
{
topic.Title = HttpUtility.UrlEncode(topic.Title, this.SiteEncoding);
topic.Content = HttpUtility.UrlEncode(topic.Content, this.SiteEncoding);
url = this.BaseUrl + "/newthread.php?do=postthread&f=" + topic.SectionId;
data = this.Details.Format(topic, this.TemplateReplacements);
rawData = this.SiteEncoding.GetBytes(data);
req = Http.Prepare(url);
req.Method = "POST";
req.ContentType = Res.vBulletin_400_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
requests.Add(req);
}
return requests.ToArray();
}
public TopicPostedEventArgs(SiteTopic topic, HttpResult result)
{
this.Topic = topic;
this.Result = result;
}
public override HttpWebRequest CreateTopic(SiteTopic topic)
{
if (!this.User.IsLoggedIn || !this.IsReady) return null;
string url = this.BaseUrl + "/index.php";
string data = this.Details.Format(topic, this.TemplateReplacements).Replace("\r", "");
byte[] rawData = this.SiteEncoding.GetBytes(data);
HttpWebRequest req = Http.Prepare(url);
Stream stream;
req.Method = "POST";
req.ContentType = Res.IPBoard_300_ContentType;
req.ContentLength = rawData.Length;
req.Referer = url;
//req.AllowAutoRedirect = false;
stream = req.GetRequestStream();
stream.Write(rawData, 0, rawData.Length);
stream.Close();
return req;
}
public abstract HttpWebRequest CreateTopic(SiteTopic topic);