protected virtual void AsyncWorker()
{
//var sw = new System.Diagnostics.Stopwatch();
int pageStep = 1;
int page = (this.StartPage < 1) ? 1 : this.StartPage;
int maxPages = (this.MaxPages < 1) ? int.MaxValue : this.MaxPages;
string[] urls = new string[0];
while (this.pagesLeeched < maxPages && page > 0)
{
try
{
var pageReq = this.Reader.GetPage(this.LeechSectionId, page, this.Reader.TopicsPerPage);
pageReq.CookieContainer.Add(this.Reader.Type.User.Cookies);
var pageRes = (!this.Reader.Type.AllowRedirects) ?
Http.Request(pageReq) : Http.HandleRedirects(Http.Request(pageReq), false);
if (pageRes.HasError)
{
this.OnError(this, new ErrorEventArgs(pageRes.Error));
page = this.PageMove(page, pageStep);
continue;
}
this.OnClientMessage("Reading page #" + page + ".");
urls = this.Reader.GetTopicUrls(pageRes.Data);
this.OnClientMessage(String.Format("Extracted {0} topics from page #{1}.", urls.Length, page));
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
ParallelOptions po = new ParallelOptions()
{
MaxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit
};
var plr = Parallel.For(0, urls.Length, po, (i) =>
{
if (!this.IsRunning)
{
return;
}
try
{
this.OnClientMessage(String.Format("Reading topic #{0} from page #{1}.", i + 1, page));
string hash = SiteTopic.GetUrlHash(urls[i]);
if (!this._hashes.Contains(hash))
{
this.AddHash(hash);
var t = this.Reader.GetTopic(urls[i]);
this.OnTopicRead(this, new TopicReadEventArgs(t));
if (!this.IsRunning)
{
return;
}
if (t != null)
{
this.OnClientMessage(String.Format("Posting topic: {0}.", t.Title));
var postReq = this.PostSite.CreateTopic(this.ProcessTopic(t));
postReq.CookieContainer.Add(this.PostSite.User.Cookies);
var postRes = (!this.Reader.Type.AllowRedirects) ?
Http.FastRequest(postReq) : Http.HandleRedirects(Http.FastRequest(postReq), true);
if (postRes.HasError)
{
this.OnError(this, new ErrorEventArgs(postRes.Error));
}
}
}
else
{
this.OnClientMessage("Skipping duplicate topics...");
}
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
});
/*try
* {
* while (!plr.IsCompleted) Thread.Sleep(10);
* }
* catch (ThreadAbortException)
* {
* this.IsRunning = false;
* this.OnStopped(this, new EventArgs());
* return;
* }
* catch (Exception error)
* {
* this.OnError(this, new ErrorEventArgs(error));
* }*/
this.StartPage = page = this.PageMove(page, pageStep);
this.pagesLeeched++;
}
this.IsRunning = false;
this.OnClientMessage("Finished.");
this.OnStopped(this, new EventArgs());
}
protected virtual void Worker()
{
var sw = new System.Diagnostics.Stopwatch();
int pageStep = 1;
int page = (this.StartPage < 1) ? 1 : this.StartPage;
int maxPages = (this.MaxPages < 1) ? int.MaxValue : this.MaxPages;
string[] urls = new string[0];
while (this.pagesLeeched < maxPages && page > 0)
{
try
{
var pageReq = this.Reader.GetPage(this.LeechSectionId, page, this.Reader.TopicsPerPage);
pageReq.CookieContainer.Add(this.Reader.Type.User.Cookies);
var pageRes = (!this.Reader.Type.AllowRedirects) ?
Http.Request(pageReq) : Http.HandleRedirects(Http.Request(pageReq), false);
if (pageRes.HasError)
{
this.OnError(this, new ErrorEventArgs(pageRes.Error));
page = this.PageMove(page, pageStep);
continue;
}
this.OnClientMessage("Reading page #" + page + ".");
urls = this.Reader.GetTopicUrls(pageRes.Data);
this.OnClientMessage(String.Format("Extracted {0} topics from page #{1}.", urls.Length, page));
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
for (int i = 0; i < urls.Length; i++)
{
try
{
sw.Reset();
sw.Start();
this.OnClientMessage(String.Format("Reading topic #{0} from page #{1}.", i + 1, page));
string hash = SiteTopic.GetUrlHash(urls[i]);
if (!this._hashes.Contains(hash))
{
this.AddHash(hash);
var t = this.Reader.GetTopic(urls[i]);
this.OnTopicRead(this, new TopicReadEventArgs(t));
if (t != null)
{
this.OnClientMessage(String.Format("Posting topic: {0}.", t.Title));
var postReq = this.PostSite.CreateTopic(this.ProcessTopic(t));
postReq.CookieContainer.Add(this.PostSite.User.Cookies);
var postRes = (!this.Reader.Type.AllowRedirects) ?
Http.FastRequest(postReq) : Http.HandleRedirects(Http.FastRequest(postReq), true);
if (postRes.HasError)
{
this.OnError(this, new ErrorEventArgs(postRes.Error));
}
}
}
else
{
this.OnClientMessage("Skipping duplicate topics...");
Thread.Sleep(300);
}
sw.Stop();
if (sw.ElapsedMilliseconds < this.Pause)
{
var p = this.Pause - (int)sw.ElapsedMilliseconds;
this.OnClientMessage(String.Format("Pausing for {0:N0} ms.", p));
Thread.Sleep(p);
}
}
catch (ThreadAbortException)
{
this.IsRunning = false;
this.OnStopped(this, new EventArgs());
return;
}
catch (Exception error)
{
this.OnError(this, new ErrorEventArgs(error));
}
}
this.StartPage = page = this.PageMove(page, pageStep);
this.pagesLeeched++;
}
this.IsRunning = false;
this.OnClientMessage("Finished.");
this.OnStopped(this, new EventArgs());
}
