private static void Main()
{
var tp = new ThreadPool(2);
Console.WriteLine("Enqueuing 10 items...");
for (var i = 0; i < 10; i++)
{
var t = new Task(i);
switch (i % 3)
{
case 0:
tp.Execute(t, Priority.High);
break;
case 1:
tp.Execute(t, Priority.Normal);
break;
case 2:
tp.Execute(t, Priority.Low);
break;
}
}
tp.Stop();
Console.WriteLine();
Console.WriteLine("Workers complete!");
Console.ReadLine();
}
public void Run()
{
//Stopwatch watch = new Stopwatch();
//watch.Start();
// 必须开启多线程限制
System.Net.ServicePointManager.DefaultConnectionLimit = int.MaxValue;
CheckRunningStat();
Logger.Info("Spider " + Identify + " InitComponent...");
InitComponent();
//IMonitorableScheduler monitor = (IMonitorableScheduler)Scheduler;
Logger.Info("Spider " + Identify + " Started!");
bool firstTask = false;
while (Stat.Value == StatRunning)
{
Request request = Scheduler.Poll(this);
if (request == null)
{
if (ThreadPool.GetThreadAlive() == 0 && ExitWhenComplete)
{
break;
}
if (_waitCount > _waitCountLimit)
{
break;
}
// wait until new url added
WaitNewUrl();
}
else
{
if (_startTime == DateTime.MinValue)
{
_startTime = DateTime.Now;
}
_waitCount = 0;
ThreadPool.Execute((obj, cts) =>
{
//Logger.Info(
// $"Left: {monitor.GetLeftRequestsCount(this)} Total: {monitor.GetTotalRequestsCount(this)} AliveThread: {ThreadPool.GetThreadAlive()} ThreadNum: {ThreadPool.GetThreadNum()}");
var request1 = obj as Request;
if (request1 != null)
{
try
{
ProcessRequest(request1, cts);
OnSuccess(request1);
Uri uri = new Uri(request1.Url);
Logger.Info($"Request: { HttpUtility.HtmlDecode(HttpUtility.UrlDecode(uri.Query))} Sucess.");
return(1);
}
catch (Exception e)
{
OnError(request1);
Logger.Error("Request " + request1.Url + " failed.", e);
return(-1);
}
finally
{
if (_site.GetHttpProxyPool().Enable)
{
_site.ReturnHttpProxyToPool((HttpHost)request1.GetExtra(Request.Proxy), (int)request1.GetExtra(Request.StatusCode));
}
_pageCount.Inc();
}
}
return(0);
}, request);
if (!firstTask)
{
Thread.Sleep(3000);
firstTask = true;
}
}
}
ThreadPool.WaitToEnd();
// release some resources
if (DestroyWhenExit)
{
Close();
}
_endTime = DateTime.Now;
OnClose();
//watch.Stop();
//Logger.Info("Cost time:" + (float)watch.ElapsedMilliseconds / 1000);
Stat.Set(StatFinished);
}