public void Report(SpiderStatus status)
{
var tmp = new
{
Message = new
{
status.Error,
status.Left,
Status = status.Code,
status.Success,
Thread = status.ThreadNum,
status.Total
},
Name = status.Identity,
status.Machine,
status.UserId,
status.TaskGroup,
status.Timestamp
};
_postCounter.Inc();
var task = _client.PostAsync(_server, new StringContent(JsonConvert.SerializeObject(tmp), Encoding.UTF8, "application/json"));
task.ContinueWith(t =>
{
_postCounter.Dec();
});
}
public async Task StartAsync(CancellationToken cancellationToken)
{
if (Status == SpiderStatus.Running)
{
_logger.LogWarning("Spider is running, can not run again!");
return;
}
Status = SpiderStatus.Running;
await Task.Factory.StartNew(async() =>
{
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromMilliseconds(10));
}
_cancellationTokenSource.Cancel();
});
while (Status == SpiderStatus.Running)
{
var request = await Scheduler.PollAsync();
_ = Task.Factory.StartNew(async r =>
{
var oRequest = r as IRequestTask;
await FetchService.FetchAsync(oRequest, _cancellationTokenSource.Token);
}, request);
}
}
public void Stop()
{
if (stat == SpiderStatus.Running)
{
stat = SpiderStatus.Stop;
Debug.WriteLine("Spider " + GetGuid() + " stop success!");
}
}
private void Form1_Load(object sender, EventArgs e)
{
string host = ConfigurationManager.AppSettings["redishost"];
string pass = ConfigurationManager.AppSettings["redishostpass"];
if (!string.IsNullOrEmpty(host))
{
manager = new RedisSchedulerManager(host, pass);
}
Task.Factory.StartNew(() =>
{
while (true)
{
if ((!IsDisposed || IsHandleCreated || this.components != null))
{
Invoke(new Action(RefreshTask));
}
Thread.Sleep(60000);
}
});
Task.Factory.StartNew(() =>
{
while (true)
{
if ((!IsDisposed || IsHandleCreated || this.components != null))
{
Invoke(new Action(() =>
{
if (!string.IsNullOrEmpty(_selectedIdentify))
{
SpiderStatus spiderStatus = manager.GetTaskStatus(_selectedIdentify);
this.tbErrorPageCount.Text = spiderStatus.ErrorPageCount.ToString();
this.tbLeftRequestCount.Text = spiderStatus.LeftPageCount.ToString();
this.tbTotalRequestCount.Text = spiderStatus.TotalPageCount.ToString();
this.tbPagePerSecond.Text = spiderStatus.PagePerSecond.ToString(CultureInfo.InvariantCulture);
this.tbRunningProcessCount.Text = spiderStatus.AliveThreadCount.ToString();
this.tbProcessCount.Text = spiderStatus.ThreadCount.ToString();
this.tbStartTime.Text = spiderStatus.StartTime.ToString(CultureInfo.InvariantCulture);
this.tbEndTime.Text = spiderStatus.EndTime.ToString(CultureInfo.InvariantCulture);
this.tbTaskStatus.Text = spiderStatus.Status;
}
else
{
SentEmptyInfo();
}
}));
}
Thread.Sleep(1000);
}
});
}
public Task StopAsync()
{
if (Status == SpiderStatus.Stopped)
{
_logger.LogWarning("Spider stopped, can not stop again!");
return(Task.CompletedTask);
}
Status = SpiderStatus.Stopped;
_cancellationTokenSource.Cancel();
Thread.Sleep(2000);
return(Task.CompletedTask);
}
private void CheckRunningStat()
{
while (true)
{
var statNow = stat;
if (statNow == SpiderStatus.Running)
{
throw new Exception("Spider is already running!");
}
if (stat == statNow)
{
stat = SpiderStatus.Running;
break;
}
}
}
/// <summary>
///
/// </summary>
public void Run()
{
CheckRunningStat();
InitComponent();
Debug.WriteLine($"Spider {GetGuid()} started!");
while (!(stat == SpiderStatus.Running))
{
var request = scheduler.Poll(this);
if (request == null)
{
WaitNewUrl();
}
else
{
Task.Run(() =>
{
try
{
ProcessRequest(request);
OnSuccess(request);
}
catch (Exception e)
{
OnError(request);
Debug.WriteLine($"process request {request} error : {e.Message}");
}
finally
{
pageCount++;
SignalNewUrl();
}
});
}
}
stat = SpiderStatus.Stop;
if (destroyWhenExit == true)
{
Close();
}
Debug.WriteLine($"Spider {GetGuid()} closed! {pageCount} pages downloaded.");
}
/// <summary>
/// 运行
/// </summary>
public async Task Run(params string[] args)
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
_serviceList.ForEach(action =>
{
action(services);
});
services.AddSingleton(
new FreeSql.FreeSqlBuilder()
.UseConnectionString(FreeSql.DataType.MySql, _config.GetSection("mysql").Value)
.UseAutoSyncStructure(true).Build()
);
services.AddScoped <ITask, DbStorage>();
services.AddScoped <ICrawlerSelector, CrawlerSelector>();
serviceProvider = services.BuildServiceProvider();
status = SpiderStatus.Running;
Log.Information("爬虫开始启动");
this.SetSpeed();
await StartSpider();
await WaitForExisting();
}
public void Report(SpiderStatus spider)
{
throw new NotImplementedException();
}
private static void Spider_OnStatusChanged(Spider arg1, SpiderStatus arg2)
{
Console.WriteLine("status changed : " + arg2.ToString());
}
