private void DoProcesser()
{
while (!_isProcesserThreadStop)
{
if (_downloadResultQueue.IsEmpty)
{
Thread.Sleep(100);
continue;
}
DownloadResult downlaodResult = null;
_downloadResultQueue.TryDequeue(out downlaodResult);
if (null == downlaodResult)
{
continue;
}
try {
var urls = downlaodResult.FindAllUrls();
OnFindAllUrls?.Invoke(urls);
OnResultToPipeline?.Invoke(downlaodResult);
} catch (Exception e) {
_exceptionDownoadResult.Add(downlaodResult);
Console.WriteLine(new { DownloadResult = downlaodResult, Exception = e });
}
Thread.Sleep(0);
}
}
private async void _DoDownload(string url)
{
try {
var result = await _http.GetStringAsync(url);
var downloadResult = new DownloadResult {
CurrentUrl = url, Result = result
};
OnDownloadResult?.Invoke(downloadResult);
_downloadedResult.Enqueue(downloadResult);
if (_downloadedResult.Count >= 1000)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(x => SaveDownloadedResult(x)));
}
} catch (Exception e) {
_exceptionUrls.Enqueue(url);
Console.WriteLine(new { Url = url, Exception = e });
}
}
private void SaveDownloadedResult(object obj)
{
lock (_lock) {
var sb = new StringBuilder();
while (!_downloadedResult.IsEmpty)
{
DownloadResult downloadedReuslt = null;
_downloadedResult.TryDequeue(out downloadedReuslt);
if (null != downloadedReuslt)
{
sb.AppendLine(JsonConvert.SerializeObject(downloadedReuslt));
}
if (sb.Length >= 100000)
{
streamWriter.Write(sb.ToString());
streamWriter.Flush();
sb.Clear();
}
}
streamWriter.Write(sb.ToString());
streamWriter.Flush();
sb.Clear();
}
}
public IDownloadResultProcesser <DownloadResult, DownloadResult, List <string> > AddDownloadResult(DownloadResult downloadResult)
{
_downloadResultQueue.Enqueue(downloadResult);
return(this);
}
public override void Extract(DownloadResult previousResult)
{
var urls = previousResult.FindAllUrls();
_onResult(urls);
}