protected virtual async Task CrawlSite()
{
while (!_crawlComplete)
{
RunPreWorkChecks();
var linksToScheduleCount = _scheduler.Count;
if (linksToScheduleCount > 0)
{
Log.Debug($"There are [{linksToScheduleCount}] links to schedule...");
_threadManager.DoWork(async() => await ProcessPage(_scheduler.GetNext()));
}
else if (!_threadManager.HasRunningThreads() && _processingPageCount < 1)//Ok that _processingPageCount could be a race condition, will be caught on the next loop iteration
{
Log.Debug("No links to schedule, no threads/tasks in progress...");
_crawlComplete = true;
}
else
{
Log.Debug("Waiting for links to be scheduled...");
//Beware of issues here... https://github.com/sjdirect/abot/issues/203
await Task.Delay(2500).ConfigureAwait(false);
}
}
}
protected virtual void Crawl()
{
while (!_crawlComplete)
{
RunPreWorkChecks();
if (_crawlPause)
{
_logger.LogWarning("爬行线程暂停中...");
Thread.Sleep(2500);
continue;
}
if (_scheduler.Count > 0)
{
_logger.LogInformation($"当前队列有[{_scheduler.Count}]个待爬链接");
_threadManager.DoWork(() => ProcessPage(_scheduler.GetNext()));
}
else if (!_threadManager.HasRunningThreads())
{
_crawlComplete = true;
}
else
{
_logger.LogDebug("Waiting for links to be scheduled...");
Thread.Sleep(2500);
}
}
}
public void HasRunningThreads()
{
//No threads should be running
Assert.IsFalse(_unitUnderTest.HasRunningThreads());
//Add word to be run on a thread
_unitUnderTest.DoWork(() => System.Threading.Thread.Sleep(300));
System.Threading.Thread.Sleep(20);
//Should have 1 running thread
Assert.IsTrue(_unitUnderTest.HasRunningThreads());
//Wait for the 1 running thread to finish
System.Threading.Thread.Sleep(400);
//Should have 0 threads running since the thread should have completed by now
Assert.IsFalse(_unitUnderTest.HasRunningThreads());
}
public void DoWork_SingleThreaded_WorkIsCompletedSynchronously()
{
_unitUnderTest = GetInstance(1);
int count = 0;
for (int i = 0; i < MAXTHREADS; i++)
{
_unitUnderTest.DoWork(() =>
{
System.Threading.Thread.Sleep(5);
Interlocked.Increment(ref count);
});
}
Assert.AreEqual(MAXTHREADS, count);
}
protected virtual void CrawlSite()
{
while (!_crawlComplete)
{
RunPreWorkChecks();
if (_scheduler.Count > 0)
{
_threadManager.DoWork(() => ProcessPage(_scheduler.GetNext()));
}
else if (!_threadManager.HasRunningThreads())
{
_crawlComplete = true;
}
else
{
_logger.DebugFormat("Waiting for links to be scheduled...");
Thread.Sleep(2500);
}
}
}
//private CrawlConfiguration GetCrawlConfigurationFromConfigFile()
//{
// AbotConfigurationSectionHandler configFromFile = AbotConfigurationSectionHandler.LoadFromXml();
// if (configFromFile == null)
// throw new InvalidOperationException("abot config section was NOT found");
// _logger.LogDebug($"abot config section was found");
// return configFromFile.Convert();
//}
protected virtual async Task CrawlSite()
{
while (!_crawlComplete)
{
RunPreWorkChecks();
if (_scheduler.Count > 0)
{
_threadManager.DoWork(() => ProcessPageAsync(_scheduler.GetNext()));
}
else if (!_threadManager.HasRunningThreads())
{
_crawlComplete = true;
}
else
{
_logger.LogDebug($"Waiting for links to be scheduled...");
await Task.Delay(2500);
}
}
}
async Task CrawlSite()
{
while (!_crawlComplete)
{
RunPreWorkChecks();
if (_scheduler.Count > 0)
{
var temp = _scheduler.GetNext();
await _threadManager.DoWork(async() =>
{
await ProcessPage(temp).ConfigureAwait(false);
}).ConfigureAwait(false);
}
else if (!_threadManager.HasRunningThreads())
{
_crawlComplete = true;
}
else
{
await Task.Delay(3500).ConfigureAwait(false);
}
}
}
public void DoWork_SingleThreaded_WorkIsCompletedSynchronously()
{
_unitUnderTest = GetInstance(1);
int count = 0;
for (int i = 0; i < MAXTHREADS; i++)
{
_unitUnderTest.DoWork(() =>
{
System.Threading.Thread.Sleep(5);
Interlocked.Increment(ref count);
});
}
Assert.AreEqual(MAXTHREADS, count);
}