/// <summary>
/// Concurrent methos, each spider will run inside a thread
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="classTypes"></param>
/// <returns></returns>
public IList <T> StartConcurrent <T>(List <string> classTypes)
{
List <T> data = new List <T>();
if (disposedValue || CancellationToken.IsCancellationRequested)
{
return(data);
}
IList <ISpider> spiders = new List <ISpider>();
foreach (string classType in classTypes)
{
Type?type = Type.GetType(classType);
if (type == null)
{
continue;
}
ISpider <T> spider = _spiderFactory.GetSpider <T>(type, CancellationToken);
//add the new spider
_spiders.Add(spider);
spiders.Add(spider);
Thread thread = new Thread(new ThreadStart(spider.Go));
_threads.Add(thread);
thread.Start();
}
foreach (Thread thread in _threads)
{
thread.Join();
}
foreach (ISpider <T> spider in spiders)
{
_spiders.Remove(spider);
}
foreach (ISpider <T> spider in spiders)
{
data.AddRange(spider.ExtractData);
spider.Dispose();
}
_logger.LogDebug("Spider finished.");
return(data);
}
