/// <summary>
/// Download urls synchronizing.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="urls"></param>
/// <returns></returns>
public IList <T> GetAll <T>(params string[] urls)
{
DestroyWhenExit = false;
SpawnUrl = false;
foreach (Request request in UrlUtils.ConvertToRequests(urls))
{
AddRequest(request);
}
ICollectorPipeline collectorPipeline = GetCollectorPipeline();
Pipelines.Add(collectorPipeline);
Run();
SpawnUrl = true;
DestroyWhenExit = true;
ICollection collection = collectorPipeline.GetCollected();
return((from object o in collection select(T) o).ToList());
}
/// <summary>
///
/// </summary>
/// <param name="urls"></param>
/// <returns></returns>
public List <T> GetAll <T>(IList <string> urls)
{
destroyWhenExit = false;
spawnUrl = false;
if (startRequests != null)
{
startRequests.Clear();
}
foreach (var request in UrlUtils.ConvertToRequests(urls))
{
AddRequest(request);
}
var collectorPipeline = GetCollectorPipeline <T>();
pipelines.Add(collectorPipeline);
Run();
spawnUrl = true;
destroyWhenExit = true;
return(collectorPipeline.GetCollector());
}
/// <summary>
/// Set startUrls of Spider.
/// Prior to startUrls of Site.
/// </summary>
/// <param name="startUrls"></param>
/// <returns></returns>
public Spider AddStartUrls(IList <string> startUrls)
{
CheckIfRunning();
Site.StartRequests.AddRange(UrlUtils.ConvertToRequests(startUrls, 1));
return(this);
}
/// <summary>
/// Set startUrls of Spider.
/// Prior to startUrls of Site.
/// </summary>
/// <param name="startUrls"></param>
/// <returns></returns>
public Spider StartUrls(IList <string> startUrls)
{
CheckIfRunning();
StartRequests = new HashSet <Request>(UrlUtils.ConvertToRequests(startUrls));
return(this);
}
/// <summary>
///
/// </summary>
/// <param name="startUrls"></param>
/// <returns></returns>
public Spider StartUrls(List <string> startUrls)
{
CheckIfRunning();
this.startRequests = UrlUtils.ConvertToRequests(startUrls);
return(this);
}
