public bool Crawl(string absoluteUri, int depth, UriClassificationType restrictCrawlTo, UriClassificationType restrictDiscoveriesTo, double priority, RenderType renderType, RenderType renderTypeForChildren)
{
var discovery = new Discovery <ArachnodeDAO>(absoluteUri);
var crawlRequest = new CrawlRequest <ArachnodeDAO>(discovery, depth, restrictCrawlTo, restrictDiscoveriesTo, priority, renderType, renderTypeForChildren);
return(Crawler.Crawl(crawlRequest));
}
/// <summary>
/// A CrawlRequest.
/// </summary>
/// <param name = "discovery">The Discovery to be crawled.</param>
/// <param name = "depth">The depth to which this CrawlRequest should crawl.</param>
/// <param name = "restrictCrawlTo">if set to <c>true</c> [restrict crawl to].</param>
/// <param name = "restrictDiscoveriesTo">if set to <c>true</c> [restrict discoveries to].</param>
/// <param name = "priority">The Priority of CrawlRequest.</param>
public CrawlRequest(Discovery <TArachnodeDAO> discovery, int depth, UriClassificationType restrictCrawlTo, UriClassificationType restrictDiscoveriesTo, double priority, RenderType renderType, RenderType renderTypeForChildren)
{
if (depth <= 0)
{
throw new ArgumentException("Depth cannot be less than or equal to zero.", "depth");
}
//ANODET: This is a bug (Parent) - the user needs to be able to specify a file and a WebPage parent if the user is submitting an explicit Request for a File or an Image...
Created = DateTime.Now;
if (restrictCrawlTo >= UriClassificationType.OriginalDirectoryLevelUp || restrictDiscoveriesTo >= UriClassificationType.OriginalDirectoryLevelUp)
{
Originator = discovery;
}
Parent = discovery;
CurrentDepth = 1;
Discovery = discovery;
IsStorable = true;
InsertDiscovery = true;
MaximumDepth = depth;
RenderType = renderType;
RenderTypeForChildren = renderTypeForChildren;
RestrictCrawlTo = (short)restrictCrawlTo;
RestrictDiscoveriesTo = (short)restrictDiscoveriesTo;
Priority = priority;
//WasUsingDesriedMaximumMemoryInMegabytes = MemoryManager.IsUsingDesiredMaximumMemoryInMegabytes();
Discoveries = new Discoveries <TArachnodeDAO>();
}
private void ParseCrawlRequests()
{
try
{
lock (_crawlRequestsLock)
{
string[] crawlRequests = File.ReadAllLines(Path.Combine(Path.GetDirectoryName(Environment.CommandLine.Replace("\"", string.Empty)), "CrawlRequests.txt"));
foreach (string crawlRequest in crawlRequests)
{
if (crawlRequest.Trim().StartsWith("//") || string.IsNullOrWhiteSpace(crawlRequest))
{
continue;
}
string[] crawlRequestSplit = crawlRequest.Split(",".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
UriClassificationType restrictCrawlTo = UriClassificationType.None;
foreach (string uriClassificationType in crawlRequestSplit[2].Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
{
restrictCrawlTo |= (UriClassificationType)Enum.Parse(typeof(UriClassificationType), uriClassificationType);
}
UriClassificationType restrictDiscoveriesTo = UriClassificationType.None;
foreach (string uriClassificationType in crawlRequestSplit[3].Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries))
{
restrictDiscoveriesTo |= (UriClassificationType)Enum.Parse(typeof(UriClassificationType), uriClassificationType);
}
CrawlRequest <ArachnodeDAO> crawlRequest2 = new CrawlRequest <ArachnodeDAO>(new Discovery <ArachnodeDAO>(crawlRequestSplit[0]), int.Parse(crawlRequestSplit[1]), restrictCrawlTo, restrictDiscoveriesTo, double.Parse(crawlRequestSplit[4]), (RenderType)Enum.Parse(typeof(RenderType), crawlRequestSplit[5]), (RenderType)Enum.Parse(typeof(RenderType), crawlRequestSplit[6]));
_crawler.Crawl(crawlRequest2);
}
}
}
catch (Exception exception)
{
new ArachnodeDAO(_applicationSettings.ConnectionString, _applicationSettings, _webSettings, false, false).InsertException(null, null, exception, true);
}
}