public bool Crawl(string absoluteUri, int depth, UriClassificationType restrictCrawlTo, UriClassificationType restrictDiscoveriesTo, double priority, RenderType renderType, RenderType renderTypeForChildren) { var discovery = new Discovery <ArachnodeDAO>(absoluteUri); var crawlRequest = new CrawlRequest <ArachnodeDAO>(discovery, depth, restrictCrawlTo, restrictDiscoveriesTo, priority, renderType, renderTypeForChildren); return(Crawler.Crawl(crawlRequest)); }
/// <summary> /// A CrawlRequest. /// </summary> /// <param name = "discovery">The Discovery to be crawled.</param> /// <param name = "depth">The depth to which this CrawlRequest should crawl.</param> /// <param name = "restrictCrawlTo">if set to <c>true</c> [restrict crawl to].</param> /// <param name = "restrictDiscoveriesTo">if set to <c>true</c> [restrict discoveries to].</param> /// <param name = "priority">The Priority of CrawlRequest.</param> public CrawlRequest(Discovery <TArachnodeDAO> discovery, int depth, UriClassificationType restrictCrawlTo, UriClassificationType restrictDiscoveriesTo, double priority, RenderType renderType, RenderType renderTypeForChildren) { if (depth <= 0) { throw new ArgumentException("Depth cannot be less than or equal to zero.", "depth"); } //ANODET: This is a bug (Parent) - the user needs to be able to specify a file and a WebPage parent if the user is submitting an explicit Request for a File or an Image... Created = DateTime.Now; if (restrictCrawlTo >= UriClassificationType.OriginalDirectoryLevelUp || restrictDiscoveriesTo >= UriClassificationType.OriginalDirectoryLevelUp) { Originator = discovery; } Parent = discovery; CurrentDepth = 1; Discovery = discovery; IsStorable = true; InsertDiscovery = true; MaximumDepth = depth; RenderType = renderType; RenderTypeForChildren = renderTypeForChildren; RestrictCrawlTo = (short)restrictCrawlTo; RestrictDiscoveriesTo = (short)restrictDiscoveriesTo; Priority = priority; //WasUsingDesriedMaximumMemoryInMegabytes = MemoryManager.IsUsingDesiredMaximumMemoryInMegabytes(); Discoveries = new Discoveries <TArachnodeDAO>(); }
private void ParseCrawlRequests() { try { lock (_crawlRequestsLock) { string[] crawlRequests = File.ReadAllLines(Path.Combine(Path.GetDirectoryName(Environment.CommandLine.Replace("\"", string.Empty)), "CrawlRequests.txt")); foreach (string crawlRequest in crawlRequests) { if (crawlRequest.Trim().StartsWith("//") || string.IsNullOrWhiteSpace(crawlRequest)) { continue; } string[] crawlRequestSplit = crawlRequest.Split(",".ToCharArray(), StringSplitOptions.RemoveEmptyEntries); UriClassificationType restrictCrawlTo = UriClassificationType.None; foreach (string uriClassificationType in crawlRequestSplit[2].Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)) { restrictCrawlTo |= (UriClassificationType)Enum.Parse(typeof(UriClassificationType), uriClassificationType); } UriClassificationType restrictDiscoveriesTo = UriClassificationType.None; foreach (string uriClassificationType in crawlRequestSplit[3].Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)) { restrictDiscoveriesTo |= (UriClassificationType)Enum.Parse(typeof(UriClassificationType), uriClassificationType); } CrawlRequest <ArachnodeDAO> crawlRequest2 = new CrawlRequest <ArachnodeDAO>(new Discovery <ArachnodeDAO>(crawlRequestSplit[0]), int.Parse(crawlRequestSplit[1]), restrictCrawlTo, restrictDiscoveriesTo, double.Parse(crawlRequestSplit[4]), (RenderType)Enum.Parse(typeof(RenderType), crawlRequestSplit[5]), (RenderType)Enum.Parse(typeof(RenderType), crawlRequestSplit[6])); _crawler.Crawl(crawlRequest2); } } } catch (Exception exception) { new ArachnodeDAO(_applicationSettings.ConnectionString, _applicationSettings, _webSettings, false, false).InsertException(null, null, exception, true); } }