public Crawler(IWebLoader loader, IFileSystemWrapper fileSystem, IHtmlParserFactory parserFactory, string uri, bool isNeedUploadOtherDomens = false, int parallelDownloadFactor = 10) { if (loader == null) { throw new ArgumentNullException("loader"); } if (fileSystem == null) { throw new ArgumentNullException("fileSystem"); } if (parserFactory == null) { throw new ArgumentNullException("parserFactory"); } if (!Uri.IsWellFormedUriString(uri, UriKind.Absolute)) { throw new ArgumentException("Не верный формат uri " + uri); } this.loader = new RestrictedWebLoader(loader, parallelDownloadFactor); this.fileSystem = fileSystem; this.parserFactory = parserFactory; this.isNeedUploadOtherDomens = isNeedUploadOtherDomens; this.baseUri = new Uri(uri); }
public void Initialize() { loader = new LoaderFromResource(); loaderMoq = new Mock <IWebLoader>(); loaderMoq.Setup(m => m.DownloadAsync(It.IsAny <string>(), It.IsAny <Stream>())).Returns <string, Stream>((uri, stream) => loader.DownloadAsync(uri, stream)); fileSystemMoq = new Mock <IFileSystemWrapper>(); fileSystemMoq.Setup(m => m.OpenStreamFor(It.IsAny <string>())).Returns(() => new MemoryStream()); }
internal static async Task <IDictionary <string, string> > CrawlAsync(string uri, int nestedLevel, bool isNeedUploadOtherDomens, IWebLoader loader, IFileSystemWrapper fileSystem, IHtmlParserFactory parserFactory, int parallelDownloadFactor = 10) { if (nestedLevel <= 0) { throw new ArgumentOutOfRangeException("Параметр nestedLevel должен быть больше нуля"); } var crawler = new Crawler(loader, fileSystem, parserFactory, uri, isNeedUploadOtherDomens, parallelDownloadFactor); await crawler.CrawlItAsync(uri, "index.html", nestedLevel); return(crawler.map); }
public ContentCacheLoader(IContentStorage storage, IWebLoader webLoader) { Storage = storage; WebLoader = webLoader; }
public RestrictedWebLoader(IWebLoader loader, int parallelRestrictNumber) { this.loader = loader; this.parallelRestrictNumber = parallelRestrictNumber; }
protected AParseResultBuilder() { _webLoader = new SimpleWebLoader(); }