public void Test_Analyze() { var httpClientMock = new Mock<IHttpClient>(); httpClientMock.Setup(it => it.DownloadString(It.IsAny<string>())) .Returns<string>((url) => { return "script"; }); var siteFileProviderMock = new Mock<ISiteFileProvider>(); Dictionary<string, string> styles = new Dictionary<string, string>(); siteFileProviderMock.Setup(it => it.AddFile(It.IsAny<Site>(), It.IsAny<string>(), It.IsAny<string>())) .Callback<Site, string, string>((site1, path, content) => { styles.Add(path, content); }); var scriptAnalyzer = new ScriptAnalyzer(httpClientMock.Object, siteFileProviderMock.Object); var pageHtml = @"<html> <head> <script src=""/script1.js"" type=""text/javascript""></script> <script src=""/script2.js"" type=""text/javascript""></script> </head> <body> </body> </html>"; SiteDownloadContext siteDownloadContext = new SiteDownloadContext(null, new DownloadOptions() { SiteName = "Test_Analyze", Url = "localhost", Pages = 20, Deep = 1 }, null, null); PageDownloadContext pageDownloadContext = new PageDownloadContext(siteDownloadContext, new PageLevel("http://localhost", 1), pageHtml); scriptAnalyzer.Analyze(pageDownloadContext); Assert.AreEqual(2, styles.Count()); Assert.AreEqual("script", styles["\\Scripts\\script1.js"]);
public void Download(PageLevel pageLevel, SiteDownloadContext siteDownloadContext) { var absolutePath = new Uri(pageLevel.Url).AbsolutePath; var pageName = absolutePath.Replace("/", "-"); var page = new Page(siteDownloadContext.Site, pageName) { Routes = new[] { new PageRoute() { Identifier = absolutePath } }, IsDefault = pageLevel.Level == 0 }; if (_pageProvider.Get(page) == null) { var text = _httpClient.DownloadString(pageLevel.Url); if (!string.IsNullOrEmpty(text)) { var pageDownloadContext = new PageDownloadContext(siteDownloadContext, pageLevel, text); foreach (var analyzer in _analyzers) { analyzer.Analyze(pageDownloadContext); } page.Html = pageDownloadContext.HtmlDocument.DocumentNode.InnerHtml; _pageProvider.Add(page); siteDownloadContext.DownloadedPages.Add(pageLevel); } } }
public PageDownloadContext(SiteDownloadContext siteDownloadContext, PageLevel pageLevel, string pageHtml) { this.SiteDownloadContext = siteDownloadContext; this.PageLevel = pageLevel; this.PageHtml = pageHtml; HtmlDocument = new HtmlDocument(); HtmlDocument.LoadHtml(pageHtml); }
public IEnumerable <PageLevel> Download() { var site = CreateSite(_options.SiteName); var siteDownloadContext = new SiteDownloadContext(site, _options, _downloadedPages, _downloadQueue); siteDownloadContext.DownloadQueue.Enqueue(new PageLevel(_options.Url, 0)); while (siteDownloadContext.DownloadQueue.Count > 0) { var page = siteDownloadContext.DownloadQueue.Dequeue(); _pageDownloader.Download(page, siteDownloadContext); OnPageDownloaded(new PageDownloadedEventArgs(page)); } IsCompleted = true; OnDownLoadCompleted(new DownloadCompletedEventArgs(DownloadedPages)); return(siteDownloadContext.DownloadedPages); }
public IEnumerable<PageLevel> Download() { var site = CreateSite(_options.SiteName); var siteDownloadContext = new SiteDownloadContext(site, _options, _downloadedPages, _downloadQueue); siteDownloadContext.DownloadQueue.Enqueue(new PageLevel(_options.Url, 0)); while (siteDownloadContext.DownloadQueue.Count > 0) { var page = siteDownloadContext.DownloadQueue.Dequeue(); _pageDownloader.Download(page, siteDownloadContext); OnPageDownloaded(new PageDownloadedEventArgs(page)); } IsCompleted = true; OnDownLoadCompleted(new DownloadCompletedEventArgs(DownloadedPages)); return siteDownloadContext.DownloadedPages; }