public void Test_Analyze()
{
var httpClientMock = new Mock<IHttpClient>();
httpClientMock.Setup(it => it.DownloadString(It.IsAny<string>()))
.Returns<string>((url) =>
{
return "script";
});
var siteFileProviderMock = new Mock<ISiteFileProvider>();
Dictionary<string, string> styles = new Dictionary<string, string>();
siteFileProviderMock.Setup(it => it.AddFile(It.IsAny<Site>(), It.IsAny<string>(), It.IsAny<string>()))
.Callback<Site, string, string>((site1, path, content) =>
{
styles.Add(path, content);
});
var scriptAnalyzer = new ScriptAnalyzer(httpClientMock.Object, siteFileProviderMock.Object);
var pageHtml = @"<html>
<head>
<script src=""/script1.js"" type=""text/javascript""></script>
<script src=""/script2.js"" type=""text/javascript""></script>
</head>
<body>
</body>
</html>";
SiteDownloadContext siteDownloadContext = new SiteDownloadContext(null, new DownloadOptions() { SiteName = "Test_Analyze", Url = "localhost", Pages = 20, Deep = 1 }, null, null);
PageDownloadContext pageDownloadContext = new PageDownloadContext(siteDownloadContext, new PageLevel("http://localhost", 1), pageHtml);
scriptAnalyzer.Analyze(pageDownloadContext);
Assert.AreEqual(2, styles.Count());
Assert.AreEqual("script", styles["\\Scripts\\script1.js"]);
public void Download(PageLevel pageLevel, SiteDownloadContext siteDownloadContext)
{
var absolutePath = new Uri(pageLevel.Url).AbsolutePath;
var pageName = absolutePath.Replace("/", "-");
var page = new Page(siteDownloadContext.Site, pageName)
{
Routes = new[] { new PageRoute()
{
Identifier = absolutePath
} },
IsDefault = pageLevel.Level == 0
};
if (_pageProvider.Get(page) == null)
{
var text = _httpClient.DownloadString(pageLevel.Url);
if (!string.IsNullOrEmpty(text))
{
var pageDownloadContext = new PageDownloadContext(siteDownloadContext, pageLevel, text);
foreach (var analyzer in _analyzers)
{
analyzer.Analyze(pageDownloadContext);
}
page.Html = pageDownloadContext.HtmlDocument.DocumentNode.InnerHtml;
_pageProvider.Add(page);
siteDownloadContext.DownloadedPages.Add(pageLevel);
}
}
}
public PageDownloadContext(SiteDownloadContext siteDownloadContext, PageLevel pageLevel, string pageHtml)
{
this.SiteDownloadContext = siteDownloadContext;
this.PageLevel = pageLevel;
this.PageHtml = pageHtml;
HtmlDocument = new HtmlDocument();
HtmlDocument.LoadHtml(pageHtml);
}
public IEnumerable <PageLevel> Download()
{
var site = CreateSite(_options.SiteName);
var siteDownloadContext = new SiteDownloadContext(site, _options, _downloadedPages, _downloadQueue);
siteDownloadContext.DownloadQueue.Enqueue(new PageLevel(_options.Url, 0));
while (siteDownloadContext.DownloadQueue.Count > 0)
{
var page = siteDownloadContext.DownloadQueue.Dequeue();
_pageDownloader.Download(page, siteDownloadContext);
OnPageDownloaded(new PageDownloadedEventArgs(page));
}
IsCompleted = true;
OnDownLoadCompleted(new DownloadCompletedEventArgs(DownloadedPages));
return(siteDownloadContext.DownloadedPages);
}
public IEnumerable<PageLevel> Download()
{
var site = CreateSite(_options.SiteName);
var siteDownloadContext = new SiteDownloadContext(site, _options, _downloadedPages, _downloadQueue);
siteDownloadContext.DownloadQueue.Enqueue(new PageLevel(_options.Url, 0));
while (siteDownloadContext.DownloadQueue.Count > 0)
{
var page = siteDownloadContext.DownloadQueue.Dequeue();
_pageDownloader.Download(page, siteDownloadContext);
OnPageDownloaded(new PageDownloadedEventArgs(page));
}
IsCompleted = true;
OnDownLoadCompleted(new DownloadCompletedEventArgs(DownloadedPages));
return siteDownloadContext.DownloadedPages;
}
