public async Task Skip_Referencies_From_Other_Domain_If_Flag_Is_Reset()
{
var siteMap = await CrawlAsync("http://sitewithreferencies.com/pageWithReferenciesToOtherDomain.html", 5, false);
AssertExtensions.AreListEqual(
new[] {
"http://sitewithreferencies.com/pages/page2.html",
"http://sitewithreferencies.com/pagewithreferenciestootherdomain.html",
},
siteMap.Keys.Select(v => v.ToLower()).OrderBy(s => s));
}
public async Task Can_Handle_Cicle_Referencies()
{
var siteMap = await CrawlAsync("http://sitewithreferencies.com/pageWithCicleReferencies.html", 5, false);
AssertExtensions.AreListEqual(
new[] {
"http://sitewithreferencies.com/pages/ciclereferencepage.html",
"http://sitewithreferencies.com/pagewithciclereferencies.html",
},
siteMap.Keys.Select(v => v.ToLower()).OrderBy(s => s));
}
public async Task Can_Upload_Site_With_Referencies()
{
var siteMap = await CrawlAsync("http://sitewithreferencies.com", 2, false);
AssertExtensions.AreListEqual(
new[] {
"http://sitewithreferencies.com",
"http://sitewithreferencies.com/pages/page1.html",
"http://sitewithreferencies.com/pages/page2.html"
},
siteMap.Keys.Select(v => v.ToLower()).OrderBy(s => s));
}
public async Task Skip_Anchors()
{
var siteMap = await CrawlAsync("http://sitewithreferencies.com/pageWithAnchors.html", 5, false);
AssertExtensions.AreListEqual(
new[] {
"http://sitewithreferencies.com/pages/page1.html",
"http://sitewithreferencies.com/pagewithanchors.html",
},
siteMap.Keys.Select(v => v.ToLower()).OrderBy(s => s));
loaderMoq.Verify(m => m.DownloadAsync(It.IsAny <string>(), It.IsAny <Stream>()), Times.Exactly(2));
}
public async Task Can_Restrict_Nested_Level()
{
var siteMap = await CrawlAsync("http://sitewithreferencies.com/pageWithFourNestedReferensies.html", 4, true);
AssertExtensions.AreListEqual(
new[] {
"http://sitewithreferencies.com/pages/firstlevelpage.html",
"http://sitewithreferencies.com/pages/secondlevelpage.html",
"http://sitewithreferencies.com/pages/thirdlevelpage.html",
"http://sitewithreferencies.com/pagewithfournestedreferensies.html",
},
siteMap.Keys.Select(v => v.ToLower()).OrderBy(s => s));
}
public async Task Can_Upload_Site_Resources()
{
var siteMap = await CrawlAsync("http://complexsite.com", 2, false);
AssertExtensions.AreListEqual(
new[] {
"http://complexsite.com",
"http://complexsite.com/content/other.css",
"http://complexsite.com/content/test.css",
"http://complexsite.com/images/parovoz.jpg",
"http://complexsite.com/pages/page1.html",
"http://complexsite.com/scripts/otherscript.js",
"http://complexsite.com/scripts/script.js",
},
siteMap.Keys.Select(v => v.ToLower()).OrderBy(s => s));
}
public async Task Can_Upload_One_Page_Site()
{
var siteMap = await CrawlAsync("http://OnePage.com", 1, false);
AssertExtensions.AreListEqual(new[] { "http://OnePage.com" }, siteMap.Keys);
}