public void Should_return_default_settings_when_host_dont_have()
{
#region get default settings
HostSetting defaultSettings;
using (var ctx = _db.CreateDbContext())
{
defaultSettings = ctx.HostSettings.Single(s => s.Host == string.Empty);
}
#endregion
const string testUrl = "http://sub.testhost.com/page?param=1¶m=2";
const string testHost = "testhost.com";
#region get settings for host
var urlItem = new UrlItem
{
Url = testUrl, Host = testHost
};
var settingsRep = new CrawlerSettingsRepository(Mock.Of <IActivityLogRepository>());
var hostSettings = settingsRep.GetSettings(urlItem);
#endregion
Assert.NotNull(hostSettings);
Assert.Equal(defaultSettings.Host, hostSettings.Host);
Assert.Equal(defaultSettings.CrawlDelay, hostSettings.CrawlDelay);
Assert.Equal(defaultSettings.RobotsTxt, hostSettings.RobotsTxt);
Assert.True(defaultSettings.Disallow.SequenceEqual(hostSettings.Disallow));
}
public void Should_return_settings_for_host()
{
using (_db.CreateTransaction())
{
const string testUrl = "http://sub.testhost.com/page?param=1¶m=2";
const string testHost = "testhost.com";
#region add test settings for some host
var testSettings = new HostSetting
{
CrawlDelay = 60,
Disallow = null,
Host = testHost,
RobotsTxt = string.Empty
};
using (var ctx = _db.CreateDbContext())
{
ctx.HostSettings.Add(testSettings);
ctx.Commit();
}
#endregion
#region get settings for host
var urlItem = new UrlItem
{
Url = testUrl, Host = testHost
};
var settingsRep = new CrawlerSettingsRepository(Mock.Of <IActivityLogRepository>());
var hostSettings = settingsRep.GetSettings(urlItem);
#endregion
Assert.NotNull(hostSettings);
Assert.Equal(testSettings.Host, hostSettings.Host);
Assert.Equal(testSettings.CrawlDelay, hostSettings.CrawlDelay);
Assert.Equal(testSettings.RobotsTxt, hostSettings.RobotsTxt);
Assert.Equal(testSettings.Disallow, hostSettings.Disallow);
}
}
public void Should_store_host_settings()
{
using (_db.CreateTransaction())
{
const string testUrl = "http://sub.testhost.com/page?param=1¶m=2";
const string testHost = "testhost.com";
var testSetting = new HostSetting
{
CrawlDelay = 60,
Disallow = null,
Host = testHost,
RobotsTxt = string.Empty
};
#region get settings for host
var urlItem = new UrlItem
{
Url = testUrl,
Host = testHost
};
var settingsRep = new CrawlerSettingsRepository(Mock.Of <IActivityLogRepository>());
settingsRep.SetSettings(urlItem, testSetting);
#endregion
using (var ctx = _db.CreateDbContext())
{
var storedSetting = ctx.HostSettings.Single(s => s.Host == testHost);
Assert.Equal(testSetting.Host, storedSetting.Host);
Assert.Equal(testSetting.CrawlDelay, storedSetting.CrawlDelay);
Assert.Equal(testSetting.RobotsTxt, storedSetting.RobotsTxt);
Assert.Equal(testSetting.Disallow, storedSetting.Disallow);
}
}
}
public void Should_return_correct_userAgent()
{
var settings = new CrawlerSettingsRepository(Mock.Of <IActivityLogRepository>());
Assert.Equal("FreeskiHD Crawler v1.0 (www.binary-notes.ru/web-crawler-design) / development version", settings.GetUserAgent());
}
