public void Should_return_url_only_once()
{
#region arrange data
const string url1 = "http://sub.domain.com?p1=v1";
const string url2 = "http://sub.domain.com?p1=v2";
var expectedDate1 = new DateTime(2016, 1, 1);
var expectedDate2 = new DateTime(2016, 1, 2);
var asOfDate = new DateTime(2016, 1, 3);
using (_db.CreateTransaction())
{
var urlRep = new UrlFrontierRepository();
urlRep.AddOrUpdateUrl(url1, expectedDate1);
urlRep.AddOrUpdateUrl(url2, expectedDate2);
#endregion
urlRep.GetAvailableUrls(1, asOfDate);
var url = urlRep.GetAvailableUrls(1, asOfDate).First();
Assert.NotNull(url);
Assert.Equal(url2, url.Url);
}
}
public void Should_store_data_block()
{
const string expectedUrl = "http://test.com";
const string expectedDataLink = "link";
using (_db.CreateTransaction())
{
var frontier = new UrlFrontierRepository();
var nextAvailableTime = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); // already available
frontier.AddOrUpdateUrl(expectedUrl, nextAvailableTime);
var urlItem = frontier.GetAvailableUrls(1, DateTime.UtcNow).First(); // should return one item
var jobs = new JobRepository(Mock.Of <IActivityLogRepository>());
var jobItem = jobs.Start(urlItem);
// add a data block by means of the repository
var dataRep = new DataRepository(Mock.Of <IActivityLogRepository>());
var blockId = dataRep.StoreData(jobItem, DataBlockType.Link, expectedDataLink);
// try to get the data block from DB directly
using (var ctx = _db.CreateDbContext())
{
var dataBlock = ctx.DataBlocks.Include(b => b.Url).SingleOrDefault(b => b.Id == blockId);
Assert.NotNull(dataBlock);
Assert.Equal(expectedDataLink, dataBlock.Data);
Assert.Equal(expectedUrl, dataBlock.Url.Url);
}
}
}
public void Should_release_URL_when_job_is_stopped()
{
const string testUrl = "http://test.com";
using (_db.CreateTransaction())
{
#region action
var frontier = new UrlFrontierRepository();
var nextAvailableTime = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); // already available
frontier.AddOrUpdateUrl(testUrl, nextAvailableTime);
var urlItem = frontier.GetAvailableUrls(1, DateTime.UtcNow).First(); // should return one item
var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>());
var job = jobRep.Start(urlItem); // adds the 1st log message
// wait a little bit to force difference in time between two log messages and sort them later
Thread.Sleep(100);
jobRep.Stop(job); // marks the job finished
#endregion
#region assertion
// make sure the URL is not in progress
using (var ctx = _db.CreateDbContext())
{
urlItem = ctx.UrlItems.Single(url => url.Id == urlItem.Id);
Assert.False(urlItem.IsInProgress);
}
#endregion
}
}
public void Should_add_url_only_once()
{
const string url = "http://sub.domain.com?p1=v1&p2=v2";
var expectedDate1 = new DateTime(2016, 1, 1);
var urlRep = new UrlFrontierRepository();
using (_db.CreateTransaction())
{
urlRep.AddOrUpdateUrl(url, expectedDate1);
urlRep.AddOrUpdateUrl(url, expectedDate1);
using (var ctx = _db.CreateDbContext())
{
var actualRecords = ctx.UrlItems.Count(u => u.Url == url);
Assert.Equal(1, actualRecords);
}
}
}
public void Should_add_url()
{
const string url = "http://sub.domain.com?p1=v1&p2=v2";
const string host = "sub.domain.com";
var expectedDate = new DateTime(2016, 1, 1);
var urlRep = new UrlFrontierRepository();
using (_db.CreateTransaction())
{
urlRep.AddOrUpdateUrl(url, expectedDate);
using (var ctx = _db.CreateDbContext())
{
var dbUrlItem = ctx.UrlItems.SingleOrDefault(u => u.Url == url);
Assert.NotNull(dbUrlItem);
Assert.Equal(url, dbUrlItem.Url);
Assert.Equal(host, dbUrlItem.Host);
}
}
}
