public void Should_return_url_only_once() { #region arrange data const string url1 = "http://sub.domain.com?p1=v1"; const string url2 = "http://sub.domain.com?p1=v2"; var expectedDate1 = new DateTime(2016, 1, 1); var expectedDate2 = new DateTime(2016, 1, 2); var asOfDate = new DateTime(2016, 1, 3); using (_db.CreateTransaction()) { var urlRep = new UrlFrontierRepository(); urlRep.AddOrUpdateUrl(url1, expectedDate1); urlRep.AddOrUpdateUrl(url2, expectedDate2); #endregion urlRep.GetAvailableUrls(1, asOfDate); var url = urlRep.GetAvailableUrls(1, asOfDate).First(); Assert.NotNull(url); Assert.Equal(url2, url.Url); } }
public void Should_store_data_block() { const string expectedUrl = "http://test.com"; const string expectedDataLink = "link"; using (_db.CreateTransaction()) { var frontier = new UrlFrontierRepository(); var nextAvailableTime = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); // already available frontier.AddOrUpdateUrl(expectedUrl, nextAvailableTime); var urlItem = frontier.GetAvailableUrls(1, DateTime.UtcNow).First(); // should return one item var jobs = new JobRepository(Mock.Of <IActivityLogRepository>()); var jobItem = jobs.Start(urlItem); // add a data block by means of the repository var dataRep = new DataRepository(Mock.Of <IActivityLogRepository>()); var blockId = dataRep.StoreData(jobItem, DataBlockType.Link, expectedDataLink); // try to get the data block from DB directly using (var ctx = _db.CreateDbContext()) { var dataBlock = ctx.DataBlocks.Include(b => b.Url).SingleOrDefault(b => b.Id == blockId); Assert.NotNull(dataBlock); Assert.Equal(expectedDataLink, dataBlock.Data); Assert.Equal(expectedUrl, dataBlock.Url.Url); } } }
public void Should_release_URL_when_job_is_stopped() { const string testUrl = "http://test.com"; using (_db.CreateTransaction()) { #region action var frontier = new UrlFrontierRepository(); var nextAvailableTime = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); // already available frontier.AddOrUpdateUrl(testUrl, nextAvailableTime); var urlItem = frontier.GetAvailableUrls(1, DateTime.UtcNow).First(); // should return one item var jobRep = new JobRepository(Mock.Of <IActivityLogRepository>()); var job = jobRep.Start(urlItem); // adds the 1st log message // wait a little bit to force difference in time between two log messages and sort them later Thread.Sleep(100); jobRep.Stop(job); // marks the job finished #endregion #region assertion // make sure the URL is not in progress using (var ctx = _db.CreateDbContext()) { urlItem = ctx.UrlItems.Single(url => url.Id == urlItem.Id); Assert.False(urlItem.IsInProgress); } #endregion } }
public void Should_add_url_only_once() { const string url = "http://sub.domain.com?p1=v1&p2=v2"; var expectedDate1 = new DateTime(2016, 1, 1); var urlRep = new UrlFrontierRepository(); using (_db.CreateTransaction()) { urlRep.AddOrUpdateUrl(url, expectedDate1); urlRep.AddOrUpdateUrl(url, expectedDate1); using (var ctx = _db.CreateDbContext()) { var actualRecords = ctx.UrlItems.Count(u => u.Url == url); Assert.Equal(1, actualRecords); } } }
public void Should_add_url() { const string url = "http://sub.domain.com?p1=v1&p2=v2"; const string host = "sub.domain.com"; var expectedDate = new DateTime(2016, 1, 1); var urlRep = new UrlFrontierRepository(); using (_db.CreateTransaction()) { urlRep.AddOrUpdateUrl(url, expectedDate); using (var ctx = _db.CreateDbContext()) { var dbUrlItem = ctx.UrlItems.SingleOrDefault(u => u.Url == url); Assert.NotNull(dbUrlItem); Assert.Equal(url, dbUrlItem.Url); Assert.Equal(host, dbUrlItem.Host); } } }