public void Test4(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
crawlQueue.Push(new CrawlerQueueEntry());
crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
var actualValue = crawlQueue.Pop();
Assert.IsNull(actualValue);
}
public void Test4(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
crawlQueue.Push(new CrawlerQueueEntry());
crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
var actualValue = crawlQueue.Pop();
Assert.IsNull(actualValue);
}
private void ProcessNextInQueue()
{
CrawlerQueueEntry crawlerQueueEntry = m_CrawlerQueue.Pop();
if (crawlerQueueEntry.IsNull() || !OnBeforeDownload(crawlerQueueEntry.CrawlStep))
{
return;
}
PropertyBag propertyBag = Download(crawlerQueueEntry.CrawlStep);
if (propertyBag.IsNull())
{
return;
}
// Assign initial properties to propertybag
if (!crawlerQueueEntry.Properties.IsNull())
{
crawlerQueueEntry.Properties.
ForEach(key => propertyBag[key.Key].Value = key.Value);
}
propertyBag.Referrer = crawlerQueueEntry.Referrer;
if (OnAfterDownload(crawlerQueueEntry.CrawlStep, propertyBag))
{
ExecutePipeLine(propertyBag);
}
}
private void StartDownload()
{
CrawlerQueueEntry crawlerQueueEntry = m_CrawlerQueue.Pop();
if (crawlerQueueEntry.IsNull() || !OnBeforeDownload(crawlerQueueEntry.CrawlStep))
{
return;
}
IWebDownloader webDownloader = m_WebDownloaderFactory();
webDownloader.MaximumDownloadSizeInRam = MaximumDownloadSizeInRam;
webDownloader.ConnectionTimeout = ConnectionTimeout;
webDownloader.MaximumContentSize = MaximumContentSize;
webDownloader.DownloadBufferSize = DownloadBufferSize;
webDownloader.UserAgent = UserAgent;
webDownloader.UseCookies = UseCookies;
webDownloader.ReadTimeout = ConnectionReadTimeout;
webDownloader.RetryCount = DownloadRetryCount;
webDownloader.RetryWaitDuration = DownloadRetryWaitDuration;
m_Logger.Verbose("Downloading {0}", crawlerQueueEntry.CrawlStep.Uri);
ThreadSafeCounter.ThreadSafeCounterCookie threadSafeCounterCookie = m_ThreadInUse.EnterCounterScope(crawlerQueueEntry);
Interlocked.Increment(ref m_VisitedCount);
webDownloader.DownloadAsync(crawlerQueueEntry.CrawlStep, crawlerQueueEntry.Referrer, DownloadMethod.GET,
EndDownload, OnDownloadProgress, threadSafeCounterCookie);
}
public void Test5(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
DateTime now = DateTime.Now;
crawlQueue.Push(new CrawlerQueueEntry
{
CrawlStep = new CrawlStep(new Uri("http://www.biz.org/"), 0),
Properties = new Dictionary <string, object>
{
{ "one", "string" },
{ "two", 123 },
{ "three", now },
},
Referrer = new CrawlStep(new Uri("http://www.biz3.org/"), 1)
});
Assert.AreEqual(1, crawlQueue.Count);
CrawlerQueueEntry entry = crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
Assert.NotNull(entry);
Assert.NotNull(entry.CrawlStep);
Assert.NotNull(entry.Properties);
Assert.NotNull(entry.Referrer);
Assert.AreEqual(0, entry.CrawlStep.Depth);
Assert.AreEqual("http://www.biz.org/", entry.CrawlStep.Uri.ToString());
Assert.AreEqual("one", entry.Properties.Keys.First());
Assert.AreEqual("two", entry.Properties.Keys.Skip(1).First());
Assert.AreEqual("three", entry.Properties.Keys.Skip(2).First());
Assert.AreEqual("string", entry.Properties["one"]);
Assert.AreEqual(123, entry.Properties["two"]);
Assert.AreEqual(now, entry.Properties["three"]);
Assert.AreEqual(0, crawlQueue.Count);
}
public void Test3(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
crawlQueue.Push(new CrawlerQueueEntry());
crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
}
public void Test5(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
DateTime now = DateTime.Now;
crawlQueue.Push(new CrawlerQueueEntry
{
CrawlStep = new CrawlStep(new Uri("http://www.biz.org/"), 0),
Properties = new Dictionary<string, object>
{
{"one", "string"},
{"two", 123},
{"three", now},
},
Referrer = new CrawlStep(new Uri("http://www.biz3.org/"), 1)
});
Assert.AreEqual(1, crawlQueue.Count);
CrawlerQueueEntry entry = crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
Assert.NotNull(entry);
Assert.NotNull(entry.CrawlStep);
Assert.NotNull(entry.Properties);
Assert.NotNull(entry.Referrer);
Assert.AreEqual(0, entry.CrawlStep.Depth);
Assert.AreEqual("http://www.biz.org/", entry.CrawlStep.Uri.ToString());
Assert.AreEqual("one", entry.Properties.Keys.First());
Assert.AreEqual("two", entry.Properties.Keys.Skip(1).First());
Assert.AreEqual("three", entry.Properties.Keys.Skip(2).First());
Assert.AreEqual("string", entry.Properties["one"]);
Assert.AreEqual(123, entry.Properties["two"]);
Assert.AreEqual(now, entry.Properties["three"]);
Assert.AreEqual(0, crawlQueue.Count);
}
public void Test3(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
crawlQueue.Push(new CrawlerQueueEntry());
crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
}
public void Test3(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
crawlQueue.Push(new CrawlerQueueEntry());
crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
if (crawlQueue is IDisposable)
{
((IDisposable)crawlQueue).Dispose();
}
}
public void Test3(ICrawlerQueue crawlQueue)
{
Assert.NotNull(crawlQueue);
crawlQueue.Push(new CrawlerQueueEntry());
crawlQueue.Pop();
Assert.AreEqual(0, crawlQueue.Count);
if (crawlQueue is IDisposable)
{
((IDisposable)crawlQueue).Dispose();
}
}