public ConsumerBroker(int maxDownloadThreadNumber, IMonitorableScheduler scheduler, IEnumerable<IDownloader> downloaders, ResultPipeline resultPipeline, IEnumerable<KeyValuePair<string, Type>> pageAnalyzerTypes)
{
_maxDownloadThreadNumber = maxDownloadThreadNumber;
_scheduler = scheduler;
_downloaders = new List<IDownloader>();
_downloaders.AddRange(downloaders);
_downloaders.GroupBy(e => e.Topic).ToList().ForEach(g =>
{
if (g.Count() > 1)
throw new ArgumentException("downloader.Topic不能重复", nameof(downloaders));
});
_resultPipeline = resultPipeline;
_pageAnalyzerTypes = new ConcurrentDictionary<string, Type>();
pageAnalyzerTypes.ToList().ForEach(t => _pageAnalyzerTypes.TryAdd(t.Key, t.Value));
_messagePullAutoResetEvent = new AutoResetEvent(false);
}
public void ConsumeTest()
{
var site = new Site() {Domain = "www.usashopcn.com"};
var requestMessages = NewTestRequestMessages(site);
var bloomFilter = new MemoryBloomFilter<string>(1000 * 10, 1000 * 10 * 20);
var lineScheduler = new SequenceScheduler(bloomFilter);
lineScheduler.Push(requestMessages);
Trace.WriteLine("CurrentQueueCount:" + lineScheduler.CurrentQueueCount());
var resultPipeline = new ResultPipeline();
resultPipeline.RegisterModule(new TestPipelineMoudle());
var pageAnalyzers = new List<KeyValuePair<string, Type>>();
pageAnalyzers.Add(new KeyValuePair<string, Type>(site.Domain, typeof(TestPageAnalyzer)));
var consumerBroker = new ConsumerBroker(2, lineScheduler, new List<IDownloader>() {new TestDownloader()}, new ResultPipeline(), pageAnalyzers);
consumerBroker.Start();
var timer = new Timer((state) =>
{
Trace.WriteLine("Timer结束:");
Assert.IsTrue(lineScheduler.CurrentQueueCount() == 0, "添加消息失败");
}, null, 20000, 0);
//lineScheduler.Dispose();
}
