public void Start()
{
while (true)
{
Console.ReadLine();
_statisticPort.Post(s => s.GetCrawledCount(this));
_statisticPort.Post(s => s.GetContent(this, @"http://home.cnblogs.com/q/"));
}
}
void ICrawlRequestHandler.Crawl(IPort <ICrawlResponseHandler> collector, string url)
{
var client = new WebClient();
client.DownloadStringCompleted += (sender, e) =>
{
if (e.Error == null)
{
var links = _processingUnit.Processing(e.Result);
collector.Post(c => c.Succeeded(this, url, e.Result, links));
}
else
{
collector.Post(c => c.Failed(this, url, e.Error));
}
};
client.DownloadStringAsync(new Uri(url));
}
void IStatisticRequestHandelr.GetContent(IPort <IStatisticResponseHandler> requester, string url)
{
string content;
if (!_urlContent.TryGetValue(url, out content))
{
content = null;
}
requester.Post(r => r.ReplyContent(url, content));
}
void ICrawlRequestHandler.Crawl(IPort <ICrawlResponseHandler> collector, string url)
{
WebClient client = new WebClient();
client.DownloadStringCompleted += (sender, e) =>
{
if (e.Error == null)
{
var matches = Regex.Matches(e.Result, @"href=""(http://[^""]+)""").Cast <Match>();
var links = matches.Select(m => m.Groups[1].Value).Distinct().ToList();
collector.Post(c => c.Succeeded(this, url, e.Result, links));
}
else
{
collector.Post(c => c.Failed(this, url, e.Error));
}
};
client.DownloadStringAsync(new Uri(url));
}
void ICountResponseHandler.OutCurrentTotal(IPort <ICountRequestHandler> counter, int newValue)
{
this.currentValue = newValue;
this.index++;
if (this.index <= this.maxValue)
{
counter.Post(ct => ct.Count(this, currentValue, index));
}
else
{
Console.WriteLine("CurrentTotal: {0}", newValue);
}
}
private void DispatchCrawlingTasks(IPort <ICrawlRequestHandler> reusableCrawler)
{
if (_readyToCrawl.Count <= 0)
{
WorkingCrawlerCount--;
}
var url = _readyToCrawl.Dequeue();
reusableCrawler.Post(c => c.Crawl(this, url));
while ((_readyToCrawl.Count > 0) && (WorkingCrawlerCount < MaxCrawlerCount))
{
var newUrl = _readyToCrawl.Dequeue();
IPort <ICrawlRequestHandler> crawler = new Crawler(_processingUnit);
var a = new Action <ICrawlRequestHandler>(c => c.Crawl(this, newUrl));
crawler.Post(a);
WorkingCrawlerCount++;
}
}
private void DispatchCrawlingTasks(IPort <ICrawlRequestHandler> reusableCrawler)
{
if (this.m_readyToCrawl.Count <= 0)
{
this.WorkingCrawlerCount--;
}
var url = this.m_readyToCrawl.Dequeue();
reusableCrawler.Post(c => c.Crawl(this, url));
while (this.m_readyToCrawl.Count > 0 &&
this.WorkingCrawlerCount < this.MaxCrawlerCount)
{
var newUrl = this.m_readyToCrawl.Dequeue();
IPort <ICrawlRequestHandler> crawler = new Crawler();
var a = new Action <ICrawlRequestHandler>(c => c.Crawl(this, newUrl));
crawler.Post(a);
this.WorkingCrawlerCount++;
}
}
public void StartCalculate(int maxValue)
{
this.maxValue = maxValue;
counter = new Counter();
counter.Post(ct => ct.Count(this, currentValue, index));
}
void ICountRequestHandler.Count(IPort <ICountResponseHandler> collector, int baseValue, int value)
{
collector.Post(c => c.OutCurrentTotal(this, baseValue + value));
}
void IStatisticRequestHandelr.GetCrawledCount(IPort <IStatisticResponseHandler> requester)
{
requester.Post(r => r.ReplyCrawledCount(_urlContent.Count));
}
