public PageCrawlResult Crawl(Uri url)
{
var crawlResult = new PageCrawlResult();
var downloadResponse = pageDownloader.Download(url);
crawlResult.StatusCode = downloadResponse.StatusCode;
if (downloadResponse.IsSuccessful)
{
crawlResult.Contents = downloadResponse.Contents;
crawlResult.Links = htmlParser.ExtractLinks(url, downloadResponse.Contents);
}
crawlResult.CrawlEndedAt = DateTimeOffset.Now;
return(crawlResult);
}
public IEnumerable <PageLevel> Download()
{
var site = CreateSite(_options.SiteName);
var siteDownloadContext = new SiteDownloadContext(site, _options, _downloadedPages, _downloadQueue);
siteDownloadContext.DownloadQueue.Enqueue(new PageLevel(_options.Url, 0));
while (siteDownloadContext.DownloadQueue.Count > 0)
{
var page = siteDownloadContext.DownloadQueue.Dequeue();
_pageDownloader.Download(page, siteDownloadContext);
OnPageDownloaded(new PageDownloadedEventArgs(page));
}
IsCompleted = true;
OnDownLoadCompleted(new DownloadCompletedEventArgs(DownloadedPages));
return(siteDownloadContext.DownloadedPages);
}
public Method Parse(Uri url)
{
var methodJsonString = pageDownloader.Download(url);
var jObject = JObject.Parse(methodJsonString);
var description = GetDescription(jObject);
var block = GetBlock(jObject);
var name = GetName(jObject);
var requestParameters = GetRequestParameters(jObject);
var responseParameters = GetResponseParameters(jObject);
return(new Method
{
Name = name,
Block = block,
Description = description,
RequestParameters = requestParameters,
ResponseParameters = responseParameters
});
}