private CrawlingRequest buildCrawlingRequest(SearchRequest request)
{
var crawlingRequest = new CrawlingRequest();
crawlingRequest.QueryWord = request.QueryWord;
//TODO: read from configuration
crawlingRequest.Take = 5;
return(crawlingRequest);
}
public async Task <CrawlingResult> CrawleAsync(CrawlingRequest request)
{
try
{
if (false == ValidateRequest(request))
{
return(null);
}
return(await CrawleInnerAsync(request));
}
catch (Exception ex)
{
Logger.Error(ex);
return(null);
}
}
protected virtual bool ValidateRequest(CrawlingRequest request)
{
string errorPrefix = "Invalid search engine page web crawling request.";
if (request == null)
{
Logger.Error(string.Format("{0} Request is NULL", errorPrefix));
return(false);
}
if (string.IsNullOrEmpty(request.QueryWord))
{
Logger.Error(string.Format("{0} Query word is NULL or EMPTY", errorPrefix));
return(false);
}
if (request.Take < 1)
{
Logger.Error(string.Format("{0} Ivalid take argument value {1}.", errorPrefix, request.Take));
return(false);
}
return(true);
}
protected abstract Task <CrawlingResult> CrawleInnerAsync(CrawlingRequest request);
