private List <SerpWebPage> Execute(string query, IRuntimeModule <HttpProviderRuntimeConfig, HttpProviderData> httpModule, IRuntimeModule <AntigateTaskConfig, AntigateTaskResult> antigateModule) { if (httpModule == null) { throw new NullReferenceException("Http module"); } if (antigateModule == null) { throw new NullReferenceException("Antigate module"); } List <SerpWebPage> result = new List <SerpWebPage>(); int pageLimit = Configuration.PageLimit.HasValue ? Configuration.PageLimit.Value : 1; YandexHtmlSerpParser parser = new YandexHtmlSerpParser(); HtmlTagPageParser tagParser = new HtmlTagPageParser(); for (int i = 0; i < pageLimit; i++) { var serpList = ExecuteRequest(new YandexQuery(query, i, Configuration.Region), parser, httpModule, antigateModule); result.AddRange(LoadSerpItemDetails(tagParser, httpModule, serpList)); } return(result); }
private List <YandexHtmlSerpItem> ExecuteRequest(YandexQuery query, YandexHtmlSerpParser parser, IRuntimeModule <HttpProviderRuntimeConfig, HttpProviderData> httpModule, IRuntimeModule <AntigateTaskConfig, AntigateTaskResult> antigateModule) { List <YandexHtmlSerpItem> result = new List <YandexHtmlSerpItem>(); try { result = ExecuteRequest(query.Generate(QueryGenerator), parser, httpModule, antigateModule); } catch (CaptchaException exc) { var captchaResult = ResolveCaptcha(exc.ImageUrl, httpModule, antigateModule); if (string.IsNullOrEmpty(captchaResult.ErrorCode)) { result = ExecuteRequest(new YandexCheckCaptchaQuery(query.Query, query.Page, query.Region, exc.Key, captchaResult.Solution.Text, exc.Retpath), parser, httpModule, antigateModule); } } return(result); }
private List <YandexHtmlSerpItem> ExecuteRequest(string query, YandexHtmlSerpParser parser, IRuntimeModule <HttpProviderRuntimeConfig, HttpProviderData> httpModule, IRuntimeModule <AntigateTaskConfig, AntigateTaskResult> antigateModule) { List <YandexHtmlSerpItem> result = new List <YandexHtmlSerpItem>(); var httpResult = RuntimeTask.Run(Context, httpModule, new HttpProviderRuntimeConfig() { Query = query }); if (httpResult.IsSuccessfully) { using (var stream = httpResult.Data.GetContent()) { var parserResult = RuntimeTask.Run(Context, parser, stream); if (!parserResult.IsSuccessfully) { throw parserResult.Error; } result.AddRange(parserResult.Data); } } return(result); }