public async Task <SearchDataAndResults> ExecuteSearchAsync(string searchArea, CancellationToken cancellationToken)
{
try
{
_configuration = _configResolver.ResolveConfig(searchArea);
var searchDataAndResults = new SearchDataAndResults {
CookieContainer = new CookieContainer()
};
var handler = HttpClientHelpers.CreateHttpClientHandler(_systemConfig, _configuration, searchDataAndResults.CookieContainer);
await _logger.LogInformationAsync($"Beginning searches for {searchArea.ToUpper()}...", cancellationToken);
using (var client = new HttpClientWrapper(_configuration.BaseUri, handler, _logger, _systemConfig))
{
await LogSearchInputsAsync(cancellationToken);
await client.GetAsync(_configuration.SearchRoute, new CancellationToken());
var searchDates = await DateChunker.SplitDateRange(_searchConfig.StartDate, _searchConfig.EndDate, _configuration.ChunkSizeDays);
searchDataAndResults.SearchResultsPages = new List <HttpResponseMessage>();
foreach (KeyValuePair <string, string> range in searchDates)
{
client.DefaultRequestHeaders.Remove("Referer");
client.DefaultRequestHeaders.Add("Referer", $"{_configuration.BaseUri}{_configuration.SearchRoute}");
async Task <HttpRequestMessage> PostRequestBuilder() => await BuildPostFormUrlEncodedRequestAsync(range);
var initialSearchResponse = await client.PostAsync(PostRequestBuilder, cancellationToken);
if (!await SearchResultsExist(initialSearchResponse))
{
continue;
}
client.DefaultRequestHeaders.Remove("Referer");
client.DefaultRequestHeaders.Add("Referer", $"{_configuration.BaseUri}{_configuration.AdvancedSearchRoute}");
async Task <HttpRequestMessage> PagedRequestBuilder() => await BuildPagedSearchResultsRequestAsync();
var searchResults = await client.PostAsync(PagedRequestBuilder, new CancellationToken());
await _logger.LogInformationAsync($"Post search response status: {searchResults.StatusCode}", cancellationToken);
searchDataAndResults.SearchResultsPages.Add(searchResults);
}
return(searchDataAndResults);
}
}
catch (Exception ex)
{
await _logger.LogExceptionAsync($"Search failed!", ex, cancellationToken);
throw new SearchFailedException(ex.Message, ex.InnerException);
}
}
public async Task <IEnumerable <PlanningApplication> > ExtractDataAsync(string searchArea, List <HttpResponseMessage> searchResultPages, CookieContainer cookieContainer, CancellationToken cancellationToken)
{
try
{
_configuration = _configResolver.ResolveConfig(searchArea);
var currentPage = 0;
await _logger.LogInformationAsync($"Processing {searchResultPages.Count} search result pages for {searchArea.ToUpper()}", cancellationToken);
var client = HttpClientHelpers.CreateClient(_configuration.BaseUri, _systemConfig, _configuration, _logger, cookieContainer);
foreach (var searchResults in searchResultPages)
{
currentPage++;
var searchResultsHtml = await searchResults.Content.ReadAsStringAsync();
var searchPageResponseDoc = CQ.Create(searchResultsHtml);
var appSummaryPaths = GetAppSummaryPaths(searchPageResponseDoc);
await _logger.LogInformationAsync($"Found {appSummaryPaths.Count} planning applications in page {currentPage}...", cancellationToken);
var row = 0;
foreach (var appSummaryPath in appSummaryPaths)
{
row++;
var planningApplication = new PlanningApplication();
await _logger.LogInformationAsync($"Getting application detail for result number {row} application {appSummaryPath}", cancellationToken);
await ExtractApplicationSummary(cancellationToken, appSummaryPath, client, planningApplication);
var appDetailsPath = await ExtractApplicationDetails(cancellationToken, appSummaryPath, client, planningApplication);
await ExtractApplicationContact(cancellationToken, appDetailsPath, client, planningApplication);
_planningApplications.Add(planningApplication);
if (_configuration.UseProxy)
{
// refresh client/handler to get a new IP address
client = HttpClientHelpers.CreateClient(_configuration.BaseUri, _systemConfig, _configuration, _logger, cookieContainer);
}
}
}
await _logger.LogInformationAsync($"Finished extracting planning data for {searchArea.ToUpper()}...", cancellationToken);
client.Dispose();
return(_planningApplications);
}
catch (Exception ex)
{
throw new ExtractDataFailedException(ex.Message, ex.InnerException);
}
}
