private async Task IndexDocument(Uri baseUri, SiteResource siteResource, string newIndexName) { _queryTimer.Start(); _logger.Info($" Downloading Index Records for type {typeof(T).Name}..."); var searchItemCountUri = new Uri(baseUri, string.Format(siteResource.SearchTotalItemsUrl, 1)); string totalSearchItemsString = null; var retryCount = 0; do { totalSearchItemsString = await _dataSource.Download(searchItemCountUri); }while (_dataSource.LastCode == HttpStatusCode.Unauthorized && ++retryCount < 3); ValidateDownResponse(_dataSource.LastCode); if (!int.TryParse(totalSearchItemsString, out int totalSearchItems)) { var errorMsg = $"Get Total Search Item Count returned invalid data from : {totalSearchItemsString}"; throw new InvalidCastException(errorMsg); } _logger.Info($"Estimated Total Search Items Count for type {typeof(T).Name} equals {totalSearchItems}"); var pages = (int)Math.Ceiling(totalSearchItems / (double)_pageSize); for (int pageNumber = 1; pageNumber <= pages; pageNumber++) { var searchUri = new Uri(baseUri, string.Format(siteResource.SearchItemsUrl, _pageSize, pageNumber)); IEnumerable <T> searchItems; retryCount = 0; do { searchItems = await _dataSource.Download <IEnumerable <T> >(searchUri); }while (_dataSource.LastCode == HttpStatusCode.Unauthorized && ++retryCount < 3); try { ValidateDownResponse(_dataSource.LastCode); } catch (Exception ex) { _logger.Error(ex, $" Error while retriving page {pageNumber} for type {typeof(T).Name}."); // removed throw to allow index to be created when 1 or 2 pages fail to be retrieved. } _indexTimer.Start(); _logger.Info($" Indexing Documents for type {typeof(T).Name}...page : {pageNumber}"); _indexProvider.IndexDocuments(newIndexName, searchItems); _indexTimer.Stop(); _logger.Info($"Indexing Time {_indexTimer.Elapsed} page : {pageNumber}"); } _queryTimer.Stop(); _logger.Info($"Query Elapse Time For {typeof(T).Name} : {_queryTimer.Elapsed}"); }