private async Task IndexDocument(Uri baseUri, SiteResource siteResource, string newIndexName)
{
_queryTimer.Start();
_logger.Info($" Downloading Index Records for type {typeof(T).Name}...");
var searchItemCountUri = new Uri(baseUri, string.Format(siteResource.SearchTotalItemsUrl, 1));
string totalSearchItemsString = null;
var retryCount = 0;
do
{
totalSearchItemsString = await _dataSource.Download(searchItemCountUri);
}while (_dataSource.LastCode == HttpStatusCode.Unauthorized && ++retryCount < 3);
ValidateDownResponse(_dataSource.LastCode);
if (!int.TryParse(totalSearchItemsString, out int totalSearchItems))
{
var errorMsg = $"Get Total Search Item Count returned invalid data from : {totalSearchItemsString}";
throw new InvalidCastException(errorMsg);
}
_logger.Info($"Estimated Total Search Items Count for type {typeof(T).Name} equals {totalSearchItems}");
var pages = (int)Math.Ceiling(totalSearchItems / (double)_pageSize);
for (int pageNumber = 1; pageNumber <= pages; pageNumber++)
{
var searchUri = new Uri(baseUri, string.Format(siteResource.SearchItemsUrl, _pageSize, pageNumber));
IEnumerable <T> searchItems;
retryCount = 0;
do
{
searchItems = await _dataSource.Download <IEnumerable <T> >(searchUri);
}while (_dataSource.LastCode == HttpStatusCode.Unauthorized && ++retryCount < 3);
try
{
ValidateDownResponse(_dataSource.LastCode);
}
catch (Exception ex)
{
_logger.Error(ex, $" Error while retriving page {pageNumber} for type {typeof(T).Name}.");
// removed throw to allow index to be created when 1 or 2 pages fail to be retrieved.
}
_indexTimer.Start();
_logger.Info($" Indexing Documents for type {typeof(T).Name}...page : {pageNumber}");
_indexProvider.IndexDocuments(newIndexName, searchItems);
_indexTimer.Stop();
_logger.Info($"Indexing Time {_indexTimer.Elapsed} page : {pageNumber}");
}
_queryTimer.Stop();
_logger.Info($"Query Elapse Time For {typeof(T).Name} : {_queryTimer.Elapsed}");
}