private async Task IndexDocument(Uri baseUri, SiteResource siteResource, string newIndexName)
{
_queryTimer.Start();
_logger.Info($" Downloading Index Records for type {typeof(T).Name}...");
var searchItemCountUri = new Uri(baseUri, string.Format(siteResource.SearchTotalItemsUrl, 1));
string totalSearchItemsString = null;
var retryCount = 0;
do
{
totalSearchItemsString = await _dataSource.Download(searchItemCountUri);
}while (_dataSource.LastCode == HttpStatusCode.Unauthorized && ++retryCount < 3);
ValidateDownResponse(_dataSource.LastCode);
if (!int.TryParse(totalSearchItemsString, out int totalSearchItems))
{
var errorMsg = $"Get Total Search Item Count returned invalid data from : {totalSearchItemsString}";
throw new InvalidCastException(errorMsg);
}
_logger.Info($"Estimated Total Search Items Count for type {typeof(T).Name} equals {totalSearchItems}");
var pages = (int)Math.Ceiling(totalSearchItems / (double)_pageSize);
for (int pageNumber = 1; pageNumber <= pages; pageNumber++)
{
var searchUri = new Uri(baseUri, string.Format(siteResource.SearchItemsUrl, _pageSize, pageNumber));
IEnumerable <T> searchItems;
retryCount = 0;
do
{
searchItems = await _dataSource.Download <IEnumerable <T> >(searchUri);
}while (_dataSource.LastCode == HttpStatusCode.Unauthorized && ++retryCount < 3);
try
{
ValidateDownResponse(_dataSource.LastCode);
}
catch (Exception ex)
{
_logger.Error(ex, $" Error while retriving page {pageNumber} for type {typeof(T).Name}.");
// removed throw to allow index to be created when 1 or 2 pages fail to be retrieved.
}
_indexTimer.Start();
_logger.Info($" Indexing Documents for type {typeof(T).Name}...page : {pageNumber}");
_indexProvider.IndexDocuments(newIndexName, searchItems);
_indexTimer.Stop();
_logger.Info($"Indexing Time {_indexTimer.Elapsed} page : {pageNumber}");
}
_queryTimer.Stop();
_logger.Info($"Query Elapse Time For {typeof(T).Name} : {_queryTimer.Elapsed}");
}
private async Task <ResourceResultModel> GetPage(string url)
{
var result = new ResourceResultModel();
/// var queryString = AddQueryString(url);
result.Resource = await _siteConnector.Download(new Uri(url));
result.StatusCode = _siteConnector.LastCode;
result.Exception = _siteConnector.LastException;
return(result);
}
public async Task <IHttpActionResult> Get()
{
// Get the status of each site
// add it to this one and output the results
var sites = _siteSettings.BaseUrls.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).ToList();
var localResult = new
{
ServiceName = AddServiceName(),
ServiceVersion = AddServiceVersion(),
ServiceTime = AddServerTime(),
Request = AddRequestContext(),
SubSites = new Dictionary <SupportServiceIdentity, dynamic>(),
Sites = sites
};
try
{
var subSites = new Dictionary <SupportServiceIdentity, Uri>();
foreach (var subSite in sites)
{
var siteElements = subSite.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
if (siteElements.Length != 2)
{
continue;
}
if (string.IsNullOrWhiteSpace(siteElements[0]))
{
continue;
}
if (string.IsNullOrWhiteSpace(siteElements[1]))
{
continue;
}
subSites.Add((SupportServiceIdentity)Enum.Parse(typeof(SupportServiceIdentity), siteElements[0]), new Uri(siteElements[1]));
}
foreach (var subSite in subSites)
{
var uri = new Uri(subSite.Value, "api/status");
try
{
await _siteConnector.Download <dynamic>(uri);
localResult.SubSites.Add(subSite.Key, new
{
Result = _siteConnector.HttpStatusCodeDecision,
HttpStatusCode = _siteConnector.LastCode,
Exception = _siteConnector.LastException,
Content = _siteConnector.LastContent,
});
}
catch (Exception e)
{
localResult.SubSites.Add(subSite.Key, new
{
Result = _siteConnector.HttpStatusCodeDecision,
HttpStatusCode = _siteConnector.LastCode,
Exception = e,
Content = _siteConnector.LastContent,
});
}
}
}
catch
{
// ignored
}
return(Ok(localResult));
}