public async Task <IEnumerable <Match> > GetMatchesPageAsync(string guid, HashSet <int> tagIds, int pageNumber, bool includeAdditionalInfo, Throttle throttle, ProgressData progressData)
{
var nameUnavailableCount = 0;
var nameUnavailableMax = 60;
var retryCount = 0;
var retryMax = 5;
while (true)
{
await throttle.WaitAsync();
var throttleReleased = false;
try
{
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matches/list?page={pageNumber}&bookmarkdata={{\"moreMatchesAvailable\":true,\"lastMatchesServicePageIdx\":{pageNumber - 1}}}"))
{
throttle.Release();
throttleReleased = true;
testsResponse.EnsureSuccessStatusCode();
var matches = await testsResponse.Content.ReadAsAsync <MatchesV2>();
var result = matches.MatchGroups.SelectMany(matchGroup => matchGroup.Matches)
.Select(match => ConvertMatch(match, tagIds))
.ToList();
// Sometimes Ancestry returns matches with partial data.
// If that happens, retry and hope to get full data the next time.
if (result.Any(match => match.Name == "name unavailable") && ++nameUnavailableCount < nameUnavailableMax)
{
await Task.Delay(3000);
continue;
}
progressData.Increment();
if (includeAdditionalInfo)
{
try
{
await GetAdditionalInfoAsync(guid, result, throttle);
}
catch
{
// non-fatal if unable to download trees
}
if (pageNumber == 1)
{
try
{
await GetParentsAsync(guid, result, throttle);
}
catch
{
// non-fatal if unable to download parents
}
}
}
progressData.Increment();
return(result);
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
FileUtils.LogException(ex, true);
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
return(Enumerable.Empty <Match>());
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
if (!throttleReleased)
{
throttle.Release();
}
}
}
}
private async Task <int> CountMatches(string guid, Func <Match, bool> criteria, int minPage, int maxPage, Throttle throttle, ProgressData progressData)
{
IEnumerable <Match> pageMatches = new Match[0];
// Try to find some page that is at least as high as the highest valid match.
do
{
pageMatches = await GetMatchesPageAsync(guid, new HashSet <int>(), maxPage, false, throttle, progressData);
if (pageMatches.Any(match => !criteria(match)) || !pageMatches.Any())
{
break;
}
maxPage *= 2;
} while (true);
// Back down to find the the page that is exactly as high as the highest valid match
var midPage = minPage;
while (maxPage > minPage)
{
midPage = (maxPage + minPage) / 2;
pageMatches = await GetMatchesPageAsync(guid, new HashSet <int>(), midPage, false, throttle, progressData);
if (pageMatches.Any(match => criteria(match)))
{
if (pageMatches.Any(match => !criteria(match)))
{
break;
}
minPage = midPage + 1;
}
else
{
maxPage = midPage;
}
}
return((midPage - 1) * MatchesPerPage + pageMatches.Count(match => criteria(match)));
}