private async Task <IEnumerable <Match> > GetMatchesPageAsync(string guid, int pageNumber, bool includeTreeInfo, Throttle throttle, ProgressData progressData)
{
var nameUnavailableCount = 0;
var nameUnavailableMax = 60;
var retryCount = 0;
var retryMax = 5;
while (true)
{
try
{
await throttle.WaitAsync();
using (var testsResponse = await _dnaHomeClient.GetAsync($"dna/secure/tests/{guid}/matches?filterBy=ALL&sortBy=RELATIONSHIP&rows={_matchesPerPage}&page={pageNumber}"))
{
testsResponse.EnsureSuccessStatusCode();
var matches = await testsResponse.Content.ReadAsAsync <Matches>();
var result = matches.MatchGroups.SelectMany(matchGroup => matchGroup.Matches);
// Sometimes Ancestry returns matches with partial data.
// If that happens, retry and hope to get full data the next time.
if (result.Any(match => match.Name == "name unavailable") && ++nameUnavailableCount < nameUnavailableMax)
{
await Task.Delay(3000);
continue;
}
throttle.Release();
progressData.Increment();
if (includeTreeInfo)
{
try
{
await GetLinkedTreesAsync(guid, result, throttle);
}
catch
{
// non-fatal if unable to download trees
}
}
progressData.Increment();
return(result);
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
throttle.Release();
throw;
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
}
}
private async Task GetAdditionalInfoAsync(string guid, IEnumerable <Match> matches, Throttle throttle)
{
while (true)
{
await throttle.WaitAsync();
var throttleReleased = false;
try
{
var matchesDictionary = matches.ToDictionary(match => match.TestGuid);
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matchesv2/additionalInfo?ids=[{"%22" + string.Join("%22,%22", matchesDictionary.Keys) + "%22"}]&ancestors=true&tree=true";
using (var testsResponse = await _dnaHomeClient.GetAsync(url))
{
throttle.Release();
throttleReleased = true;
testsResponse.EnsureSuccessStatusCode();
var undeterminedCount = 0;
var additionalInfos = await testsResponse.Content.ReadAsAsync <List <AdditionalInfo> >();
foreach (var additionalInfo in additionalInfos)
{
if (matchesDictionary.TryGetValue(additionalInfo.TestGuid, out var match))
{
match.TreeSize = additionalInfo.TreeSize ?? 0;
match.TreeType =
additionalInfo.PrivateTree == true ? TreeType.Private // might also be unlinked
: additionalInfo.UnlinkedTree == true ? TreeType.Unlinked
: additionalInfo.PublicTree == true && match.TreeSize > 0 ? TreeType.Public
: additionalInfo.NoTrees == true ? TreeType.None
: TreeType.Undetermined;
if (match.TreeType == TreeType.Undetermined)
{
++undeterminedCount;
}
match.HasCommonAncestors = additionalInfo.CommonAncestors ?? false;
}
}
return;
}
}
catch (Exception ex)
{
}
finally
{
if (!throttleReleased)
{
throttle.Release();
}
}
}
}
public async Task AddTagAsync(string guid, string testGuid, int tagId, Throttle throttle)
{
await throttle.WaitAsync();
try
{
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}/tags/{tagId}";
var request = new
{
headers = new
{
normalizedNames = new { },
lazyUpdate = (bool?)null,
lazyInit = (bool?)null,
headers = new { },
}
};
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.PutAsJsonAsync(url, request))
{
testsResponse.EnsureSuccessStatusCode();
}
}
finally
{
throttle.Release();
}
}
private async Task GetParentsAsync(string guid, IEnumerable <Match> matches, Throttle throttle)
{
await throttle.WaitAsync();
try
{
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matchesv2/parents";
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync(url))
{
testsResponse.EnsureSuccessStatusCode();
var parentsInfo = await testsResponse.Content.ReadAsAsync <ParentsInfo>();
if (!string.IsNullOrEmpty(parentsInfo.FatherSampleId) || !string.IsNullOrEmpty(parentsInfo.MotherSampleId))
{
foreach (var match in matches)
{
if (match.TestGuid == parentsInfo.FatherSampleId)
{
match.IsFather = true;
}
if (match.TestGuid == parentsInfo.MotherSampleId)
{
match.IsMother = true;
}
}
}
}
}
finally
{
throttle.Release();
}
}
private async Task <(IEnumerable <Match>, int)> GetMatchesInCommonPageAsync(string guid, string guidInCommon, int pageNumber, Throttle throttle)
{
if (guid == guidInCommon)
{
return(Enumerable.Empty <Match>(), 0);
}
var nameUnavailableCount = 0;
var nameUnavailableMax = 60;
var retryCount = 0;
var retryMax = 5;
while (true)
{
await throttle.WaitAsync();
try
{
using (var testsResponse = await _dnaHomeClient.GetAsync($"dna/secure/tests/{guid}/matchesInCommon?filterBy=ALL&sortBy=RELATIONSHIP&page={pageNumber}&matchTestGuid={guidInCommon}"))
{
if (testsResponse.StatusCode == System.Net.HttpStatusCode.Gone)
{
return(Enumerable.Empty <Match>(), 0);
}
if (testsResponse.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable)
{
await Task.Delay(120000);
continue;
}
testsResponse.EnsureSuccessStatusCode();
var matches = await testsResponse.Content.ReadAsAsync <Matches>();
var matchesInCommon = matches.MatchGroups.SelectMany(matchGroup => matchGroup.Matches);
if (matchesInCommon.Any(match => match.Name == "name unavailable") && ++nameUnavailableCount < nameUnavailableMax)
{
await Task.Delay(3000);
continue;
}
return(matchesInCommon, matches.PageCount);
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
throw;
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
throttle.Release();
}
}
}
private async Task GetPublicTreeAsync(string guid, Match match, Throttle throttle, bool doThrow)
{
var retryCount = 0;
var retryMax = 5;
while (true)
{
try
{
await throttle.WaitAsync();
using (var testsResponse = await _dnaHomeClient.GetAsync($"dna/secure/tests/{guid}/matches/{match.TestGuid}/treeDetails"))
{
testsResponse.EnsureSuccessStatusCode();
var treeDetails = await testsResponse.Content.ReadAsAsync <TreeDetails>();
if (treeDetails.MatchTestHasTree)
{
match.TreeType = TreeType.Public;
match.TreeSize = treeDetails.MatchTreeNodeCount;
}
else if (treeDetails.PublicTreeInformationList?.Any(info => info.IsPublic) == true)
{
match.TreeType = TreeType.Unlinked;
}
else if (treeDetails.PublicTreeInformationList?.Any() == true && treeDetails.MatchTreeNodeCount > 0)
{
match.TreeType = TreeType.Private;
}
else
{
match.TreeType = TreeType.None;
}
return;
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
if (doThrow)
{
throw;
}
return;
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
throttle.Release();
}
}
}
private async Task <List <string> > GetCommonAncestorsAsync(string guid, string testGuid, Throttle throttle)
{
while (true)
{
await throttle.WaitAsync();
var throttleReleased = false;
try
{
var url = $"/discoveryui-matchesservice/api/compare/{guid}/with/{testGuid}/commonancestors/";
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync(url))
{
throttle.Release();
throttleReleased = true;
testsResponse.EnsureSuccessStatusCode();
var ancestorCouples = await testsResponse.Content.ReadAsAsync <Ancestors>();
var result = ancestorCouples.AncestorCouples.SelectMany(couple => new[] { couple.Father, couple.Mother })
.Select(ancestor => ancestor?.PersonData?.DisplayName)
.Where(name => !string.IsNullOrEmpty(name))
.ToList();
return(result.Count > 0 ? result : null);
}
}
catch (Exception ex)
{
}
finally
{
if (!throttleReleased)
{
throttle.Release();
}
}
}
}
public async Task UpdateNotesAsync(string guid, string testGuid, string note, Throttle throttle)
{
await throttle.WaitAsync();
try
{
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}";
using (var testsResponse = await _dnaHomeClient.PutAsJsonAsync(url, new { note }))
{
testsResponse.EnsureSuccessStatusCode();
}
}
finally
{
throttle.Release();
}
}
public async Task DeleteTagAsync(string guid, string testGuid, int tagId, Throttle throttle)
{
await throttle.WaitAsync();
try
{
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}/tags/{tagId}";
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.DeleteAsync(url))
{
testsResponse.EnsureSuccessStatusCode();
}
}
finally
{
throttle.Release();
}
}
public async Task UpdateStarredAsync(string guid, string testGuid, bool starred, Throttle throttle)
{
await throttle.WaitAsync();
try
{
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}";
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.PutAsJsonAsync(url, new { starred }))
{
testsResponse.EnsureSuccessStatusCode();
}
}
finally
{
throttle.Release();
}
}
public async Task <Match> GetMatchAsync(string guid, string testGuid, Throttle throttle, ProgressData progressData)
{
await throttle.WaitAsync();
try
{
using (var testsResponse = await _dnaHomeClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}/details"))
{
return(testsResponse.IsSuccessStatusCode ? await testsResponse.Content.ReadAsAsync <Match>() : null);
}
}
finally
{
progressData.Increment();
throttle.Release();
}
}
public async Task <List <Tag> > GetTagsAsync(string guid, Throttle throttle)
{
await throttle.WaitAsync();
try
{
var url = $"/discoveryui-matchesservice/api/samples/{guid}/tags";
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync(url))
{
testsResponse.EnsureSuccessStatusCode();
return(await testsResponse.Content.ReadAsAsync <List <Tag> >());
}
}
finally
{
throttle.Release();
}
}
public async Task <Match> GetMatchAsync(string guid, string testGuid, HashSet <int> tagIds, Throttle throttle, ProgressData progressData)
{
await throttle.WaitAsync();
try
{
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}/details"))
{
if (!testsResponse.IsSuccessStatusCode)
{
return(null);
}
return(ConvertMatch(await testsResponse.Content.ReadAsAsync <MatchV2>(), tagIds));
}
}
finally
{
progressData.Increment();
throttle.Release();
}
}
public async Task <Match> GetMatchAsync(string guid, string testGuid, HashSet <int> tagIds, Throttle throttle, ProgressData progressData)
{
var retryCount = 0;
var retryMax = 5;
while (true)
{
await throttle.WaitAsync();
try
{
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matches/{testGuid}/details"))
{
if (!testsResponse.IsSuccessStatusCode)
{
return(null);
}
return(ConvertMatch(await testsResponse.Content.ReadAsAsync <MatchV2>(), tagIds));
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
FileUtils.LogException(ex, true);
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
return(null);
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
progressData.Increment();
throttle.Release();
}
}
}
private async Task GetLinkedTreesAsync(string guid, IEnumerable <Match> matches, Throttle throttle)
{
if (!matches.Any())
{
return;
}
while (true)
{
try
{
await throttle.WaitAsync();
var matchesDictionary = matches.ToDictionary(match => match.TestGuid);
var url = $"/discoveryui-matchesservice/api/samples/{guid}/matchesv2/additionalInfo?ids=[{"%22" + string.Join("%22,%22", matchesDictionary.Keys) + "%22"}]&tree=true";
using (var testsResponse = await _dnaHomeClient.GetAsync(url))
{
testsResponse.EnsureSuccessStatusCode();
var treeInfos = await testsResponse.Content.ReadAsAsync <List <TreeInfoV2> >();
foreach (var treeInfo in treeInfos)
{
if (matchesDictionary.TryGetValue(treeInfo.TestGuid, out var match))
{
match.TreeSize = treeInfo.TreeSize ?? 0;
match.TreeType = treeInfo.UnlinkedTree == true ? TreeType.Unlinked
: treeInfo.PrivateTree == true ? TreeType.Private
: treeInfo.PublicTree == true && match.TreeSize > 0 ? TreeType.Public
: treeInfo.NoTrees == true ? TreeType.None
: TreeType.Undetermined;
}
}
return;
}
}
catch (Exception ex)
{
}
finally
{
throttle.Release();
}
/*
* var retryCount = 0;
* var retryMax = 5;
* try
* {
* await throttle.WaitAsync();
* var matchesDictionary = matches.ToDictionary(match => match.TestGuid);
* using (var testsResponse = await _dnaHomeClient.PostAsJsonAsync($"dna/secure/tests/{guid}/treeAncestors", matchesDictionary.Keys))
* {
* testsResponse.EnsureSuccessStatusCode();
* var treeDetails = await testsResponse.Content.ReadAsAsync<Dictionary<string, LinkedTreeDetails>>();
* foreach (var kvp in treeDetails)
* {
* if (matchesDictionary.TryGetValue(kvp.Key, out var match))
* {
* match.TreeSize = kvp.Value.PersonCount;
* if (match.TreeSize > 0)
* {
* match.TreeType = kvp.Value.PrivateTree ? TreeType.Private : TreeType.Public;
* }
* }
* }
* return;
* }
* }
* catch (Exception ex)
* {
* if (++retryCount >= retryMax)
* {
* throw;
* }
* await Task.Delay(ex is UnsupportedMediaTypeException ? 30000 : 3000);
* }
* finally
* {
* throttle.Release();
* }
*/
}
}
private async Task <IEnumerable <Match> > GetMatchesPageAsync(string guid, int pageNumber, bool includeTreeInfo, Throttle throttle, ProgressData progressData)
{
var nameUnavailableCount = 0;
var nameUnavailableMax = 60;
var retryCount = 0;
var retryMax = 5;
while (true)
{
await throttle.WaitAsync();
var throttleReleased = false;
try
{
using (var testsResponse = await _dnaHomeClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matchesv2?page={pageNumber}&bookmarkdata={{\"moreMatchesAvailable\":true,\"lastMatchesServicePageIdx\":{pageNumber - 1}}}"))
{
throttle.Release();
throttleReleased = true;
testsResponse.EnsureSuccessStatusCode();
var matches = await testsResponse.Content.ReadAsAsync <MatchesV2>();
var result = matches.MatchGroups.SelectMany(matchGroup => matchGroup.Matches)
.Select(match => new Match
{
MatchTestAdminDisplayName = match.AdminDisplayName,
MatchTestDisplayName = match.DisplayName,
TestGuid = match.TestGuid,
SharedCentimorgans = match.Relationship?.SharedCentimorgans ?? 0,
SharedSegments = match.Relationship?.SharedSegments ?? 0,
Starred = match.Starred,
Note = match.Note,
})
.ToList();
// Sometimes Ancestry returns matches with partial data.
// If that happens, retry and hope to get full data the next time.
if (result.Any(match => match.Name == "name unavailable") && ++nameUnavailableCount < nameUnavailableMax)
{
await Task.Delay(3000);
continue;
}
progressData.Increment();
if (includeTreeInfo)
{
try
{
await GetLinkedTreesAsync(guid, result, throttle);
}
catch
{
// non-fatal if unable to download trees
}
}
progressData.Increment();
return(result);
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
throw;
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
if (!throttleReleased)
{
throttle.Release();
}
}
}
}
public async Task <IEnumerable <Match> > GetMatchesPageAsync(string guid, HashSet <int> tagIds, int pageNumber, bool includeAdditionalInfo, Throttle throttle, ProgressData progressData)
{
var nameUnavailableCount = 0;
var nameUnavailableMax = 60;
var retryCount = 0;
var retryMax = 5;
while (true)
{
await throttle.WaitAsync();
var throttleReleased = false;
try
{
using (var testsResponse = await _ancestryLoginHelper.AncestryClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matches/list?page={pageNumber}&bookmarkdata={{\"moreMatchesAvailable\":true,\"lastMatchesServicePageIdx\":{pageNumber - 1}}}"))
{
throttle.Release();
throttleReleased = true;
testsResponse.EnsureSuccessStatusCode();
var matches = await testsResponse.Content.ReadAsAsync <MatchesV2>();
var result = matches.MatchGroups.SelectMany(matchGroup => matchGroup.Matches)
.Select(match => ConvertMatch(match, tagIds))
.ToList();
// Sometimes Ancestry returns matches with partial data.
// If that happens, retry and hope to get full data the next time.
if (result.Any(match => match.Name == "name unavailable") && ++nameUnavailableCount < nameUnavailableMax)
{
await Task.Delay(3000);
continue;
}
progressData.Increment();
if (includeAdditionalInfo)
{
try
{
await GetAdditionalInfoAsync(guid, result, throttle);
}
catch
{
// non-fatal if unable to download trees
}
if (pageNumber == 1)
{
try
{
await GetParentsAsync(guid, result, throttle);
}
catch
{
// non-fatal if unable to download parents
}
}
}
progressData.Increment();
return(result);
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
FileUtils.LogException(ex, true);
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
return(Enumerable.Empty <Match>());
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
if (!throttleReleased)
{
throttle.Release();
}
}
}
}
private async Task <(IEnumerable <Match>, bool)> GetMatchesInCommonPageAsync(string guid, string guidInCommon, int pageNumber, Throttle throttle)
{
if (guid == guidInCommon)
{
return(Enumerable.Empty <Match>(), false);
}
var nameUnavailableCount = 0;
var nameUnavailableMax = 60;
var retryCount = 0;
var retryMax = 5;
while (true)
{
await throttle.WaitAsync();
var throttleReleased = false;
try
{
using (var testsResponse = await _dnaHomeClient.GetAsync($"discoveryui-matchesservice/api/samples/{guid}/matchesv2?page={pageNumber}&relationguid={guidInCommon}&bookmarkdata={{\"moreMatchesAvailable\":true,\"lastMatchesServicePageIdx\":{pageNumber - 1}}}"))
{
throttle.Release();
throttleReleased = true;
if (testsResponse.StatusCode == System.Net.HttpStatusCode.Gone)
{
return(Enumerable.Empty <Match>(), false);
}
if (testsResponse.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable)
{
await Task.Delay(120000);
continue;
}
testsResponse.EnsureSuccessStatusCode();
var matches = await testsResponse.Content.ReadAsAsync <MatchesV2>();
var matchesInCommon = matches.MatchGroups.SelectMany(matchGroup => matchGroup.Matches)
.Select(match => new Match
{
MatchTestAdminDisplayName = match.AdminDisplayName,
MatchTestDisplayName = match.DisplayName,
TestGuid = match.TestGuid,
SharedCentimorgans = match.Relationship?.SharedCentimorgans ?? 0,
SharedSegments = match.Relationship?.SharedSegments ?? 0,
Starred = match.Starred,
Note = match.Note,
})
.ToList();
if (matchesInCommon.Any(match => match.Name == "name unavailable") && ++nameUnavailableCount < nameUnavailableMax)
{
await Task.Delay(3000);
continue;
}
return(matchesInCommon, matches.BookmarkData.moreMatchesAvailable);
}
}
catch (Exception ex)
{
if (++retryCount >= retryMax)
{
throw;
}
await Task.Delay(ex is UnsupportedMediaTypeException? 30000 : 3000);
}
finally
{
if (!throttleReleased)
{
throttle.Release();
}
}
}
}