private async Task UpdatePartitionAsync(IMatchHistoryGetNextPlayerCollectionResult partition, PartitionUpdateResult stats)
{
// We'll decrement this each time we update one.
stats.NumPlayersFailed = partition.Players.Count;
// This scanner may support performance tuning options that allow the administrator
// to configure the number of simultaneous threads used in scanning. This is so that
// we can safely turn on scanning in the web worker roles without worrying about
// too much of a perf impact on the site itself. We refresh this value each time
// so that we adjust when the configuration is changed.
int? degreeOfParallelism = _scannerConfig.RefreshMaxDegreeOfParallelism();
await UpdateAppState("Updating Players in set " + partition.CollectionId).ConfigureAwait(false);
Log.TraceInformation("Updating partition running up to {0} simultaneous workers", degreeOfParallelism.HasValue ? degreeOfParallelism.ToString() : "unlimited");
int numWinsForPartition = 0;
// Update the players in this partition.
await Async.ForEachAsync(partition.Players, degreeOfParallelism, async (player) =>
{
int numWinsForPlayer = await UpdatePlayerAsync(player, stats).ConfigureAwait(false);
Interlocked.Add(ref numWinsForPartition, numWinsForPlayer);
}).ConfigureAwait(false);
// This will naturally be updated in table storage when the lock is released.
partition.PlayerSet.TotalWinsForEvent += numWinsForPartition;
}
private async Task RecordLatestRefreshResult(PartitionUpdateResult refreshResult)
{
var record = _participationHandle.CreateDetailedInfoRecord(InfoLevel.Progress);
record.AddEntry("PartitionId", refreshResult.PartitionId);
record.AddEntry("TimeTilNextUpdate", refreshResult.TimeTilNextUpdate.HasValue ? (long)refreshResult.TimeTilNextUpdate.Value.TotalSeconds : -1);
record.AddEntry("TotalDuration", (long)refreshResult.TotalDuration.Value.TotalMilliseconds);
record.AddEntry("GetPartitionDuration", (long)refreshResult.GetPartitionDuration.Value.TotalMilliseconds);
record.AddEntry("UpdatePartitionDuration", (long)refreshResult.UpdatePartitionDuration.Value.TotalMilliseconds);
record.AddEntry("NumPlayersUpdated", refreshResult.NumPlayersUpdated);
record.AddEntry("NumNewMatchesFound", refreshResult.NumNewMatchesFound);
record.AddEntry("NumNewWinsFound", refreshResult.NumNewWinsFound);
record.AddEntry("NumRetrieveMatchesFailures", refreshResult.NumRetrieveMatchesFailures);
record.AddEntry("NumMatchResultsProviderFailures", refreshResult.NumMatchResultsProviderFailures);
record.AddEntry("NumPlayersFailed", refreshResult.NumPlayersFailed);
record.AddEntry("NumSaveMatchesFailures", refreshResult.NumSaveMatchesFailures);
record.AddEntry("NumUpdateRegistrationFailures", refreshResult.NumUpdateRegistrationFailures);
record.AddEntry("AreResultsOfficial", refreshResult.AreResultsOfficial ? 1 : 0);
await _participationHandle.IncrementWorkItemsCompletedAsync(refreshResult.NumPlayersUpdated).ConfigureAwait(false);
await record.SaveAsync().ConfigureAwait(false);
}
private async Task<int> UpdatePlayerAsync(IMatchHistoryWritablePlayerResults player, PartitionUpdateResult stats)
{
Log.TraceInformation("Updating player {0}", player.Registration.PlayerId);
IMatchResultsProviderResult latestMatches = null;
List<IMatchResult> previousMatches = null;
const int maxRetries = 3;
int numAttempts = 0;
int previousErrorCount = player.ErrorCount;
while ((numAttempts < maxRetries)
&& (latestMatches == null || previousMatches == null))
{
if (numAttempts > 0)
{
Thread.Sleep(500);
}
numAttempts++;
ConfiguredTaskAwaitable<IMatchResultsProviderResult> latestMatchesTask = Task.FromResult<IMatchResultsProviderResult>(null).ConfigureAwait(false);
ConfiguredTaskAwaitable<List<IMatchResult>> previousMatchesTask = Task.FromResult<List<IMatchResult>>(null).ConfigureAwait(false);
Log.TraceInformation("Starting the task to retrieve player {0}'s stored matches", player.Registration.PlayerId);
// Start the task to grab the matches for this player from our storage.
if (previousMatches == null)
{
previousMatchesTask = player.GetMatchesAsync(Traits.NumMatchesPerQuery).ConfigureAwait(false);
}
Log.TraceInformation("Starting the task to retrieve player {0}'s latest results", player.Registration.PlayerId);
// Start the task to grab the most recent matches for this player.
if (latestMatches == null)
{
latestMatchesTask = _resultsProvider.GetMatchesForPlayerAsync(
player.Registration.ResultsToken,
player.ContinuationToken).ConfigureAwait(false);
}
Log.TraceInformation("Finishing the task to retrieve player {0}'s stored matches", player.Registration.PlayerId);
try
{
previousMatches = await previousMatchesTask;
}
catch (Exception exception)
{
Log.TraceEvent(TraceEventType.Error, 0, "Failed to retrieve the stored matches for player {0} with the following exception: {1}", player.Registration.PlayerId, exception);
_participationHandle.AddInfoAsync(InfoLevel.Error,
String.Format("Worker {0} failed to get old matches from storage for player {1}", _workerId, player.Registration.PlayerId),
ExtractExceptionMessage(exception))
.Wait();
if (IsCriticalException(exception))
{
throw;
}
}
Log.TraceInformation("Finishing the task to retrieve player {0}'s latest results", player.Registration.PlayerId);
try
{
latestMatches = await latestMatchesTask;
}
catch (Exception exception)
{
Log.TraceEvent(TraceEventType.Error, 0, "Failed to retrieve the latest matches for player {0} with the following exception: {1}", player.Registration.PlayerId, exception);
_participationHandle.AddInfoAsync(InfoLevel.Error,
String.Format("Worker {0} failed to get new matches for player {1}", _workerId, player.Registration.PlayerId),
ExtractExceptionMessage(exception))
.Wait();
if (IsCriticalException(exception))
{
throw;
}
}
}
// Count the error and then abandon updating this player.
if (latestMatches == null)
{
Interlocked.Increment(ref stats.NumMatchResultsProviderFailures);
}
if (previousMatches == null)
{
Interlocked.Increment(ref stats.NumRetrieveMatchesFailures);
}
if (latestMatches == null || previousMatches == null)
{
Log.TraceEvent(TraceEventType.Error, 0, "Abandoning the update for player {0}", player.Registration.PlayerId);
await player.UpdateErrorCountAsync(previousErrorCount + 1).ConfigureAwait(false);
return 0;
}
// I decided to force the provider to specify the order so that it was always an explicit decision.
// Without doing this, I often forgot which order was used. But only one is really supported. Better
// to explicitly throw if the wrong one is used rather than silently fail.
if (latestMatches.Order == MatchOrdering.LeastRecentlyPlayedFirst)
{
throw new NotSupportedException("This ordering is not supported. You must return matches in most-recently-played order.");
}
Log.TraceInformation("Merging player {0}'s matches", player.Registration.PlayerId);
// Merge the two lists, and extract the matches that are new.
var newMatches = MergeMatches(previousMatches, latestMatches.Matches);
Log.TraceInformation("Found {0} new matches for player {1}", newMatches.Count, player.Registration.PlayerId);
Interlocked.Add(ref stats.NumNewMatchesFound, newMatches.Count);
// We may be in the baseline phase, in which case we want to add matches
// to the history, but we don't want to count them towards their goal
// yet.
bool shouldCountOfficialResults = ShouldCountOfficialResults();
if (shouldCountOfficialResults)
{
Log.TraceInformation("Adding wins to the official result count for player {0}", player.Registration.PlayerId);
Interlocked.Add(ref stats.NumNewWinsFound, newMatches.Where(m => m.IsWin).Count());
}
// We also should only count matches that occurred in the event timeframe.
// When we're finishing the event, we'll do a scan after it completes to make sure
// we don't miss any results, but we might pick up matches that were played after
// the deadline. Exclude those. Similarly, during baselining we exclude matches
// that occurred before the event started.
int numWinsToCountTowardsEvent = (!shouldCountOfficialResults) ? 0 :
newMatches.Where(m => (m.IsWin)
&& (m.DatePlayed > _scanTraits.EventStart)
&& (m.DatePlayed < _scanTraits.EventEnd)).Count();
// Flush the matches we've found for the current player.
Log.TraceInformation("Flushing matches for player {0}", player.Registration.PlayerId);
try
{
await player.AddMatchesAndFlushAsync(newMatches, latestMatches.ContinuationToken, numWinsToCountTowardsEvent).ConfigureAwait(false);
Interlocked.Decrement(ref stats.NumPlayersFailed);
}
catch(Exception exception)
{
Log.TraceEvent(TraceEventType.Error, 0, "Failed to flush the new matches for player {0} with exception {1}",
player.Registration.PlayerId,
ExtractExceptionMessage(exception));
Interlocked.Increment(ref stats.NumSaveMatchesFailures);
player.UpdateErrorCountAsync(previousErrorCount + 1).Wait();
previousErrorCount++;
if (IsCriticalException(exception))
{
throw;
}
}
Log.TraceInformation("Updating the registration for player {0}", player.Registration.PlayerId);
try
{
await GGCharityDatabase.GetRetryPolicy().ExecuteAsync(async () =>
{
using (GGCharityDatabase db = new GGCharityDatabase(null, null))
{
await db.PerformAsync(async () =>
{
var eventRegistration = await db.EventRegistrations.FindAsync(player.Registration.PlayerId, Int32.Parse(_scanTraits.EventId));
eventRegistration.WinsAchieved += numWinsToCountTowardsEvent;
await db.SaveChangesAsync();
}).ConfigureAwait(false);
}
});
}
catch (Exception exception)
{
Log.TraceEvent(TraceEventType.Error, 0, "Failed to update the registration for player {0} with exception {1}",
player.Registration.PlayerId,
ExtractExceptionMessage(exception));
Interlocked.Increment(ref stats.NumUpdateRegistrationFailures);
player.UpdateErrorCountAsync(previousErrorCount + 1).Wait();
previousErrorCount++;
if (IsCriticalException(exception))
{
throw;
}
}
return numWinsToCountTowardsEvent;
}
public async Task<PartitionUpdateResult> RefreshNextPartitionAsync()
{
Log.TraceInformation("Refreshing the next partition...");
await UpdateAppState("Searching for stale partitions").ConfigureAwait(false);
PartitionUpdateResult result = new PartitionUpdateResult();
using (new DurationMeasurement(result.TotalDuration))
{
TimeSpan getEntriesStalerThan = GetRefreshTime();
Log.TraceInformation("The refresh time for this iteration is {0}", getEntriesStalerThan);
// Get the most stale collection that hasn't been refreshed in the specified
// time period.
IMatchHistoryGetNextPlayerCollectionResult nextCollectionResult;
using (new DurationMeasurement(result.GetPartitionDuration))
{
Log.TraceInformation("Getting the next collection");
nextCollectionResult = await _collection.GetNextPlayerCollectionForUpdateAsync(getEntriesStalerThan, TimeSpan.FromMinutes(2)).ConfigureAwait(false);
}
// The partition is empty, which means there are currently no accounts that
// are staler than the specified time period. For test passes, or for official
// runs after the event has ended, this means we're done. For official runs,
// while the event is in progress, this means we sleep.
if (nextCollectionResult.Players == null)
{
Log.TraceInformation("Did not find a stale collection");
if ((Traits.Type == ScanType.TestPass)
|| (Time.UtcNow > Traits.EventEnd))
{
Log.TraceInformation("Found that the scan is over");
// Only hit each player once. If there are no more
// stale entries then we're done.
result.TimeTilNextUpdate = null;
}
else
{
result.TimeTilNextUpdate = nextCollectionResult.timeTilNextUpdate.Value;
Log.TraceInformation("No work to do at the moment, but the scan is ongoing. Wait time of {0}", result.TimeTilNextUpdate);
}
}
else
{
Log.TraceInformation("Found stale player collection {0}, refreshing...", nextCollectionResult.CollectionId);
result.TimeTilNextUpdate = TimeSpan.Zero;
result.PartitionId = nextCollectionResult.CollectionId;
result.AreResultsOfficial = ShouldCountOfficialResults();
result.NumPlayersUpdated = nextCollectionResult.Players.Count;
// There are players that we need to scan. The lock is already held.
using (new DurationMeasurement(result.UpdatePartitionDuration))
{
using (nextCollectionResult.Lock)
{
Log.TraceInformation("Refreshing partition {0}", nextCollectionResult.CollectionId);
try
{
await UpdatePartitionAsync(nextCollectionResult, result).ConfigureAwait(false);
}
catch (LockHasExpiredException)
{
// Lock expired, we need to move on to the next partition.
Log.TraceEvent(TraceEventType.Critical, 0, "The lock expired while updating collection {0}", nextCollectionResult.CollectionId);
}
}
}
}
}
await RecordLatestRefreshResult(result);
return result;
}
