private async Task UpdatePartitionAsync(IMatchHistoryGetNextPlayerCollectionResult partition, PartitionUpdateResult stats) { // We'll decrement this each time we update one. stats.NumPlayersFailed = partition.Players.Count; // This scanner may support performance tuning options that allow the administrator // to configure the number of simultaneous threads used in scanning. This is so that // we can safely turn on scanning in the web worker roles without worrying about // too much of a perf impact on the site itself. We refresh this value each time // so that we adjust when the configuration is changed. int? degreeOfParallelism = _scannerConfig.RefreshMaxDegreeOfParallelism(); await UpdateAppState("Updating Players in set " + partition.CollectionId).ConfigureAwait(false); Log.TraceInformation("Updating partition running up to {0} simultaneous workers", degreeOfParallelism.HasValue ? degreeOfParallelism.ToString() : "unlimited"); int numWinsForPartition = 0; // Update the players in this partition. await Async.ForEachAsync(partition.Players, degreeOfParallelism, async (player) => { int numWinsForPlayer = await UpdatePlayerAsync(player, stats).ConfigureAwait(false); Interlocked.Add(ref numWinsForPartition, numWinsForPlayer); }).ConfigureAwait(false); // This will naturally be updated in table storage when the lock is released. partition.PlayerSet.TotalWinsForEvent += numWinsForPartition; }
private async Task RecordLatestRefreshResult(PartitionUpdateResult refreshResult) { var record = _participationHandle.CreateDetailedInfoRecord(InfoLevel.Progress); record.AddEntry("PartitionId", refreshResult.PartitionId); record.AddEntry("TimeTilNextUpdate", refreshResult.TimeTilNextUpdate.HasValue ? (long)refreshResult.TimeTilNextUpdate.Value.TotalSeconds : -1); record.AddEntry("TotalDuration", (long)refreshResult.TotalDuration.Value.TotalMilliseconds); record.AddEntry("GetPartitionDuration", (long)refreshResult.GetPartitionDuration.Value.TotalMilliseconds); record.AddEntry("UpdatePartitionDuration", (long)refreshResult.UpdatePartitionDuration.Value.TotalMilliseconds); record.AddEntry("NumPlayersUpdated", refreshResult.NumPlayersUpdated); record.AddEntry("NumNewMatchesFound", refreshResult.NumNewMatchesFound); record.AddEntry("NumNewWinsFound", refreshResult.NumNewWinsFound); record.AddEntry("NumRetrieveMatchesFailures", refreshResult.NumRetrieveMatchesFailures); record.AddEntry("NumMatchResultsProviderFailures", refreshResult.NumMatchResultsProviderFailures); record.AddEntry("NumPlayersFailed", refreshResult.NumPlayersFailed); record.AddEntry("NumSaveMatchesFailures", refreshResult.NumSaveMatchesFailures); record.AddEntry("NumUpdateRegistrationFailures", refreshResult.NumUpdateRegistrationFailures); record.AddEntry("AreResultsOfficial", refreshResult.AreResultsOfficial ? 1 : 0); await _participationHandle.IncrementWorkItemsCompletedAsync(refreshResult.NumPlayersUpdated).ConfigureAwait(false); await record.SaveAsync().ConfigureAwait(false); }
private async Task<int> UpdatePlayerAsync(IMatchHistoryWritablePlayerResults player, PartitionUpdateResult stats) { Log.TraceInformation("Updating player {0}", player.Registration.PlayerId); IMatchResultsProviderResult latestMatches = null; List<IMatchResult> previousMatches = null; const int maxRetries = 3; int numAttempts = 0; int previousErrorCount = player.ErrorCount; while ((numAttempts < maxRetries) && (latestMatches == null || previousMatches == null)) { if (numAttempts > 0) { Thread.Sleep(500); } numAttempts++; ConfiguredTaskAwaitable<IMatchResultsProviderResult> latestMatchesTask = Task.FromResult<IMatchResultsProviderResult>(null).ConfigureAwait(false); ConfiguredTaskAwaitable<List<IMatchResult>> previousMatchesTask = Task.FromResult<List<IMatchResult>>(null).ConfigureAwait(false); Log.TraceInformation("Starting the task to retrieve player {0}'s stored matches", player.Registration.PlayerId); // Start the task to grab the matches for this player from our storage. if (previousMatches == null) { previousMatchesTask = player.GetMatchesAsync(Traits.NumMatchesPerQuery).ConfigureAwait(false); } Log.TraceInformation("Starting the task to retrieve player {0}'s latest results", player.Registration.PlayerId); // Start the task to grab the most recent matches for this player. if (latestMatches == null) { latestMatchesTask = _resultsProvider.GetMatchesForPlayerAsync( player.Registration.ResultsToken, player.ContinuationToken).ConfigureAwait(false); } Log.TraceInformation("Finishing the task to retrieve player {0}'s stored matches", player.Registration.PlayerId); try { previousMatches = await previousMatchesTask; } catch (Exception exception) { Log.TraceEvent(TraceEventType.Error, 0, "Failed to retrieve the stored matches for player {0} with the following exception: {1}", player.Registration.PlayerId, exception); _participationHandle.AddInfoAsync(InfoLevel.Error, String.Format("Worker {0} failed to get old matches from storage for player {1}", _workerId, player.Registration.PlayerId), ExtractExceptionMessage(exception)) .Wait(); if (IsCriticalException(exception)) { throw; } } Log.TraceInformation("Finishing the task to retrieve player {0}'s latest results", player.Registration.PlayerId); try { latestMatches = await latestMatchesTask; } catch (Exception exception) { Log.TraceEvent(TraceEventType.Error, 0, "Failed to retrieve the latest matches for player {0} with the following exception: {1}", player.Registration.PlayerId, exception); _participationHandle.AddInfoAsync(InfoLevel.Error, String.Format("Worker {0} failed to get new matches for player {1}", _workerId, player.Registration.PlayerId), ExtractExceptionMessage(exception)) .Wait(); if (IsCriticalException(exception)) { throw; } } } // Count the error and then abandon updating this player. if (latestMatches == null) { Interlocked.Increment(ref stats.NumMatchResultsProviderFailures); } if (previousMatches == null) { Interlocked.Increment(ref stats.NumRetrieveMatchesFailures); } if (latestMatches == null || previousMatches == null) { Log.TraceEvent(TraceEventType.Error, 0, "Abandoning the update for player {0}", player.Registration.PlayerId); await player.UpdateErrorCountAsync(previousErrorCount + 1).ConfigureAwait(false); return 0; } // I decided to force the provider to specify the order so that it was always an explicit decision. // Without doing this, I often forgot which order was used. But only one is really supported. Better // to explicitly throw if the wrong one is used rather than silently fail. if (latestMatches.Order == MatchOrdering.LeastRecentlyPlayedFirst) { throw new NotSupportedException("This ordering is not supported. You must return matches in most-recently-played order."); } Log.TraceInformation("Merging player {0}'s matches", player.Registration.PlayerId); // Merge the two lists, and extract the matches that are new. var newMatches = MergeMatches(previousMatches, latestMatches.Matches); Log.TraceInformation("Found {0} new matches for player {1}", newMatches.Count, player.Registration.PlayerId); Interlocked.Add(ref stats.NumNewMatchesFound, newMatches.Count); // We may be in the baseline phase, in which case we want to add matches // to the history, but we don't want to count them towards their goal // yet. bool shouldCountOfficialResults = ShouldCountOfficialResults(); if (shouldCountOfficialResults) { Log.TraceInformation("Adding wins to the official result count for player {0}", player.Registration.PlayerId); Interlocked.Add(ref stats.NumNewWinsFound, newMatches.Where(m => m.IsWin).Count()); } // We also should only count matches that occurred in the event timeframe. // When we're finishing the event, we'll do a scan after it completes to make sure // we don't miss any results, but we might pick up matches that were played after // the deadline. Exclude those. Similarly, during baselining we exclude matches // that occurred before the event started. int numWinsToCountTowardsEvent = (!shouldCountOfficialResults) ? 0 : newMatches.Where(m => (m.IsWin) && (m.DatePlayed > _scanTraits.EventStart) && (m.DatePlayed < _scanTraits.EventEnd)).Count(); // Flush the matches we've found for the current player. Log.TraceInformation("Flushing matches for player {0}", player.Registration.PlayerId); try { await player.AddMatchesAndFlushAsync(newMatches, latestMatches.ContinuationToken, numWinsToCountTowardsEvent).ConfigureAwait(false); Interlocked.Decrement(ref stats.NumPlayersFailed); } catch(Exception exception) { Log.TraceEvent(TraceEventType.Error, 0, "Failed to flush the new matches for player {0} with exception {1}", player.Registration.PlayerId, ExtractExceptionMessage(exception)); Interlocked.Increment(ref stats.NumSaveMatchesFailures); player.UpdateErrorCountAsync(previousErrorCount + 1).Wait(); previousErrorCount++; if (IsCriticalException(exception)) { throw; } } Log.TraceInformation("Updating the registration for player {0}", player.Registration.PlayerId); try { await GGCharityDatabase.GetRetryPolicy().ExecuteAsync(async () => { using (GGCharityDatabase db = new GGCharityDatabase(null, null)) { await db.PerformAsync(async () => { var eventRegistration = await db.EventRegistrations.FindAsync(player.Registration.PlayerId, Int32.Parse(_scanTraits.EventId)); eventRegistration.WinsAchieved += numWinsToCountTowardsEvent; await db.SaveChangesAsync(); }).ConfigureAwait(false); } }); } catch (Exception exception) { Log.TraceEvent(TraceEventType.Error, 0, "Failed to update the registration for player {0} with exception {1}", player.Registration.PlayerId, ExtractExceptionMessage(exception)); Interlocked.Increment(ref stats.NumUpdateRegistrationFailures); player.UpdateErrorCountAsync(previousErrorCount + 1).Wait(); previousErrorCount++; if (IsCriticalException(exception)) { throw; } } return numWinsToCountTowardsEvent; }
public async Task<PartitionUpdateResult> RefreshNextPartitionAsync() { Log.TraceInformation("Refreshing the next partition..."); await UpdateAppState("Searching for stale partitions").ConfigureAwait(false); PartitionUpdateResult result = new PartitionUpdateResult(); using (new DurationMeasurement(result.TotalDuration)) { TimeSpan getEntriesStalerThan = GetRefreshTime(); Log.TraceInformation("The refresh time for this iteration is {0}", getEntriesStalerThan); // Get the most stale collection that hasn't been refreshed in the specified // time period. IMatchHistoryGetNextPlayerCollectionResult nextCollectionResult; using (new DurationMeasurement(result.GetPartitionDuration)) { Log.TraceInformation("Getting the next collection"); nextCollectionResult = await _collection.GetNextPlayerCollectionForUpdateAsync(getEntriesStalerThan, TimeSpan.FromMinutes(2)).ConfigureAwait(false); } // The partition is empty, which means there are currently no accounts that // are staler than the specified time period. For test passes, or for official // runs after the event has ended, this means we're done. For official runs, // while the event is in progress, this means we sleep. if (nextCollectionResult.Players == null) { Log.TraceInformation("Did not find a stale collection"); if ((Traits.Type == ScanType.TestPass) || (Time.UtcNow > Traits.EventEnd)) { Log.TraceInformation("Found that the scan is over"); // Only hit each player once. If there are no more // stale entries then we're done. result.TimeTilNextUpdate = null; } else { result.TimeTilNextUpdate = nextCollectionResult.timeTilNextUpdate.Value; Log.TraceInformation("No work to do at the moment, but the scan is ongoing. Wait time of {0}", result.TimeTilNextUpdate); } } else { Log.TraceInformation("Found stale player collection {0}, refreshing...", nextCollectionResult.CollectionId); result.TimeTilNextUpdate = TimeSpan.Zero; result.PartitionId = nextCollectionResult.CollectionId; result.AreResultsOfficial = ShouldCountOfficialResults(); result.NumPlayersUpdated = nextCollectionResult.Players.Count; // There are players that we need to scan. The lock is already held. using (new DurationMeasurement(result.UpdatePartitionDuration)) { using (nextCollectionResult.Lock) { Log.TraceInformation("Refreshing partition {0}", nextCollectionResult.CollectionId); try { await UpdatePartitionAsync(nextCollectionResult, result).ConfigureAwait(false); } catch (LockHasExpiredException) { // Lock expired, we need to move on to the next partition. Log.TraceEvent(TraceEventType.Critical, 0, "The lock expired while updating collection {0}", nextCollectionResult.CollectionId); } } } } } await RecordLatestRefreshResult(result); return result; }