private void ProcessCandidates(IAsyncCursor <Candidate> cursor, long expectedCount, IDictionary <string, int> vacancyLocalAuthorities, IDictionary <int, int> localAuthorityCountyIds, SyncType syncType, CancellationToken cancellationToken) { var count = 0; while (cursor.MoveNextAsync(cancellationToken).Result&& !cancellationToken.IsCancellationRequested) { var batch = cursor.Current.ToDictionary(c => c.Id, c => c); if (batch.Count == 0) { continue; } var candidateUsers = new List <CandidateUser>(batch.Count); _logService.Info($"Loading {batch.Count} users"); var usersCursor = _userRepository.GetUsersByIds(batch.Keys, cancellationToken).Result; while (usersCursor.MoveNextAsync(cancellationToken).Result&& !cancellationToken.IsCancellationRequested) { candidateUsers.AddRange(usersCursor.Current.Select(user => new CandidateUser { Candidate = batch[user.Id], User = user })); } _logService.Info($"Processing {candidateUsers.Count} candidates"); var maxDateCreated = candidateUsers.Max(c => c.Candidate.DateCreated); var maxDateUpdated = candidateUsers.Max(c => c.Candidate.DateUpdated) ?? DateTime.MinValue; var candidateSummaries = _candidateRepository.GetCandidateSummariesByGuid(candidateUsers.Select(c => c.Candidate.Id)); var schoolAttendedIds = _schoolAttendedRepository.GetSchoolAttendedIdsByCandidateIds(candidateSummaries.Values.Select(cs => cs.CandidateId)); var candidateHistoryIds = _candidateHistoryRepository.GetCandidateHistoryIdsByCandidateIds(candidateSummaries.Values.Select(cs => cs.CandidateId).Distinct()); var candidatesWithHistory = candidateUsers.Select(c => _candidateMappers.MapCandidateWithHistory(c, candidateSummaries, vacancyLocalAuthorities, localAuthorityCountyIds, schoolAttendedIds, candidateHistoryIds, _anonymiseData)).Where(c => c != null).ToList(); count += candidatesWithHistory.Count; _logService.Info($"Processing {candidatesWithHistory.Count} mapped candidates"); BulkUpsert(candidatesWithHistory, candidateSummaries); var syncParams = _syncRepository.GetSyncParams(); if (syncType == SyncType.Full) { syncParams.CandidateLastCreatedDate = maxDateCreated > syncParams.CandidateLastCreatedDate ? maxDateCreated : syncParams.CandidateLastCreatedDate; //Deliberate as date updated could skip some updates post full sync syncParams.CandidateLastUpdatedDate = syncParams.CandidateLastCreatedDate; } if (syncType == SyncType.PartialByDateCreated) { syncParams.CandidateLastCreatedDate = maxDateCreated > syncParams.CandidateLastCreatedDate ? maxDateCreated : syncParams.CandidateLastCreatedDate; } if (syncType == SyncType.PartialByDateUpdated) { syncParams.CandidateLastUpdatedDate = maxDateUpdated > syncParams.CandidateLastUpdatedDate ? maxDateUpdated : syncParams.CandidateLastUpdatedDate; } _syncRepository.SetCandidateSyncParams(syncParams); var percentage = ((double)count / expectedCount) * 100; _logService.Info($"Processed batch of {candidatesWithHistory.Count} candidates and {count} candidates out of {expectedCount} in total. {Math.Round(percentage, 2)}% complete. LastCreatedDate: {syncParams.CandidateLastCreatedDate} LastUpdatedDate: {syncParams.CandidateLastUpdatedDate}"); } }
public void Update(Guid candidateGuid) { var candidateUser = GetCandidateUser(candidateGuid); var candidateSummaries = _candidateRepository.GetCandidateSummariesByGuid(new[] { candidateUser.Candidate.Id }); var schoolAttendedIds = _schoolAttendedRepository.GetSchoolAttendedIdsByCandidateIds(candidateSummaries.Values.Select(cs => cs.CandidateId)); var candidateHistoryIds = _candidateHistoryRepository.GetCandidateHistoryIdsByCandidateIds(candidateSummaries.Values.Select(cs => cs.CandidateId).Distinct()); var candidateWithHistory = _candidateMappers.MapCandidateWithHistory(candidateUser, candidateSummaries, _vacancyLocalAuthorities.Value, _localAuthorityCountyIds.Value, schoolAttendedIds, candidateHistoryIds, _anonymiseData); Update(candidateWithHistory); }