public async Task <GroupMembershipMessageResponse> CollectMessagesAsync([ActivityTrigger] GraphUpdaterFunctionRequest request)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = nameof(MessageCollectorFunction) + " function started", RunId = request.RunId });
var body = new GroupMembershipMessage
{
Body = JsonConvert.DeserializeObject <GroupMembership>(request.Message),
LockToken = request.MessageLockToken,
IsCancelationMessage = request.IsCancelationRequest
};
var handleNewMessageResult = await _messageCollector.HandleNewMessageAsync(body, request.MessageSessionId);
await _loggingRepository.LogMessageAsync(new LogMessage { Message = nameof(MessageCollectorFunction) + " function completed", RunId = request.RunId });
return(handleNewMessageResult);
}
private async Task <SessionTrackerAction> TrackLastMessageTimeout(IDurableOrchestrationClient starter, IMessageSession messageSession, string messageId)
{
if (_sessionsTracker.TryGetValue(messageSession.SessionId, out var sessionTracker))
{
if (sessionTracker.ReceivedLastMessage)
{
return(SessionTrackerAction.Continue);
}
if (messageId != sessionTracker.LatestMessageId)
{
return(SessionTrackerAction.Stop);
}
int.TryParse(_configuration["GraphUpdater:LastMessageWaitTimeout"], out int timeOut);
timeOut = timeOut == 0 ? 10 : timeOut;
var elapsedTime = DateTime.UtcNow - sessionTracker.LastAccessTime;
if (elapsedTime.TotalMinutes >= timeOut)
{
_sessionsTracker.Remove(messageSession.SessionId, out _);
await messageSession.CompleteAsync(sessionTracker.LockTokens);
await messageSession.CloseAsync();
var cancelationRequest = new GraphUpdaterFunctionRequest
{
IsCancelationRequest = true,
MessageSessionId = messageSession.SessionId,
Message = JsonConvert.SerializeObject(new GroupMembership
{
SyncJobPartitionKey = sessionTracker.JobPartitionKey,
SyncJobRowKey = sessionTracker.JobRowKey,
RunId = sessionTracker.RunId
}),
};
await starter.StartNewAsync(nameof(OrchestratorFunction), cancelationRequest);
return(SessionTrackerAction.Stop);
}
}
return(SessionTrackerAction.Continue);
}
public async Task RunAsync(
[ServiceBusTrigger("%membershipQueueName%", Connection = "differenceQueueConnection", IsSessionsEnabled = true)] Message message,
[DurableClient] IDurableOrchestrationClient starter, IMessageSession messageSession)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = nameof(StarterFunction) + " function started" });
var messageDetails = _messageService.GetMessageProperties(message);
var graphRequest = new GraphUpdaterFunctionRequest()
{
Message = Encoding.UTF8.GetString(messageDetails.Body),
MessageSessionId = messageDetails.SessionId,
MessageLockToken = messageDetails.LockToken
};
var groupMembership = JsonConvert.DeserializeObject <GroupMembership>(graphRequest.Message);
SetSessionTracker(messageDetails, groupMembership);
var source = new CancellationTokenSource();
var renew = RenewMessages(starter, messageSession, source, messageDetails.MessageId);
var instanceId = await starter.StartNewAsync(nameof(OrchestratorFunction), graphRequest);
var completedGroupMembershipMessages = default(List <GroupMembershipMessage>);
var isLastMessage = false;
var orchestratorRuntimeStatusCodesWorthRetrying = new OrchestrationRuntimeStatus[]
{
OrchestrationRuntimeStatus.ContinuedAsNew,
OrchestrationRuntimeStatus.Running,
OrchestrationRuntimeStatus.Pending
};
var result = default(DurableOrchestrationStatus);
/*Understanding the Retry policy
* We have a lot of sub-second sync execution so the first query would ensure we cater to those queries
* We also have a lot of syncs that take less than 10 seconds. Having a exponetial backoff 1.25^1 would mean we would be waiting 90 seconds per sync instead of 10 seconds.
* Hence the logic to ensure retryAttempt 1 is done after 10 seconds. Following this we go back to the exponetial backoff.
*/
var retryPolicy = Policy
.HandleResult <DurableOrchestrationStatus>(status => orchestratorRuntimeStatusCodesWorthRetrying.Contains(status.RuntimeStatus))
.WaitAndRetryAsync(
MAX_RETRY_ATTEMPTS,
retryAttempt =>
{
if (retryAttempt == 1)
{
return(TimeSpan.FromSeconds(FIRST_RETRY_DELAY_IN_SECONDS));
}
else
{
return(TimeSpan.FromMinutes(Math.Pow(1.25, retryAttempt - 1)));
}
}
);
await retryPolicy.ExecuteAsync(async() =>
{
result = await starter.GetStatusAsync(instanceId);
return(result);
});
if (result.RuntimeStatus == OrchestrationRuntimeStatus.Failed || result.RuntimeStatus == OrchestrationRuntimeStatus.Terminated)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Error: Status of instance {result.InstanceId} is {result.RuntimeStatus}. The error message is : {result.Output}" });
// stop renewing the message session
source.Cancel();
}
else
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Instance processing completed for {result.InstanceId}" });
var orchestratorResponseOutput = JsonConvert.DeserializeObject <GroupMembershipMessageResponse>(result.Output.ToString());
completedGroupMembershipMessages = orchestratorResponseOutput.CompletedGroupMembershipMessages;
isLastMessage = orchestratorResponseOutput.ShouldCompleteMessage;
}
if (isLastMessage)
{
var completedLockTokens = completedGroupMembershipMessages.Select(x => x.LockToken);
await messageSession.CompleteAsync(completedLockTokens);
await messageSession.CloseAsync();
source.Cancel();
}
await _loggingRepository.LogMessageAsync(new LogMessage { Message = nameof(StarterFunction) + " function completed" });
}
public async Task <GroupMembershipMessageResponse> RunOrchestratorAsync([OrchestrationTrigger] IDurableOrchestrationContext context)
{
GroupMembership groupMembership = null;
GraphUpdaterFunctionRequest graphRequest = null;
GroupMembershipMessageResponse messageResponse = null;
SyncJob syncJob = null;
graphRequest = context.GetInput <GraphUpdaterFunctionRequest>();
groupMembership = JsonConvert.DeserializeObject <GroupMembership>(graphRequest.Message);
graphRequest.RunId = groupMembership.RunId;
try
{
syncJob = await context.CallActivityAsync <SyncJob>(nameof(JobReaderFunction),
new JobReaderRequest
{
JobPartitionKey = groupMembership.SyncJobPartitionKey,
JobRowKey = groupMembership.SyncJobRowKey,
RunId = groupMembership.RunId
});
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = $"{nameof(OrchestratorFunction)} function started", SyncJob = syncJob });
messageResponse = await context.CallActivityAsync <GroupMembershipMessageResponse>(nameof(MessageCollectorFunction), graphRequest);
if (graphRequest.IsCancelationRequest)
{
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = $"Canceling session {graphRequest.MessageSessionId}", SyncJob = syncJob });
await context.CallActivityAsync(nameof(JobStatusUpdaterFunction),
CreateJobStatusUpdaterRequest(groupMembership.SyncJobPartitionKey, groupMembership.SyncJobRowKey,
SyncStatus.Error, groupMembership.MembershipObtainerDryRunEnabled, syncJob.ThresholdViolations, groupMembership.RunId));
return(messageResponse);
}
if (messageResponse.ShouldCompleteMessage)
{
var isValidGroup = await context.CallActivityAsync <bool>(nameof(GroupValidatorFunction),
new GroupValidatorRequest
{
RunId = groupMembership.RunId,
GroupId = groupMembership.Destination.ObjectId,
JobPartitionKey = groupMembership.SyncJobPartitionKey,
JobRowKey = groupMembership.SyncJobRowKey
});
if (!isValidGroup)
{
await context.CallActivityAsync(nameof(JobStatusUpdaterFunction),
CreateJobStatusUpdaterRequest(groupMembership.SyncJobPartitionKey, groupMembership.SyncJobRowKey,
SyncStatus.Error, groupMembership.MembershipObtainerDryRunEnabled, syncJob.ThresholdViolations, groupMembership.RunId));
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = $"{nameof(OrchestratorFunction)} function did not complete", SyncJob = syncJob });
return(messageResponse);
}
var destinationGroupMembers = await context.CallSubOrchestratorAsync <List <AzureADUser> >(nameof(UsersReaderSubOrchestratorFunction),
new UsersReaderRequest { SyncJob = syncJob });
var fullMembership = new GroupMembership
{
Destination = groupMembership.Destination,
IsLastMessage = groupMembership.IsLastMessage,
RunId = groupMembership.RunId,
SourceMembers = messageResponse.CompletedGroupMembershipMessages.SelectMany(x => x.Body.SourceMembers).ToList(),
SyncJobPartitionKey = groupMembership.SyncJobPartitionKey,
SyncJobRowKey = groupMembership.SyncJobRowKey
};
var deltaResponse = await context.CallActivityAsync <DeltaResponse>(nameof(DeltaCalculatorFunction),
new DeltaCalculatorRequest
{
RunId = groupMembership.RunId,
GroupMembership = fullMembership,
MembersFromDestinationGroup = destinationGroupMembers,
});
if (deltaResponse.GraphUpdaterStatus == GraphUpdaterStatus.Error ||
deltaResponse.GraphUpdaterStatus == GraphUpdaterStatus.ThresholdExceeded)
{
var updateRequest = CreateJobStatusUpdaterRequest(groupMembership.SyncJobPartitionKey, groupMembership.SyncJobRowKey,
deltaResponse.SyncStatus, groupMembership.MembershipObtainerDryRunEnabled, syncJob.ThresholdViolations, groupMembership.RunId);
if (deltaResponse.GraphUpdaterStatus == GraphUpdaterStatus.ThresholdExceeded)
{
updateRequest.ThresholdViolations++;
if (updateRequest.ThresholdViolations >= _thresholdConfig.NumberOfThresholdViolationsToDisableJob)
{
updateRequest.Status = SyncStatus.ThresholdExceeded;
}
}
await context.CallActivityAsync(nameof(JobStatusUpdaterFunction), updateRequest);
}
if (deltaResponse.GraphUpdaterStatus != GraphUpdaterStatus.Ok)
{
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = $"{nameof(OrchestratorFunction)} function did not complete", SyncJob = syncJob });
return(messageResponse);
}
if (!deltaResponse.IsDryRunSync)
{
await context.CallSubOrchestratorAsync <GraphUpdaterStatus>(nameof(GroupUpdaterSubOrchestratorFunction),
CreateGroupUpdaterRequest(syncJob, deltaResponse.MembersToAdd, RequestType.Add, deltaResponse.IsInitialSync));
await context.CallSubOrchestratorAsync <GraphUpdaterStatus>(nameof(GroupUpdaterSubOrchestratorFunction),
CreateGroupUpdaterRequest(syncJob, deltaResponse.MembersToRemove, RequestType.Remove, deltaResponse.IsInitialSync));
if (deltaResponse.IsInitialSync)
{
var groupName = await context.CallActivityAsync <string>(nameof(GroupNameReaderFunction),
new GroupNameReaderRequest { RunId = groupMembership.RunId, GroupId = groupMembership.Destination.ObjectId });
var additonalContent = new[]
{
groupName,
groupMembership.Destination.ObjectId.ToString(),
deltaResponse.MembersToAdd.Count.ToString(),
deltaResponse.MembersToRemove.Count.ToString(),
_emailSenderAndRecipients.SupportEmailAddresses
};
await context.CallActivityAsync(nameof(EmailSenderFunction),
new EmailSenderRequest
{
ToEmail = deltaResponse.Requestor,
CcEmail = _emailSenderAndRecipients.SyncCompletedCCAddresses,
ContentTemplate = SyncCompletedEmailBody,
AdditionalContentParams = additonalContent,
RunId = groupMembership.RunId
});
}
}
var message = GetUsersDataMessage(groupMembership.Destination.ObjectId, deltaResponse.MembersToAdd.Count, deltaResponse.MembersToRemove.Count);
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = message });
await context.CallActivityAsync(nameof(JobStatusUpdaterFunction),
CreateJobStatusUpdaterRequest(groupMembership.SyncJobPartitionKey, groupMembership.SyncJobRowKey,
SyncStatus.Idle, groupMembership.MembershipObtainerDryRunEnabled, 0, groupMembership.RunId));
var timeElapsedForJob = (context.CurrentUtcDateTime - deltaResponse.Timestamp).TotalSeconds;
_telemetryClient.TrackMetric(nameof(Metric.SyncJobTimeElapsedSeconds), timeElapsedForJob);
var syncCompleteEvent = new Dictionary <string, string>
{
{ nameof(SyncJob.TargetOfficeGroupId), groupMembership.Destination.ObjectId.ToString() },
{ nameof(SyncJob.Type), deltaResponse.SyncJobType },
{ "Result", deltaResponse.SyncStatus == SyncStatus.Idle ? "Success": "Failure" },
{ nameof(SyncJob.IsDryRunEnabled), deltaResponse.IsDryRunSync.ToString() },
{ nameof(Metric.SyncJobTimeElapsedSeconds), timeElapsedForJob.ToString() },
{ nameof(DeltaResponse.MembersToAdd), deltaResponse.MembersToAdd.Count.ToString() },
{ nameof(DeltaResponse.MembersToRemove), deltaResponse.MembersToRemove.Count.ToString() },
{ nameof(Metric.ProjectedMemberCount), fullMembership.SourceMembers.Count.ToString() }
};
_telemetryClient.TrackEvent(nameof(Metric.SyncComplete), syncCompleteEvent);
}
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = $"{nameof(OrchestratorFunction)} function completed", SyncJob = syncJob });
return(messageResponse);
}
catch (Exception ex)
{
await context.CallActivityAsync(nameof(LoggerFunction), new LoggerRequest { Message = $"Caught unexpected exception, marking sync job as errored. Exception:\n{ex}", SyncJob = syncJob });
if (groupMembership != null && !string.IsNullOrWhiteSpace(groupMembership.SyncJobPartitionKey) && !string.IsNullOrWhiteSpace(groupMembership.SyncJobRowKey))
{
await context.CallActivityAsync(nameof(JobStatusUpdaterFunction),
CreateJobStatusUpdaterRequest(groupMembership.SyncJobPartitionKey, groupMembership.SyncJobRowKey,
SyncStatus.Error, groupMembership.MembershipObtainerDryRunEnabled, syncJob.ThresholdViolations, groupMembership.RunId));
}
throw;
}
}
