private static bool IsFinished(OrchestrationRuntimeStatus status)
{
return(status == OrchestrationRuntimeStatus.Canceled ||
status == OrchestrationRuntimeStatus.Failed ||
status == OrchestrationRuntimeStatus.Terminated ||
status == OrchestrationRuntimeStatus.Completed);
}
public async Task RunActivity_RetriesForCompleted(
[Values(OrchestrationRuntimeStatus.Canceled, OrchestrationRuntimeStatus.Failed, OrchestrationRuntimeStatus.Terminated, OrchestrationRuntimeStatus.Completed)] OrchestrationRuntimeStatus runtimeStatus)
{
var data = CreatePushData("test", "test");
var instanceId = CreateInstanceId(data);
_mockDurableClient.CreateCheckStatusResponse(Arg.Any <HttpRequestMessage>(), instanceId).Returns(new HttpResponseMessage
{
StatusCode = HttpStatusCode.OK,
Content = new StringContent(string.Empty)
});
var status = new DurableOrchestrationStatus
{
RuntimeStatus = runtimeStatus
};
_mockDurableClient.GetStatusAsync(instanceId).Returns(Task.FromResult(status));
var request = new HttpRequestMessage()
{
Content = new StringContent(JsonConvert.SerializeObject(data), System.Text.Encoding.UTF8, "application/json"),
};
var result = await RepositoryValidatorEndpoint.RepositoryValidatorTrigger(request, _mockDurableClient, Substitute.For <ILogger>());
Assert.AreEqual(result.StatusCode, HttpStatusCode.OK);
await _mockDurableClient.Received().StartNewAsync(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <object>());
}
private static async Task <IEnumerable <Instance> > GetInstances(
DurableOrchestrationClient client, OrchestrationRuntimeStatus runtimeStatus
, ILogger log)
{
var currentDate = DateTime.UtcNow;
IEnumerable <OrchestrationRuntimeStatus> runningInstanceStatus = new List <OrchestrationRuntimeStatus> {
OrchestrationRuntimeStatus.Running
};
IEnumerable <DurableOrchestrationStatus> instances =
await client.GetStatusAsync(currentDate.AddHours(-4), currentDate, runningInstanceStatus);
List <Instance> engineInstances = new List <Instance>();
foreach (var instance in instances)
{
engineInstances.Add(new Instance()
{
InstanceId = instance.InstanceId
, State =
JsonConvert.DeserializeObject <CustomState>(instance.CustomStatus.ToString())
});
}
;
return(engineInstances);
}
private async Task CheckRuntimeStatus(string instanceId, OrchestrationRuntimeStatus runtimeStatus, HttpStatusCode httpStatusCode = HttpStatusCode.OK)
{
var httpApiHandler = new HttpApiHandler(new DurableTaskExtensionMock()
{
NotificationUrl = new Uri(TestConstants.NotificationUrl)
}, null);
var httpResponseMessage = await httpApiHandler.WaitForCompletionOrCreateCheckStatusResponseAsync(
new HttpRequestMessage
{
RequestUri = new Uri(TestConstants.RequestUri),
},
instanceId,
new OrchestrationClientAttribute
{
TaskHub = TestConstants.TaskHub,
ConnectionName = TestConstants.ConnectionName,
},
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(8));
Assert.Equal(httpResponseMessage.StatusCode, httpStatusCode);
var content = await httpResponseMessage.Content.ReadAsStringAsync();
var response = JsonConvert.DeserializeObject <JObject>(content);
Assert.Equal(response["runtimeStatus"], runtimeStatus.ToString());
}
private EventGridEvent[] CreateEventGridEvent(
string hubName,
string functionName,
string instanceId,
string reason,
OrchestrationRuntimeStatus orchestrationRuntimeStatus)
{
return(new[]
{
new EventGridEvent
{
Id = Guid.NewGuid().ToString(),
EventType = "orchestratorEvent",
Subject = $"durable/orchestrator/{orchestrationRuntimeStatus}",
EventTime = DateTime.UtcNow,
Data = new EventGridPayload
{
HubName = hubName,
FunctionName = functionName,
InstanceId = instanceId,
Reason = reason,
EventType = orchestrationRuntimeStatus,
},
DataVersion = "1.0",
},
});
}
public static async Task <bool> TryStartSingletonAsync(this IDurableOrchestrationClient starter, string orchestrationName, string instanceId, object input)
{
var inactiveStatuses = new OrchestrationRuntimeStatus[]
{
OrchestrationRuntimeStatus.Terminated,
OrchestrationRuntimeStatus.Failed,
OrchestrationRuntimeStatus.Canceled,
OrchestrationRuntimeStatus.Completed
};
var instance = await starter.GetStatusAsync(instanceId);
if (instance == null || inactiveStatuses.Contains(instance.RuntimeStatus))
{
await starter.StartNewAsync(orchestrationName, instanceId, input);
return(true);
}
else
{
return(false);
}
}
public Task <DurableOrchestrationStatus> WaitForOrchestrationToReachStatus(string instanceId, OrchestrationRuntimeStatus desiredStatus, TimeSpan?timeout = null) => WaitForOrchestrationToReachStatus(instanceId, new[] { desiredStatus }, timeout);
public static bool IsUnknown(this OrchestrationRuntimeStatus status) => status == OrchestrationRuntimeStatus.Unknown;
public static bool IsFinal(this OrchestrationRuntimeStatus status) => status == OrchestrationRuntimeStatus.Canceled || status == OrchestrationRuntimeStatus.Completed || status == OrchestrationRuntimeStatus.Failed || status == OrchestrationRuntimeStatus.Terminated;
public static bool IsActive(this OrchestrationRuntimeStatus status) => status == OrchestrationRuntimeStatus.ContinuedAsNew || status == OrchestrationRuntimeStatus.Pending || status == OrchestrationRuntimeStatus.Running;
public static bool OrchestrationIsRunning(this OrchestrationRuntimeStatus status) => status == OrchestrationRuntimeStatus.Running || status == OrchestrationRuntimeStatus.Pending;
public override async Task <string> Execute(OrchestrationContext innerContext, string serializedInput)
{
#if !FUNCTIONS_V1
// Adding "Tags" to activity allows using App Insights to query current state of entities
var activity = Activity.Current;
OrchestrationRuntimeStatus status = OrchestrationRuntimeStatus.Running;
DurableTaskExtension.TagActivityWithOrchestrationStatus(status, this.context.InstanceId, true);
#endif
if (this.operationBatch.Count == 0 &&
this.lockRequest == null &&
(this.toBeRescheduled == null || this.toBeRescheduled.Count == 0))
{
// we are idle after a ContinueAsNew - the batch is empty.
// Wait for more messages to get here (via extended sessions)
await this.doneProcessingMessages.Task;
}
if (!this.messageDataConverter.IsDefault)
{
innerContext.MessageDataConverter = this.messageDataConverter;
}
if (!this.errorDataConverter.IsDefault)
{
innerContext.ErrorDataConverter = this.errorDataConverter;
}
this.Config.TraceHelper.FunctionStarting(
this.context.HubName,
this.context.Name,
this.context.InstanceId,
this.Config.GetIntputOutputTrace(serializedInput),
FunctionType.Entity,
isReplay: false);
if (this.NumberEventsToReceive > 0)
{
await this.doneProcessingMessages.Task;
}
// Commit the effects of this batch, if
// we have not already run into an internal error
// (in which case we will abort the batch instead of committing it)
if (this.context.InternalError == null)
{
bool writeBackSuccessful = true;
ResponseMessage serializationErrorMessage = null;
if (this.RollbackFailedOperations)
{
// the state has already been written back, since it is
// done right after each operation.
}
else
{
// we are writing back the state here, after executing
// the entire batch of operations.
writeBackSuccessful = this.context.TryWriteback(out serializationErrorMessage);
}
// Reschedule all signals that were received before their time
this.context.RescheduleMessages(innerContext, this.toBeRescheduled);
// Send all buffered outgoing messages
this.context.SendOutbox(innerContext, writeBackSuccessful, serializationErrorMessage);
var jstate = JToken.FromObject(this.context.State);
// continue as new
innerContext.ContinueAsNew(jstate);
}
if (this.context.ErrorsPresent(out var description))
{
this.Config.TraceHelper.FunctionFailed(
this.context.HubName,
this.context.Name,
this.context.InstanceId,
description,
functionType: FunctionType.Entity,
isReplay: false);
}
else
{
this.Config.TraceHelper.FunctionCompleted(
this.context.HubName,
this.context.Name,
this.context.InstanceId,
this.Config.GetIntputOutputTrace(this.context.State.EntityState),
continuedAsNew: true,
functionType: FunctionType.Entity,
isReplay: false);
}
// The return value is not used.
return(string.Empty);
}
public async Task <DurableOrchestrationStatus> WaitForOrchestrationToReachStatus(string instanceId, OrchestrationRuntimeStatus desiredStatus, TimeSpan?timeout = null)
{
if (!_instances.TryGetValue(instanceId, out var context))
{
throw new Exception("WaitForOrchestrationToReachStatus found no instance with id " + instanceId);
}
var totalWait = TimeSpan.Zero;
var maxWait = timeout ?? DefaultTimeout;
TimeSpan wait = TimeSpan.FromMilliseconds(10);
while (context.Status != desiredStatus)
{
await Task.Delay(10);
totalWait = totalWait.Add(wait);
if (totalWait > maxWait)
{
throw new TimeoutException("WaitForOrchestrationToReachStatus exceeded max wait time of " + maxWait);
}
}
return(ToStatusObject(new KeyValuePair <string, InMemoryOrchestrationContext>(instanceId, context)));
}
public async Task RunAsync(
[ServiceBusTrigger("%membershipQueueName%", Connection = "differenceQueueConnection", IsSessionsEnabled = true)] Message message,
[DurableClient] IDurableOrchestrationClient starter, IMessageSession messageSession)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = nameof(StarterFunction) + " function started" });
var messageDetails = _messageService.GetMessageProperties(message);
var graphRequest = new GraphUpdaterFunctionRequest()
{
Message = Encoding.UTF8.GetString(messageDetails.Body),
MessageSessionId = messageDetails.SessionId,
MessageLockToken = messageDetails.LockToken
};
var groupMembership = JsonConvert.DeserializeObject <GroupMembership>(graphRequest.Message);
SetSessionTracker(messageDetails, groupMembership);
var source = new CancellationTokenSource();
var renew = RenewMessages(starter, messageSession, source, messageDetails.MessageId);
var instanceId = await starter.StartNewAsync(nameof(OrchestratorFunction), graphRequest);
var completedGroupMembershipMessages = default(List <GroupMembershipMessage>);
var isLastMessage = false;
var orchestratorRuntimeStatusCodesWorthRetrying = new OrchestrationRuntimeStatus[]
{
OrchestrationRuntimeStatus.ContinuedAsNew,
OrchestrationRuntimeStatus.Running,
OrchestrationRuntimeStatus.Pending
};
var result = default(DurableOrchestrationStatus);
/*Understanding the Retry policy
* We have a lot of sub-second sync execution so the first query would ensure we cater to those queries
* We also have a lot of syncs that take less than 10 seconds. Having a exponetial backoff 1.25^1 would mean we would be waiting 90 seconds per sync instead of 10 seconds.
* Hence the logic to ensure retryAttempt 1 is done after 10 seconds. Following this we go back to the exponetial backoff.
*/
var retryPolicy = Policy
.HandleResult <DurableOrchestrationStatus>(status => orchestratorRuntimeStatusCodesWorthRetrying.Contains(status.RuntimeStatus))
.WaitAndRetryAsync(
MAX_RETRY_ATTEMPTS,
retryAttempt =>
{
if (retryAttempt == 1)
{
return(TimeSpan.FromSeconds(FIRST_RETRY_DELAY_IN_SECONDS));
}
else
{
return(TimeSpan.FromMinutes(Math.Pow(1.25, retryAttempt - 1)));
}
}
);
await retryPolicy.ExecuteAsync(async() =>
{
result = await starter.GetStatusAsync(instanceId);
return(result);
});
if (result.RuntimeStatus == OrchestrationRuntimeStatus.Failed || result.RuntimeStatus == OrchestrationRuntimeStatus.Terminated)
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Error: Status of instance {result.InstanceId} is {result.RuntimeStatus}. The error message is : {result.Output}" });
// stop renewing the message session
source.Cancel();
}
else
{
await _loggingRepository.LogMessageAsync(new LogMessage { Message = $"Instance processing completed for {result.InstanceId}" });
var orchestratorResponseOutput = JsonConvert.DeserializeObject <GroupMembershipMessageResponse>(result.Output.ToString());
completedGroupMembershipMessages = orchestratorResponseOutput.CompletedGroupMembershipMessages;
isLastMessage = orchestratorResponseOutput.ShouldCompleteMessage;
}
if (isLastMessage)
{
var completedLockTokens = completedGroupMembershipMessages.Select(x => x.LockToken);
await messageSession.CompleteAsync(completedLockTokens);
await messageSession.CloseAsync();
source.Cancel();
}
await _loggingRepository.LogMessageAsync(new LogMessage { Message = nameof(StarterFunction) + " function completed" });
}
public static async Task <HttpResponseMessage> HttpStart(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequestMessage req,
[Table("RunningInstances")] CloudTable tableInput,
[Table("RunningInstances")] IAsyncCollector <InstanceTableEntity> tableOutput,
[OrchestrationClient] DurableOrchestrationClient starter,
ILogger log)
{
string requestBody = await req.Content.ReadAsStringAsync();
try
{
Update update = JsonConvert.DeserializeObject <Update>(requestBody);
string chatId = update.GetChatId();
TableQuery <InstanceTableEntity> sessionQuery = new TableQuery <InstanceTableEntity>().Where(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, chatId));
TableQuerySegment <InstanceTableEntity> sessions = await tableInput.ExecuteQuerySegmentedAsync(sessionQuery, null);
if ((update.Message?.Entities?.Length ?? 0) != 0 &&
(update.Message?.Entities?.First().Type ?? MessageEntityType.Unknown) == MessageEntityType.BotCommand &&
(update.Message?.EntityValues?.First()?.ToLowerInvariant().Equals("/cancel") ?? false))
{
//Cancel current user session and reset all state!
foreach (var entity in sessions)
{
await tableInput.ExecuteAsync(TableOperation.Delete(entity));
}
await botClient.SendTextMessageAsync(
chatId : chatId,
text : $"All running ({sessions.Count()}) sessions have now been purged");
}
else
{
bool didContinueASession = false;
if (sessions.Count() > 0)
{
foreach (InstanceTableEntity session in sessions)
{
OrchestrationRuntimeStatus status = (await starter.GetStatusAsync(session.RowKey)).RuntimeStatus;
if (status == OrchestrationRuntimeStatus.Failed ||
status == OrchestrationRuntimeStatus.Canceled ||
status == OrchestrationRuntimeStatus.Completed ||
status == OrchestrationRuntimeStatus.Terminated ||
status == OrchestrationRuntimeStatus.Unknown)
{
log.LogInformation($"Removing session with ID = '{session.RowKey}' because it was terminated");
await tableInput.ExecuteAsync(TableOperation.Delete(session));
await botClient.SendTextMessageAsync(
chatId : chatId,
text : $"FYI: I purged a old session with ID {session.RowKey} that had status: {status.ToString()} (Was started at UTC: {session.Timestamp.UtcDateTime.ToShortDateString()})");
}
else
{
if (update.CallbackQuery != null)
{
//One should not have more than 1 running session, but I will not crash because of it. So just continue all the session until they get cleaned up.
await starter.RaiseEventAsync(session.RowKey, "Callback", update);
log.LogInformation($"Continuing on session with ID = '{session.RowKey}'");
didContinueASession = true;
}
}
}
}
if (!didContinueASession)
{
var newInstance = new InstanceTableEntity
{
PartitionKey = chatId,
RowKey = await starter.StartNewAsync("BotBrainFunction", update)
};
await tableOutput.AddAsync(newInstance);
log.LogInformation($"Started orchestration with ID = '{newInstance.RowKey}'.");
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Could not parse update data. {ex.Message}");
log.LogInformation(requestBody);
}
// Function input comes from the request content.
//string instanceId = await starter.StartNewAsync("Function2", null);
//log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
//return starter.CreateCheckStatusResponse(req, instanceId);
HttpResponseMessage response = req.CreateResponse(HttpStatusCode.OK);
//await Task.Delay(1);
return(response);
}
