public async Task OrchestrationHang()
{
// This test needs to wait for the lock timeout to expire after the injected fault
// before it will retry the checkpoint.
this.testService.OrchestrationServiceOptions.WorkItemLockTimeout = TimeSpan.FromSeconds(5);
var resumeSignal = new ManualResetEvent(initialState: false);
// Set up a mock that will hang on the first call, but succeed the second call
bool firstCall = true;
this.testService.OrchestrationServiceMock.Setup(
svc => svc.CompleteTaskOrchestrationWorkItemAsync(
It.IsAny <TaskOrchestrationWorkItem>(),
It.IsAny <OrchestrationRuntimeState>(),
It.IsAny <IList <TaskMessage> >(),
It.IsAny <IList <TaskMessage> >(),
It.IsAny <IList <TaskMessage> >(),
It.IsAny <TaskMessage>(),
It.IsAny <OrchestrationState>()))
.Callback(() =>
{
if (firstCall)
{
firstCall = false;
resumeSignal.WaitOne();
}
});
// Block background work-item renewal to ensure another thread tries to execute the orchestration work item
this.testService.OrchestrationServiceMock
.Setup(svc => svc.ReleaseTaskOrchestrationWorkItemAsync(It.IsAny <TaskOrchestrationWorkItem>()))
.Callback(() => resumeSignal.WaitOne());
// Does nothing except return the original input
string orchestrationName = "EmptyOrchestration";
TestInstance <string> instance = await this.testService.RunOrchestration(
input : "Hello, world!",
orchestrationName,
implementation : (ctx, input) => Task.FromResult(input));
// Give the orchestration extra time to finish. Time is needed for the lock timeout
// to expire.
await instance.WaitForCompletion(TimeSpan.FromSeconds(30));
// Unblock the hung thread. This should result in a failure due to a detected duplicate execution.
resumeSignal.Set();
// Give the unblocked thread time to execute its logic and fail.
await Task.Delay(TimeSpan.FromSeconds(3));
// Verify that the CompleteTaskOrchestrationWorkItemAsync method was called exactly twice, and that the second call failed
this.testService.OrchestrationServiceMock.Verify(svc => svc.CompleteTaskOrchestrationWorkItemAsync(
It.IsAny <TaskOrchestrationWorkItem>(),
It.IsAny <OrchestrationRuntimeState>(),
It.IsAny <IList <TaskMessage> >(),
It.IsAny <IList <TaskMessage> >(),
It.IsAny <IList <TaskMessage> >(),
It.IsAny <TaskMessage>(),
It.IsAny <OrchestrationState>()), Times.Exactly(2));
// Check logs to confirm that a duplicate execution was detected
LogAssert.Contains(
this.testService.LogProvider,
LogAssert.CheckpointStarting(orchestrationName),
LogAssert.CheckpointCompleted(orchestrationName),
LogAssert.CheckpointStarting(orchestrationName),
LogAssert.DuplicateExecutionDetected(orchestrationName));
}
public async Task ActivityHang()
{
// This test needs to wait for the lock timeout to expire after the injected fault
// before it will retry the checkpoint.
this.testService.OrchestrationServiceOptions.WorkItemLockTimeout = TimeSpan.FromSeconds(5);
var secondCallSignal = new ManualResetEvent(initialState: false);
var resumeSignal = new ManualResetEvent(initialState: false);
// Set up a mock that will hang on the first call, but succeed the second call
bool firstCall = true;
this.testService.OrchestrationServiceMock.Setup(
svc => svc.CompleteTaskActivityWorkItemAsync(
It.IsAny <TaskActivityWorkItem>(),
It.IsAny <TaskMessage>()))
.Callback(() =>
{
if (firstCall)
{
firstCall = false;
resumeSignal.WaitOne();
}
else
{
secondCallSignal.Set();
}
});
// Block background activity renewal to ensure another thread tries to execute the activity work item
this.testService.OrchestrationServiceMock
.Setup(svc => svc.RenewTaskActivityWorkItemLockAsync(It.IsAny <TaskActivityWorkItem>()))
.Callback(() => resumeSignal.WaitOne());
// Run an orchestration with a single activity function call
string orchestrationName = "OrchestrationWithActivity";
string input = $"[{DateTime.UtcNow:o}]";
TestInstance <string> instance = await this.testService.RunOrchestration(
input,
orchestrationName,
implementation : async(ctx, input) =>
{
string result = await ctx.ScheduleTask <string>("SayHello", "", input);
await Task.Delay(Timeout.Infinite); // simulate waiting for an external event
return(result);
},
activities : ("SayHello", TestService.MakeActivity((TaskContext ctx, string input) => $"Hello, {input}!")));
// Give the orchestration extra time to finish. Time is needed for the lock timeout
// to expire.
await instance.WaitForStart(TimeSpan.FromSeconds(10));
bool resumed = secondCallSignal.WaitOne(TimeSpan.FromSeconds(10));
Assert.True(resumed);
Assert.False(firstCall);
// Give the retrying thread time to checkpoint its progress
await Task.Delay(TimeSpan.FromSeconds(3));
// Unblock the hung thread. This should result in a failure due to a detected duplicate execution.
resumeSignal.Set();
// Give the unblocked thread time to execute its logic and fail.
await Task.Delay(TimeSpan.FromSeconds(3));
// Verify that the CompleteTaskActivityWorkItemAsync method was called exactly twice, and that the second call failed
this.testService.OrchestrationServiceMock.Verify(
svc => svc.CompleteTaskActivityWorkItemAsync(
It.IsAny <TaskActivityWorkItem>(),
It.IsAny <TaskMessage>()),
Times.Exactly(2));
// Check logs to confirm that a duplicate execution was detected
LogAssert.Contains(
this.testService.LogProvider,
LogAssert.CheckpointStarting(orchestrationName),
LogAssert.CheckpointCompleted(orchestrationName),
LogAssert.CheckpointStarting(orchestrationName),
LogAssert.CheckpointCompleted(orchestrationName),
LogAssert.DuplicateExecutionDetected("SayHello"));
}