示例#1
0
        public async Task OrchestrationHang()
        {
            // This test needs to wait for the lock timeout to expire after the injected fault
            // before it will retry the checkpoint.
            this.testService.OrchestrationServiceOptions.WorkItemLockTimeout = TimeSpan.FromSeconds(5);

            var resumeSignal = new ManualResetEvent(initialState: false);

            // Set up a mock that will hang on the first call, but succeed the second call
            bool firstCall = true;

            this.testService.OrchestrationServiceMock.Setup(
                svc => svc.CompleteTaskOrchestrationWorkItemAsync(
                    It.IsAny <TaskOrchestrationWorkItem>(),
                    It.IsAny <OrchestrationRuntimeState>(),
                    It.IsAny <IList <TaskMessage> >(),
                    It.IsAny <IList <TaskMessage> >(),
                    It.IsAny <IList <TaskMessage> >(),
                    It.IsAny <TaskMessage>(),
                    It.IsAny <OrchestrationState>()))
            .Callback(() =>
            {
                if (firstCall)
                {
                    firstCall = false;
                    resumeSignal.WaitOne();
                }
            });

            // Block background work-item renewal to ensure another thread tries to execute the orchestration work item
            this.testService.OrchestrationServiceMock
            .Setup(svc => svc.ReleaseTaskOrchestrationWorkItemAsync(It.IsAny <TaskOrchestrationWorkItem>()))
            .Callback(() => resumeSignal.WaitOne());

            // Does nothing except return the original input
            string orchestrationName       = "EmptyOrchestration";
            TestInstance <string> instance = await this.testService.RunOrchestration(
                input : "Hello, world!",
                orchestrationName,
                implementation : (ctx, input) => Task.FromResult(input));

            // Give the orchestration extra time to finish. Time is needed for the lock timeout
            // to expire.
            await instance.WaitForCompletion(TimeSpan.FromSeconds(30));

            // Unblock the hung thread. This should result in a failure due to a detected duplicate execution.
            resumeSignal.Set();

            // Give the unblocked thread time to execute its logic and fail.
            await Task.Delay(TimeSpan.FromSeconds(3));

            // Verify that the CompleteTaskOrchestrationWorkItemAsync method was called exactly twice, and that the second call failed
            this.testService.OrchestrationServiceMock.Verify(svc => svc.CompleteTaskOrchestrationWorkItemAsync(
                                                                 It.IsAny <TaskOrchestrationWorkItem>(),
                                                                 It.IsAny <OrchestrationRuntimeState>(),
                                                                 It.IsAny <IList <TaskMessage> >(),
                                                                 It.IsAny <IList <TaskMessage> >(),
                                                                 It.IsAny <IList <TaskMessage> >(),
                                                                 It.IsAny <TaskMessage>(),
                                                                 It.IsAny <OrchestrationState>()), Times.Exactly(2));

            // Check logs to confirm that a duplicate execution was detected
            LogAssert.Contains(
                this.testService.LogProvider,
                LogAssert.CheckpointStarting(orchestrationName),
                LogAssert.CheckpointCompleted(orchestrationName),
                LogAssert.CheckpointStarting(orchestrationName),
                LogAssert.DuplicateExecutionDetected(orchestrationName));
        }
示例#2
0
        public async Task ActivityHang()
        {
            // This test needs to wait for the lock timeout to expire after the injected fault
            // before it will retry the checkpoint.
            this.testService.OrchestrationServiceOptions.WorkItemLockTimeout = TimeSpan.FromSeconds(5);

            var secondCallSignal = new ManualResetEvent(initialState: false);
            var resumeSignal     = new ManualResetEvent(initialState: false);

            // Set up a mock that will hang on the first call, but succeed the second call
            bool firstCall = true;

            this.testService.OrchestrationServiceMock.Setup(
                svc => svc.CompleteTaskActivityWorkItemAsync(
                    It.IsAny <TaskActivityWorkItem>(),
                    It.IsAny <TaskMessage>()))
            .Callback(() =>
            {
                if (firstCall)
                {
                    firstCall = false;
                    resumeSignal.WaitOne();
                }
                else
                {
                    secondCallSignal.Set();
                }
            });

            // Block background activity renewal to ensure another thread tries to execute the activity work item
            this.testService.OrchestrationServiceMock
            .Setup(svc => svc.RenewTaskActivityWorkItemLockAsync(It.IsAny <TaskActivityWorkItem>()))
            .Callback(() => resumeSignal.WaitOne());

            // Run an orchestration with a single activity function call
            string orchestrationName       = "OrchestrationWithActivity";
            string input                   = $"[{DateTime.UtcNow:o}]";
            TestInstance <string> instance = await this.testService.RunOrchestration(
                input,
                orchestrationName,
                implementation : async(ctx, input) =>
            {
                string result = await ctx.ScheduleTask <string>("SayHello", "", input);
                await Task.Delay(Timeout.Infinite);     // simulate waiting for an external event
                return(result);
            },
                activities : ("SayHello", TestService.MakeActivity((TaskContext ctx, string input) => $"Hello, {input}!")));

            // Give the orchestration extra time to finish. Time is needed for the lock timeout
            // to expire.
            await instance.WaitForStart(TimeSpan.FromSeconds(10));

            bool resumed = secondCallSignal.WaitOne(TimeSpan.FromSeconds(10));

            Assert.True(resumed);
            Assert.False(firstCall);

            // Give the retrying thread time to checkpoint its progress
            await Task.Delay(TimeSpan.FromSeconds(3));

            // Unblock the hung thread. This should result in a failure due to a detected duplicate execution.
            resumeSignal.Set();

            // Give the unblocked thread time to execute its logic and fail.
            await Task.Delay(TimeSpan.FromSeconds(3));

            // Verify that the CompleteTaskActivityWorkItemAsync method was called exactly twice, and that the second call failed
            this.testService.OrchestrationServiceMock.Verify(
                svc => svc.CompleteTaskActivityWorkItemAsync(
                    It.IsAny <TaskActivityWorkItem>(),
                    It.IsAny <TaskMessage>()),
                Times.Exactly(2));

            // Check logs to confirm that a duplicate execution was detected
            LogAssert.Contains(
                this.testService.LogProvider,
                LogAssert.CheckpointStarting(orchestrationName),
                LogAssert.CheckpointCompleted(orchestrationName),
                LogAssert.CheckpointStarting(orchestrationName),
                LogAssert.CheckpointCompleted(orchestrationName),
                LogAssert.DuplicateExecutionDetected("SayHello"));
        }