internal void SetGateway(Gateway gateway)
{
this.gateway = gateway;
// Only start ClientRefreshTimer if this silo has a gateway.
// Need to start the timer in the system target context.
scheduler.QueueAction(Start, this.SchedulingContext).Ignore();
}
private void InjectDependencies()
{
healthCheckParticipants.Add(membershipOracle);
catalog.SiloStatusOracle = LocalSiloStatusOracle;
localGrainDirectory.CatalogSiloStatusListener = catalog;
LocalSiloStatusOracle.SubscribeToSiloStatusEvents(localGrainDirectory);
messageCenter.SiloDeadOracle = LocalSiloStatusOracle.IsDeadSilo;
// consistentRingProvider is not a system target per say, but it behaves like the localGrainDirectory, so it is here
LocalSiloStatusOracle.SubscribeToSiloStatusEvents((ISiloStatusListener)RingProvider);
LocalSiloStatusOracle.SubscribeToSiloStatusEvents(DeploymentLoadPublisher.Instance);
// start the reminder service system target
reminderService = reminderFactory.CreateReminderService(this).WithTimeout(initTimeout).Result;
RegisterSystemTarget((SystemTarget)reminderService);
RegisterSystemTarget(catalog);
scheduler.QueueAction(catalog.Start, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
// SystemTarget for provider init calls
providerManagerSystemTarget = new ProviderManagerSystemTarget(this);
RegisterSystemTarget(providerManagerSystemTarget);
}
internal void ProcessRequestsToInvalidActivation(
List <Message> messages,
ActivationAddress oldAddress,
ActivationAddress forwardingAddress,
string failedOperation,
Exception exc = null,
bool rejectMessages = false)
{
// IMPORTANT: do not do anything on activation context anymore, since this activation is invalid already.
scheduler.QueueAction(
() =>
{
foreach (var message in messages)
{
if (rejectMessages)
{
RejectMessage(message, Message.RejectionTypes.Transient, exc, failedOperation);
}
else
{
this.messagingTrace.OnDispatcherForwardingMultiple(messages.Count, oldAddress, forwardingAddress, failedOperation, exc);
TryForwardRequest(message, oldAddress, forwardingAddress, failedOperation, exc);
}
}
},
catalog);
}
private async Task InjectDependencies()
{
catalog.SiloStatusOracle = this.siloStatusOracle;
this.siloStatusOracle.SubscribeToSiloStatusEvents(localGrainDirectory);
// consistentRingProvider is not a system target per say, but it behaves like the localGrainDirectory, so it is here
this.siloStatusOracle.SubscribeToSiloStatusEvents((ISiloStatusListener)RingProvider);
this.siloStatusOracle.SubscribeToSiloStatusEvents(typeManager);
this.siloStatusOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService <DeploymentLoadPublisher>());
this.siloStatusOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService <ClientObserverRegistrar>());
var reminderTable = Services.GetService <IReminderTable>();
if (reminderTable != null)
{
logger.Info($"Creating reminder grain service for type={reminderTable.GetType()}");
// Start the reminder service system target
reminderService = new LocalReminderService(this, reminderTable, this.initTimeout, this.loggerFactory);;
RegisterSystemTarget((SystemTarget)reminderService);
}
RegisterSystemTarget(catalog);
await scheduler.QueueAction(catalog.Start, catalog.SchedulingContext)
.WithTimeout(initTimeout, $"Starting Catalog failed due to timeout {initTimeout}");
// SystemTarget for provider init calls
this.fallbackScheduler = Services.GetRequiredService <FallbackSystemTarget>();
RegisterSystemTarget(fallbackScheduler);
}
private async Task InjectDependencies()
{
healthCheckParticipants.Add(membershipOracle);
catalog.SiloStatusOracle = this.membershipOracle;
this.membershipOracle.SubscribeToSiloStatusEvents(localGrainDirectory);
messageCenter.SiloDeadOracle = this.membershipOracle.IsDeadSilo;
// consistentRingProvider is not a system target per say, but it behaves like the localGrainDirectory, so it is here
this.membershipOracle.SubscribeToSiloStatusEvents((ISiloStatusListener)RingProvider);
this.membershipOracle.SubscribeToSiloStatusEvents(typeManager);
this.membershipOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService <DeploymentLoadPublisher>());
this.membershipOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService <ClientObserverRegistrar>());
IOptions <ReminderOptions> reminderOptions = this.Services.GetRequiredService <IOptions <ReminderOptions> >();
if (!reminderOptions.Value.ReminderService.Equals(ReminderOptions.BuiltIn.Disabled))
{
// start the reminder service system target
reminderService = Services.GetRequiredService <LocalReminderServiceFactory>()
.CreateReminderService(this, initTimeout, this.runtimeClient);
var reminderServiceSystemTarget = this.reminderService as SystemTarget;
if (reminderServiceSystemTarget != null)
{
RegisterSystemTarget(reminderServiceSystemTarget);
}
}
RegisterSystemTarget(catalog);
await scheduler.QueueAction(catalog.Start, catalog.SchedulingContext)
.WithTimeout(initTimeout, $"Starting Catalog failed due to timeout {initTimeout}");
// SystemTarget for provider init calls
this.fallbackScheduler = Services.GetRequiredService <FallbackSystemTarget>();
RegisterSystemTarget(fallbackScheduler);
}
private void InjectDependencies()
{
healthCheckParticipants.Add(membershipOracle);
catalog.SiloStatusOracle = this.membershipOracle;
this.membershipOracle.SubscribeToSiloStatusEvents(localGrainDirectory);
messageCenter.SiloDeadOracle = this.membershipOracle.IsDeadSilo;
// consistentRingProvider is not a system target per say, but it behaves like the localGrainDirectory, so it is here
this.membershipOracle.SubscribeToSiloStatusEvents((ISiloStatusListener)RingProvider);
this.membershipOracle.SubscribeToSiloStatusEvents(typeManager);
this.membershipOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService <DeploymentLoadPublisher>());
this.membershipOracle.SubscribeToSiloStatusEvents(Services.GetRequiredService <ClientObserverRegistrar>());
if (!GlobalConfig.ReminderServiceType.Equals(GlobalConfiguration.ReminderServiceProviderType.Disabled))
{
// start the reminder service system target
reminderService = Services.GetRequiredService <LocalReminderServiceFactory>()
.CreateReminderService(this, initTimeout, this.runtimeClient);
var reminderServiceSystemTarget = this.reminderService as SystemTarget;
if (reminderServiceSystemTarget != null)
{
RegisterSystemTarget(reminderServiceSystemTarget);
}
}
RegisterSystemTarget(catalog);
scheduler.QueueAction(catalog.Start, catalog.SchedulingContext)
.WaitWithThrow(initTimeout);
// SystemTarget for provider init calls
providerManagerSystemTarget = Services.GetRequiredService <ProviderManagerSystemTarget>();
RegisterSystemTarget(providerManagerSystemTarget);
}
/// <summary>
/// Receive a new message:
/// - validate order constraints, queue (or possibly redirect) if out of order
/// - validate transactions constraints
/// - invoke handler if ready, otherwise enqueue for later invocation
/// </summary>
/// <param name="message"></param>
public void ReceiveMessage(Message message)
{
this.messagingTrace.OnDispatcherReceiveMessage(message);
// Don't process messages that have already timed out
if (message.IsExpired)
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
this.messagingTrace.OnDropExpiredMessage(message, MessagingStatisticsGroup.Phase.Dispatch);
return;
}
// check if its targeted at a new activation
if (message.TargetGrain.IsSystemTarget())
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
throw new InvalidOperationException("Dispatcher was called ReceiveMessage on system target for " + message);
}
try
{
Task ignore;
ActivationData target = catalog.GetOrCreateActivation(
message.TargetAddress,
message.IsNewPlacement,
message.RequestContextData,
out ignore);
if (ignore != null)
{
ignore.Ignore();
}
if (message.Direction == Message.Directions.Response)
{
ReceiveResponse(message, target);
}
else // Request or OneWay
{
if (target.State == ActivationState.Valid)
{
this.activationCollector.TryRescheduleCollection(target);
}
// Silo is always capable to accept a new request. It's up to the activation to handle its internal state.
// If activation is shutting down, it will queue and later forward this request.
ReceiveRequest(message, target);
}
}
catch (Exception ex)
{
try
{
MessagingProcessingStatisticsGroup.OnDispatcherMessageProcessedError(message);
var nea = ex as Catalog.NonExistentActivationException;
if (nea == null)
{
var str = $"Error creating activation for grain {message.TargetGrain} (interface: {message.InterfaceType}). Message {message}";
logger.Error(ErrorCode.Dispatcher_ErrorCreatingActivation, str, ex);
throw new OrleansException(str, ex);
}
if (nea.IsStatelessWorker)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate StatelessWorker NonExistentActivation for message {message}, Exception {ex}");
}
}
else
{
logger.Info(ErrorCode.Dispatcher_Intermediate_GetOrCreateActivation,
$"Intermediate NonExistentActivation for message {message}, with Exception {ex}");
}
ActivationAddress nonExistentActivation = nea.NonExistentActivation;
if (message.Direction != Message.Directions.Response)
{
// Un-register the target activation so we don't keep getting spurious messages.
// The time delay (one minute, as of this writing) is to handle the unlikely but possible race where
// this request snuck ahead of another request, with new placement requested, for the same activation.
// If the activation registration request from the new placement somehow sneaks ahead of this un-registration,
// we want to make sure that we don't un-register the activation we just created.
// We would add a counter here, except that there's already a counter for this in the Catalog.
// Note that this has to run in a non-null scheduler context, so we always queue it to the catalog's context
var origin = message.SendingSilo;
scheduler.QueueAction(
// don't use message.TargetAddress, cause it may have been removed from the headers by this time!
async() =>
{
try
{
if (this.logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("UnregisterAfterNonexistingActivation addr={ActivationAddress} origin={SiloAddress}", nonExistentActivation, origin);
}
await this.grainLocator.Unregister(nonExistentActivation, UnregistrationCause.NonexistentActivation);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Dispatcher_FailedToUnregisterNonExistingAct,
$"Failed to un-register NonExistentActivation {nonExistentActivation}", exc);
}
},
catalog);
ProcessRequestToInvalidActivation(message, nonExistentActivation, null, "Non-existent activation");
}
else
{
logger.Warn(
ErrorCode.Dispatcher_NoTargetActivation,
nonExistentActivation.Silo.IsClient
? "No target client {0} for response message: {1}. It's likely that the client recently disconnected."
: "No target activation {0} for response message: {1}",
nonExistentActivation,
message);
this.localGrainDirectory.InvalidateCacheEntry(nonExistentActivation);
}
}
catch (Exception exc)
{
// Unable to create activation for this request - reject message
RejectMessage(message, Message.RejectionTypes.Transient, exc);
}
}
}
public void ReceiveMessage(Message msg)
{
this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg);
IGrainContext context;
// Find the activation it targets; first check for a system activation, then an app activation
if (msg.TargetGrain.IsSystemTarget())
{
SystemTarget target = this.directory.FindSystemTarget(msg.TargetActivation);
if (target == null)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message response = this.messageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.Unrecoverable,
string.Format("SystemTarget {0} not active on this silo. Msg={1}", msg.TargetGrain, msg));
this.messageCenter.SendMessage(response);
this.log.Warn(ErrorCode.MessagingMessageFromUnknownActivation, "Received a message {0} for an unknown SystemTarget: {1}", msg, msg.TargetAddress);
return;
}
context = target;
switch (msg.Direction)
{
case Message.Directions.Request:
this.messagingTrace.OnEnqueueMessageOnActivation(msg, target);
this.scheduler.QueueWorkItem(new RequestWorkItem(target, msg));
break;
case Message.Directions.Response:
this.messagingTrace.OnEnqueueMessageOnActivation(msg, target);
this.scheduler.QueueWorkItem(new ResponseWorkItem(target, msg));
break;
default:
this.log.Error(ErrorCode.Runtime_Error_100097, "Invalid message: " + msg);
break;
}
}
else
{
// Run this code on the target activation's context, if it already exists
ActivationData targetActivation = this.directory.FindTarget(msg.TargetActivation);
if (targetActivation != null)
{
lock (targetActivation)
{
var target = targetActivation; // to avoid a warning about nulling targetActivation under a lock on it
if (target.State == ActivationState.Valid)
{
// Response messages are not subject to overload checks.
if (msg.Direction != Message.Directions.Response)
{
var overloadException = target.CheckOverloaded(this.log);
if (overloadException != null)
{
// Send rejection as soon as we can, to avoid creating additional work for runtime
this.dispatcher.RejectMessage(msg, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + target);
return;
}
}
// Run ReceiveMessage in context of target activation
context = target;
}
else
{
// Can't use this activation - will queue for another activation
target = null;
context = null;
}
EnqueueReceiveMessage(msg, target, context);
}
}
else
{
// No usable target activation currently, so run ReceiveMessage in system context
EnqueueReceiveMessage(msg, null, null);
}
}
void EnqueueReceiveMessage(Message msg, ActivationData targetActivation, IGrainContext context)
{
this.messagingTrace.OnEnqueueMessageOnActivation(msg, context);
targetActivation?.IncrementEnqueuedOnDispatcherCount();
scheduler.QueueAction(() =>
{
try
{
dispatcher.ReceiveMessage(msg);
}
finally
{
targetActivation?.DecrementEnqueuedOnDispatcherCount();
}
},
context);
}
}
public void Sched_AC_Current_TaskScheduler()
{
UnitTestSchedulingContext context = new UnitTestSchedulingContext();
OrleansTaskScheduler orleansTaskScheduler = orleansTaskScheduler = TestInternalHelper.InitializeSchedulerForTesting(context, this.loggerFactory);
ActivationTaskScheduler activationScheduler = orleansTaskScheduler.GetWorkItemGroup(context).TaskScheduler;
// RuntimeContext.InitializeThread(masterScheduler);
this.mainDone = false;
var result = new TaskCompletionSource <bool>();
Task wrapper = null;
Task finalPromise = null;
orleansTaskScheduler.QueueAction(() =>
{
Log(1, "Outer ClosureWorkItem " + Task.CurrentId + " starting");
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #0"
Log(2, "Starting wrapper Task");
wrapper = Task.Factory.StartNew(() =>
{
Log(3, "Inside wrapper Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #1"
Log(4, "Wrapper Task " + Task.CurrentId + " creating AC chain");
Task promise1 = Task.Factory.StartNew(() =>
{
Log(5, "#1 Inside AC Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #1"
SubProcess1(1);
});
Task promise2 = promise1.ContinueWith((_) =>
{
Log(6, "#2 Inside AC Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #2"
SubProcess1(2);
});
finalPromise = promise2.ContinueWith((_) =>
{
Log(7, "#3 Inside final AC Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #3"
SubProcess1(3);
result.SetResult(true);
});
finalPromise.Ignore();
Log(8, "Wrapper Task Id=" + Task.CurrentId + " sleeping");
Thread.Sleep(TimeSpan.FromSeconds(1));
Log(9, "Wrapper Task Id=" + Task.CurrentId + " finished");
});
Log(10, "Outer ClosureWorkItem Task Id=" + Task.CurrentId + " sleeping");
Thread.Sleep(TimeSpan.FromSeconds(1));
Log(11, "Outer ClosureWorkItem Task Id=" + Task.CurrentId + " awake");
Log(12, "Finished Outer TaskWorkItem Task Id=" + wrapper.Id);
this.mainDone = true;
},
context);
Log(13, "Waiting for ClosureWorkItem to spawn wrapper Task");
for (int i = 0; i < 5 * WaitFactor; i++)
{
if (wrapper != null)
{
break;
}
Thread.Sleep(TimeSpan.FromSeconds(1).Multiply(WaitFactor));
}
Assert.NotNull(wrapper); // Wrapper Task was not created
Log(14, "Waiting for wrapper Task Id=" + wrapper.Id + " to complete");
bool finished = wrapper.Wait(TimeSpan.FromSeconds(4 * WaitFactor));
Log(15, "Done waiting for wrapper Task Id=" + wrapper.Id + " Finished=" + finished);
if (!finished)
{
throw new TimeoutException();
}
Assert.False(wrapper.IsFaulted, "Wrapper Task faulted: " + wrapper.Exception);
Assert.True(wrapper.IsCompleted, "Wrapper Task should be completed");
Log(16, "Waiting for TaskWorkItem to complete");
for (int i = 0; i < 15 * WaitFactor; i++)
{
if (this.mainDone)
{
break;
}
Thread.Sleep(1000 * WaitFactor);
}
Log(17, "Done waiting for TaskWorkItem to complete MainDone=" + this.mainDone);
Assert.True(this.mainDone, "Main Task should be completed");
Assert.NotNull(finalPromise); // AC chain not created
Log(18, "Waiting for final AC promise to complete");
finalPromise.Wait(TimeSpan.FromSeconds(4 * WaitFactor));
Log(19, "Done waiting for final promise");
Assert.False(finalPromise.IsFaulted, "Final AC faulted: " + finalPromise.Exception);
Assert.True(finalPromise.IsCompleted, "Final AC completed");
Assert.True(result.Task.Result, "Timeout-1");
Assert.NotEqual(0, this.stageNum1); // "Work items did not get executed-1"
Assert.Equal(3, this.stageNum1); // "Work items executed out of order-1"
}
public void Sched_Task_Turn_Execution_Order()
{
// A unit test that checks that any turn is indeed run till completion before any other turn?
// For example, you have a long running main turn and in the middle it spawns a lot of short CWs (on Done promise) and StartNew.
// You test that no CW/StartNew runs until the main turn is fully done. And run in stress.
UnitTestSchedulingContext context = new UnitTestSchedulingContext();
OrleansTaskScheduler masterScheduler = this.orleansTaskScheduler = TestInternalHelper.InitializeSchedulerForTesting(context, this.loggerFactory);
WorkItemGroup workItemGroup = this.orleansTaskScheduler.GetWorkItemGroup(context);
ActivationTaskScheduler activationScheduler = workItemGroup.TaskScheduler;
this.mainDone = false;
this.stageNum1 = this.stageNum2 = 0;
var result1 = new TaskCompletionSource <bool>();
var result2 = new TaskCompletionSource <bool>();
Task wrapper = null;
Task finalTask1 = null;
Task finalPromise2 = null;
masterScheduler.QueueAction(() =>
{
Log(1, "Outer ClosureWorkItem " + Task.CurrentId + " starting");
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #0"
Log(2, "Starting wrapper Task");
wrapper = Task.Factory.StartNew(() =>
{
Log(3, "Inside wrapper Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #1"
// Execution chain #1
Log(4, "Wrapper Task Id=" + Task.CurrentId + " creating Task chain");
Task task1 = Task.Factory.StartNew(() =>
{
Log(5, "#11 Inside sub-Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #11"
SubProcess1(11);
});
Task task2 = task1.ContinueWith((Task task) =>
{
Log(6, "#12 Inside continuation Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #12"
if (task.IsFaulted)
{
throw task.Exception.Flatten();
}
SubProcess1(12);
});
Task task3 = task2.ContinueWith(task =>
{
Log(7, "#13 Inside continuation Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #13"
if (task.IsFaulted)
{
throw task.Exception.Flatten();
}
SubProcess1(13);
});
finalTask1 = task3.ContinueWith(task =>
{
Log(8, "#14 Inside final continuation Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #14"
if (task.IsFaulted)
{
throw task.Exception.Flatten();
}
SubProcess1(14);
result1.SetResult(true);
});
// Execution chain #2
Log(9, "Wrapper Task " + Task.CurrentId + " creating AC chain");
Task promise2 = Task.Factory.StartNew(() =>
{
Log(10, "#21 Inside sub-Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #21"
SubProcess2(21);
});
finalPromise2 = promise2.ContinueWith((_) =>
{
Log(11, "#22 Inside final continuation Task Id=" + Task.CurrentId);
Assert.Equal(activationScheduler, TaskScheduler.Current); // "TaskScheduler.Current #22"
SubProcess2(22);
result2.SetResult(true);
});
finalPromise2.Ignore();
Log(12, "Wrapper Task Id=" + Task.CurrentId + " sleeping #2");
Thread.Sleep(TimeSpan.FromSeconds(1));
Log(13, "Wrapper Task Id=" + Task.CurrentId + " finished");
});
Log(14, "Outer ClosureWorkItem Task Id=" + Task.CurrentId + " sleeping");
Thread.Sleep(TimeSpan.FromSeconds(1));
Log(15, "Outer ClosureWorkItem Task Id=" + Task.CurrentId + " awake");
Log(16, "Finished Outer ClosureWorkItem Task Id=" + wrapper.Id);
this.mainDone = true;
},
context);
Log(17, "Waiting for ClosureWorkItem to spawn wrapper Task");
for (int i = 0; i < 5 * WaitFactor; i++)
{
if (wrapper != null)
{
break;
}
Thread.Sleep(TimeSpan.FromSeconds(1).Multiply(WaitFactor));
}
Assert.NotNull(wrapper); // Wrapper Task was not created
Log(18, "Waiting for wrapper Task Id=" + wrapper.Id + " to complete");
bool finished = wrapper.Wait(TimeSpan.FromSeconds(4 * WaitFactor));
Log(19, "Done waiting for wrapper Task Id=" + wrapper.Id + " Finished=" + finished);
if (!finished)
{
throw new TimeoutException();
}
Assert.False(wrapper.IsFaulted, "Wrapper Task faulted: " + wrapper.Exception);
Assert.True(wrapper.IsCompleted, "Wrapper Task should be completed");
Log(20, "Waiting for TaskWorkItem to complete");
for (int i = 0; i < 15 * WaitFactor; i++)
{
if (this.mainDone)
{
break;
}
Thread.Sleep(1000 * WaitFactor);
}
Log(21, "Done waiting for TaskWorkItem to complete MainDone=" + this.mainDone);
Assert.True(this.mainDone, "Main Task should be completed");
Assert.NotNull(finalTask1); // Task chain #1 not created
Assert.NotNull(finalPromise2); // Task chain #2 not created
Log(22, "Waiting for final task #1 to complete");
bool ok = finalTask1.Wait(TimeSpan.FromSeconds(4 * WaitFactor));
Log(23, "Done waiting for final task #1 complete Ok=" + ok);
if (!ok)
{
throw new TimeoutException();
}
Assert.False(finalTask1.IsFaulted, "Final Task faulted: " + finalTask1.Exception);
Assert.True(finalTask1.IsCompleted, "Final Task completed");
Assert.True(result1.Task.Result, "Timeout-1");
Log(24, "Waiting for final promise #2 to complete");
finalPromise2.Wait(TimeSpan.FromSeconds(4 * WaitFactor));
Log(25, "Done waiting for final promise #2");
Assert.False(finalPromise2.IsFaulted, "Final Task faulted: " + finalPromise2.Exception);
Assert.True(finalPromise2.IsCompleted, "Final Task completed");
Assert.True(result2.Task.Result, "Timeout-2");
Assert.NotEqual(0, this.stageNum1); // "Work items did not get executed-1"
Assert.Equal(14, this.stageNum1); // "Work items executed out of order-1"
Assert.NotEqual(0, this.stageNum2); // "Work items did not get executed-2"
Assert.Equal(22, this.stageNum2); // "Work items executed out of order-2"
}