protected override async Task OnProcessWorkItem(SessionWorkItem sessionWorkItem)
{
var messagesToSend = new List <BrokeredMessage>();
var timerMessages = new List <BrokeredMessage>();
var subOrchestrationMessages = new List <BrokeredMessage>();
bool isCompleted = false;
bool continuedAsNew = false;
BrokeredMessage continuedAsNewMessage = null;
ExecutionStartedEvent continueAsNewExecutionStarted = null;
MessageSession session = sessionWorkItem.Session;
IEnumerable <BrokeredMessage> newMessages = sessionWorkItem.Messages ?? new List <BrokeredMessage>();
long rawSessionStateSize;
long newSessionStateSize;
bool isEmptySession = false;
OrchestrationRuntimeState runtimeState;
using (Stream rawSessionStream = await session.GetStateAsync())
using (Stream sessionStream = await Utils.GetDecompressedStreamAsync(rawSessionStream))
{
isEmptySession = sessionStream == null;
rawSessionStateSize = isEmptySession ? 0 : rawSessionStream.Length;
newSessionStateSize = isEmptySession ? 0 : sessionStream.Length;
runtimeState = GetOrCreateInstanceState(sessionStream, session.SessionId);
}
TraceHelper.TraceSession(TraceEventType.Information, session.SessionId,
"Size of session state is {0}, compressed {1}", newSessionStateSize, rawSessionStateSize);
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
if (!(await ReconcileMessagesWithStateAsync(session.SessionId, runtimeState, newMessages)))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(TraceEventType.Error, session.SessionId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
TraceHelper.TraceInstance(
TraceEventType.Verbose,
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
JsonConvert.SerializeObject(runtimeState.GetOrchestrationRuntimeStateDump(),
new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto,
Formatting = Formatting.Indented
}));
IEnumerable <OrchestratorAction> decisions = ExecuteOrchestration(runtimeState);
TraceHelper.TraceInstance(TraceEventType.Information,
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count(),
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}", decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
TaskMessage taskMessage =
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters);
BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(
taskMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance,
"ScheduleTask");
brokeredMessage.SessionId = session.SessionId;
messagesToSend.Add(brokeredMessage);
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
TaskMessage timerMessage = ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState);
BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject(
timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance,
"Timer");
brokeredTimerMessage.ScheduledEnqueueTimeUtc = timerOrchestratorAction.FireAt;
brokeredTimerMessage.SessionId = session.SessionId;
timerMessages.Add(brokeredTimerMessage);
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
TaskMessage createSubOrchestrationInstanceMessage =
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters);
BrokeredMessage createSubOrchestrationMessage = Utils.GetBrokeredMessageFromObject(
createSubOrchestrationInstanceMessage, settings.MessageCompressionSettings,
runtimeState.OrchestrationInstance, "Schedule Suborchestration");
createSubOrchestrationMessage.SessionId =
createSubOrchestrationInstanceMessage.OrchestrationInstance.InstanceId;
subOrchestrationMessages.Add(createSubOrchestrationMessage);
break;
case OrchestratorActionType.OrchestrationComplete:
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision,
runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
BrokeredMessage workflowCompletedBrokeredMessage = Utils.GetBrokeredMessageFromObject(
workflowInstanceCompletedMessage, settings.MessageCompressionSettings,
runtimeState.OrchestrationInstance, "Complete Suborchestration");
workflowCompletedBrokeredMessage.SessionId =
workflowInstanceCompletedMessage.OrchestrationInstance.InstanceId;
// Store the event so we can rebuild the state
if (continuedAsNew)
{
continuedAsNewMessage = workflowCompletedBrokeredMessage;
continueAsNewExecutionStarted =
workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
}
else
{
subOrchestrationMessages.Add(workflowCompletedBrokeredMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(TraceEventType.Error,
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// We cannot send more than 100 messages within a transaction, to avoid the situation
// we keep on checking the message count and stop processing the new decisions.
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count;
// Also add tracking messages as they contribute to total messages within transaction
if (isTrackingEnabled)
{
totalMessages += runtimeState.NewEvents.Count;
}
if (totalMessages > MaxMessageCount)
{
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
TaskMessage timerMessage = ProcessCreateTimerDecision(dummyTimer, runtimeState);
BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject(
timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance,
"MaxMessageCount Timer");
brokeredTimerMessage.ScheduledEnqueueTimeUtc = dummyTimer.FireAt;
brokeredTimerMessage.SessionId = session.SessionId;
timerMessages.Add(brokeredTimerMessage);
break;
}
}
}
// TODO : make async, transactions are a bit tricky
using (var ts = new TransactionScope())
{
bool isSessionSizeThresholdExceeded = false;
if (!continuedAsNew)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
using (var ms = new MemoryStream())
{
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Deleting session state");
// if session was already null and we finished the orchestration then no change is required
if (!isEmptySession)
{
session.SetState(null);
}
runtimeState.Size = 0;
runtimeState.CompressedSize = 0;
}
else
{
if (ms.Position != 0)
{
throw TraceHelper.TraceExceptionInstance(TraceEventType.Error,
runtimeState.OrchestrationInstance,
new ArgumentException("Instance state stream is partially consumed"));
}
IList <HistoryEvent> newState = runtimeState.Events;
if (continuedAsNew)
{
TraceHelper.TraceSession(TraceEventType.Information, session.SessionId,
"Updating state for continuation");
newState = new List <HistoryEvent>();
newState.Add(new OrchestratorStartedEvent(-1));
newState.Add(continueAsNewExecutionStarted);
newState.Add(new OrchestratorCompletedEvent(-1));
}
string serializedState = JsonConvert.SerializeObject(newState,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Auto
});
var writer = new StreamWriter(ms);
writer.Write(serializedState);
writer.Flush();
ms.Position = 0;
using (Stream compressedState =
settings.TaskOrchestrationDispatcherSettings.CompressOrchestrationState
? Utils.GetCompressedStream(ms)
: ms)
{
runtimeState.Size = ms.Length;
runtimeState.CompressedSize = compressedState.Length;
if (runtimeState.CompressedSize > SessionStreamTerminationThresholdInBytes)
{
// basic idea is to simply enqueue a terminate message just like how we do it from taskhubclient
// it is possible to have other messages in front of the queue and those will get processed before
// the terminate message gets processed. but that is ok since in the worst case scenario we will
// simply land in this if-block again and end up queuing up another terminate message.
//
// the interesting scenario is when the second time we *dont* land in this if-block because e.g.
// the new messages that we processed caused a new generation to be created. in that case
// it is still ok because the worst case scenario is that we will terminate a newly created generation
// which shouldn't have been created at all in the first place
isSessionSizeThresholdExceeded = true;
TraceHelper.TraceSession(TraceEventType.Critical, session.SessionId,
"Size of session state " + runtimeState.CompressedSize +
" has exceeded termination threshold of "
+ SessionStreamTerminationThresholdInBytes);
string reason =
string.Format(
"Session state size of {0} exceeded the termination threshold of {1} bytes",
runtimeState.CompressedSize, SessionStreamTerminationThresholdInBytes);
BrokeredMessage forcedTerminateMessage = CreateForcedTerminateMessage(
runtimeState.OrchestrationInstance.InstanceId, reason);
orchestratorQueueClient.Send(forcedTerminateMessage);
}
else
{
session.SetState(compressedState);
}
}
writer.Close();
}
}
if (!isSessionSizeThresholdExceeded)
{
if (runtimeState.CompressedSize > SessionStreamWarningSizeInBytes)
{
TraceHelper.TraceSession(TraceEventType.Error, session.SessionId,
"Size of session state is nearing session size limit of 256KB");
}
if (!continuedAsNew)
{
if (messagesToSend.Count > 0)
{
messagesToSend.ForEach(m => workerSender.Send(m));
}
if (timerMessages.Count > 0)
{
timerMessages.ForEach(m => orchestratorQueueClient.Send(m));
}
}
if (subOrchestrationMessages.Count > 0)
{
subOrchestrationMessages.ForEach(m => orchestratorQueueClient.Send(m));
}
if (continuedAsNewMessage != null)
{
orchestratorQueueClient.Send(continuedAsNewMessage);
}
if (isTrackingEnabled)
{
List <BrokeredMessage> trackingMessages = CreateTrackingMessages(runtimeState);
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Created {0} tracking messages", trackingMessages.Count);
if (trackingMessages.Count > 0)
{
trackingMessages.ForEach(m => trackingSender.Send(m));
}
}
}
IEnumerable <Guid> lockTokens = newMessages.Select(m => m.LockToken);
session.CompleteBatch(lockTokens);
ts.Complete();
}
}
protected override async Task OnProcessWorkItem(SessionWorkItem sessionWorkItem)
{
MessageSession session = sessionWorkItem.Session;
IEnumerable <BrokeredMessage> newMessages = sessionWorkItem.Messages;
var historyEntities = new List <OrchestrationHistoryEventEntity>();
var stateEntities = new List <OrchestrationStateEntity>();
foreach (BrokeredMessage message in newMessages)
{
Utils.CheckAndLogDeliveryCount(message, taskHubDescription.MaxTrackingDeliveryCount);
if (message.ContentType.Equals(FrameworkConstants.TaskMessageContentType,
StringComparison.OrdinalIgnoreCase))
{
object historyEventIndexObj;
if (!message.Properties.TryGetValue(FrameworkConstants.HistoryEventIndexPropertyName,
out historyEventIndexObj) ||
historyEventIndexObj == null)
{
throw new InvalidOperationException(
"Could not find a valid history event index property on tracking message " +
message.MessageId);
}
var historyEventIndex = (int)historyEventIndexObj;
TaskMessage taskMessage = await Utils.GetObjectFromBrokeredMessageAsync <TaskMessage>(message);
historyEntities.Add(new OrchestrationHistoryEventEntity(
taskMessage.OrchestrationInstance.InstanceId,
taskMessage.OrchestrationInstance.ExecutionId,
historyEventIndex,
DateTime.UtcNow,
taskMessage.Event));
}
else if (message.ContentType.Equals(FrameworkConstants.StateMessageContentType,
StringComparison.OrdinalIgnoreCase))
{
StateMessage stateMessage = await Utils.GetObjectFromBrokeredMessageAsync <StateMessage>(message);
stateEntities.Add(new OrchestrationStateEntity(stateMessage.State));
}
else
{
throw new InvalidOperationException("Invalid tracking message content type: " +
message.ContentType);
}
}
TraceEntities(TraceEventType.Verbose, "Writing tracking history event", historyEntities,
GetNormalizedHistoryEventEntityTrace);
TraceEntities(TraceEventType.Verbose, "Writing tracking state event", stateEntities,
GetNormalizedStateEntityTrace);
try
{
await Utils.ExecuteWithRetries(() => tableClient.WriteEntitesAsync(historyEntities),
session.SessionId, string.Format("WriteHistoryEntities:{0}", session.SessionId),
MaxRetriesTableStore,
IntervalBetweenRetriesSecs);
}
catch (Exception)
{
TraceEntities(TraceEventType.Critical, "Failed to write history entity", historyEntities,
GetNormalizedHistoryEventEntityTrace);
throw;
}
try
{
foreach (OrchestrationStateEntity stateEntity in stateEntities)
{
await Utils.ExecuteWithRetries(
() => tableClient.WriteEntitesAsync(new List <OrchestrationStateEntity> {
stateEntity
}),
session.SessionId, string.Format("WriteStateEntities:{0}", session.SessionId),
MaxRetriesTableStore,
IntervalBetweenRetriesSecs);
}
}
catch (Exception)
{
TraceEntities(TraceEventType.Critical, "Failed to write state entity", stateEntities,
GetNormalizedStateEntityTrace);
throw;
}
IEnumerable <Guid> lockTokens = newMessages.Select(m => m.LockToken);
Utils.SyncExecuteWithRetries <object>(() =>
{
session.CompleteBatch(lockTokens);
return(null);
},
session.SessionId,
"Complete New Tracking Messages",
MaxRetriesServiceBus,
IntervalBetweenRetriesSecs);
}
