public async Task Run(string orchestratorFunctionName, object input)
{
_orchestratorFunctionName = orchestratorFunctionName;
Input = input;
Output = null;
CreatedTime = CurrentUtcDateTime;
Status = OrchestrationRuntimeStatus.Running;
var executionStarted = new ExecutionStartedEvent(History.Count, JsonConvert.SerializeObject(input));
History.Add(executionStarted);
try
{
Output = await CallActivityFunctionByNameAsync(orchestratorFunctionName, input, reuseContext : true);
Status = OrchestrationRuntimeStatus.Completed;
History.Add(new ExecutionCompletedEvent(History.Count, JsonConvert.SerializeObject(Output), OrchestrationStatus.Completed));
}
catch (Exception ex)
{
Status = OrchestrationRuntimeStatus.Failed;
History.Add(new ExecutionCompletedEvent(History.Count, JsonConvert.SerializeObject(ex), OrchestrationStatus.Failed));
}
}
/// <summary>
/// Tests whether an instance should automatically start if receiving messages and not already being started, and inserts
/// a corresponding ExecutionStartedEvent taskMessage.
/// </summary>
/// <param name="instanceId"></param>
/// <param name="newMessages"></param>
/// <returns></returns>
public static bool AutoStart(string instanceId, IList <TaskMessage> newMessages)
{
if (IsEntityInstance(instanceId) &&
newMessages[0].Event.EventType == EventType.EventRaised &&
newMessages[0].OrchestrationInstance.ExecutionId == null)
{
// automatically start this instance
var orchestrationInstance = new OrchestrationInstance {
InstanceId = instanceId,
ExecutionId = Guid.NewGuid().ToString("N"),
};
var startedEvent = new ExecutionStartedEvent(-1, null)
{
Name = instanceId,
Version = "",
OrchestrationInstance = orchestrationInstance
};
var taskMessage = new TaskMessage()
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
};
newMessages.Insert(0, taskMessage);
return(true);
}
else
{
return(false);
}
}
public override async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
{
using SqlConnection connection = await this.GetAndOpenConnectionAsync();
using SqlCommand command = this.GetSprocCommand(connection, "dt.CreateInstance");
ExecutionStartedEvent startEvent = (ExecutionStartedEvent)creationMessage.Event;
OrchestrationInstance instance = startEvent.OrchestrationInstance;
command.Parameters.Add("@Name", SqlDbType.VarChar, size: 300).Value = startEvent.Name;
command.Parameters.Add("@Version", SqlDbType.VarChar, size: 100).Value = startEvent.Version;
command.Parameters.Add("@InstanceID", SqlDbType.VarChar, size: 100).Value = instance.InstanceId;
command.Parameters.Add("@ExecutionID", SqlDbType.VarChar, size: 50).Value = instance.ExecutionId;
command.Parameters.Add("@InputText", SqlDbType.VarChar).Value = startEvent.Input;
command.Parameters.Add("@StartTime", SqlDbType.DateTime2).Value = startEvent.ScheduledStartTime;
try
{
await SqlUtils.ExecuteNonQueryAsync(command, this.traceHelper, instance.InstanceId);
}
catch (SqlException e) when(e.Number == 50001 /* Instance ID for pending/running instance already exists */)
{
// Try to avoid leaking SQL exception for issues like this
throw new InvalidOperationException(e.Message, e);
}
}
/// <inheritdoc />
public override async Task <bool> SetNewExecutionAsync(
ExecutionStartedEvent executionStartedEvent,
string eTag /* not used */,
string inputStatusOverride)
{
var orchestrationState = new OrchestrationState()
{
Name = executionStartedEvent.Name,
Version = executionStartedEvent.Version,
OrchestrationInstance = executionStartedEvent.OrchestrationInstance,
OrchestrationStatus = OrchestrationStatus.Pending,
Input = inputStatusOverride ?? executionStartedEvent.Input,
Tags = executionStartedEvent.Tags,
CreatedTime = executionStartedEvent.Timestamp,
LastUpdatedTime = DateTime.UtcNow,
CompletedTime = Core.Common.DateTimeUtils.MinDateTime,
ScheduledStartTime = executionStartedEvent.ScheduledStartTime
};
var orchestrationStateEntity = new OrchestrationStateInstanceEntity()
{
State = orchestrationState,
SequenceNumber = 0
};
await this.instanceStore.WriteEntitiesAsync(new[] { orchestrationStateEntity });
return(true);
}
public async Task Run(string orchestratorFunctionName, object input)
{
_orchestratorFunctionName = orchestratorFunctionName;
_serializedInput = input == null
? JValue.CreateNull()
: JToken.FromObject(input);
CreatedTime = CurrentUtcDateTime;
Status = OrchestrationRuntimeStatus.Running;
var executionStarted = new ExecutionStartedEvent(History.Count, JsonConvert.SerializeObject(input));
History.Add(executionStarted);
try
{
_output = await CallActivityFunctionByNameAsync(orchestratorFunctionName, input, reuseContext : true);
Status = OrchestrationRuntimeStatus.Completed;
History.Add(new ExecutionCompletedEvent(History.Count, JsonConvert.SerializeObject(_output), OrchestrationStatus.Completed));
}
catch (Exception ex)
{
Status = OrchestrationRuntimeStatus.Failed;
History.Add(new TaskFailedEvent(History.Count, 0, ex.Message, ex.StackTrace));
History.Add(new ExecutionCompletedEvent(History.Count, null, OrchestrationStatus.Failed));
}
}
/// <inheritdoc />
public override async Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent)
{
DynamicTableEntity entity = new DynamicTableEntity(executionStartedEvent.OrchestrationInstance.InstanceId, "")
{
Properties =
{
["Input"] = new EntityProperty(executionStartedEvent.Input),
["CreatedTime"] = new EntityProperty(executionStartedEvent.Timestamp),
["Name"] = new EntityProperty(executionStartedEvent.Name),
["Version"] = new EntityProperty(executionStartedEvent.Version),
["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Pending.ToString()),
["LastUpdatedTime"] = new EntityProperty(executionStartedEvent.Timestamp),
}
};
await this.CompressLargeMessageAsync(entity);
Stopwatch stopwatch = Stopwatch.StartNew();
await this.instancesTable.ExecuteAsync(
TableOperation.InsertOrReplace(entity));
this.stats.StorageRequests.Increment();
this.stats.TableEntitiesWritten.Increment(1);
AnalyticsEventSource.Log.InstanceStatusUpdate(
this.storageAccountName,
this.taskHubName,
executionStartedEvent.OrchestrationInstance.InstanceId,
executionStartedEvent.OrchestrationInstance.ExecutionId,
executionStartedEvent.EventType.ToString(),
stopwatch.ElapsedMilliseconds);
}
/// <summary>
/// Logs that a new orchestration instance has been scheduled.
/// </summary>
/// <param name="startedEvent">The start event for the new orchestration.</param>
internal void SchedulingOrchestration(ExecutionStartedEvent startedEvent)
{
// Note that this log event is also used when orchestrations send events
if (this.IsStructuredLoggingEnabled)
{
this.WriteStructuredLog(new LogEvents.SchedulingOrchestration(startedEvent));
}
}
OrchestrationRuntimeState generateOrchestrationRuntimeState(string input)
{
IList <HistoryEvent> historyEvents = new List <HistoryEvent>();
var historyEvent = new ExecutionStartedEvent(1, input);
historyEvents.Add(historyEvent);
var newOrchestrationRuntimeState = new OrchestrationRuntimeState(historyEvents);
return(newOrchestrationRuntimeState);
}
public static async Task <OrchestrationInstance> CreateOrchestrationInstanceAsync(
ServiceBusOrchestrationService sboService,
string name,
string version,
string instanceId,
string executionId,
bool jumpStartOnly,
bool serviceBusOnly)
{
if (string.IsNullOrWhiteSpace(instanceId))
{
instanceId = Guid.NewGuid().ToString("N");
}
if (string.IsNullOrWhiteSpace(executionId))
{
executionId = Guid.NewGuid().ToString("N");
}
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = instanceId,
ExecutionId = executionId,
};
var startedEvent = new ExecutionStartedEvent(-1, null)
{
Tags = null,
Name = name,
Version = version,
OrchestrationInstance = orchestrationInstance
};
var taskMessage = new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
};
if (jumpStartOnly)
{
await sboService.UpdateJumpStartStoreAsync(taskMessage);
}
else if (serviceBusOnly)
{
await sboService.SendTaskOrchestrationMessageAsync(taskMessage);
}
else
{
await sboService.CreateTaskOrchestrationAsync(taskMessage);
}
return(orchestrationInstance);
}
public Instance CreateInstance(ExecutionStartedEvent executionStartedEvent)
{
var instance = new Instance
{
InstanceId = executionStartedEvent.OrchestrationInstance.InstanceId,
LastExecutionId = executionStartedEvent.OrchestrationInstance.ExecutionId,
LockedUntil = DateTime.UtcNow,
LastQueueName = QueueMapper.ToQueueName(executionStartedEvent.Name, executionStartedEvent.Version)
};
return(instance);
}
IEnumerable <OrchestrationHistoryEventEntity> CreateHistoryEntities(TableClient client, string instanceId,
string genId, int count)
{
var historyEntities = new List <OrchestrationHistoryEventEntity>();
for (int i = 0; i < count; i++)
{
var eeStartedEvent = new ExecutionStartedEvent(-1, "EVENT_" + instanceId + "_" + genId + "_" + i);
historyEntities.Add(new OrchestrationHistoryEventEntity(instanceId, genId, i, DateTime.Now,
eeStartedEvent));
}
client.WriteEntitesAsync(historyEntities).Wait();
return(historyEntities);
}
/// <inheritdoc />
public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
{
ExecutionStartedEvent ee = creationMessage.Event as ExecutionStartedEvent;
if (ee == null)
{
throw new InvalidOperationException("Invalid creation task message");
}
lock (this.thisLock)
{
Dictionary <string, OrchestrationState> ed;
if (!this.instanceStore.TryGetValue(creationMessage.OrchestrationInstance.InstanceId, out ed))
{
ed = new Dictionary <string, OrchestrationState>();
this.instanceStore[creationMessage.OrchestrationInstance.InstanceId] = ed;
}
var latestState = ed.Values.OrderBy(state => state.CreatedTime).FirstOrDefault(state => state.OrchestrationStatus != OrchestrationStatus.ContinuedAsNew);
if (latestState != null && (dedupeStatuses == null || dedupeStatuses.Contains(latestState.OrchestrationStatus)))
{
// An orchestration with same instance id is already running
throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' already exists. It is in state {latestState.OrchestrationStatus}");
}
OrchestrationState newState = new OrchestrationState()
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = creationMessage.OrchestrationInstance.InstanceId,
ExecutionId = creationMessage.OrchestrationInstance.ExecutionId,
},
CreatedTime = DateTime.UtcNow,
OrchestrationStatus = OrchestrationStatus.Pending,
Version = ee.Version,
Name = ee.Name,
Input = ee.Input,
};
ed.Add(creationMessage.OrchestrationInstance.ExecutionId, newState);
this.orchestratorQueue.SendMessage(creationMessage);
}
return(Task.FromResult <object>(null));
}
TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
CreateSubOrchestrationAction createSubOrchestrationAction,
OrchestrationRuntimeState runtimeState,
bool includeParameters)
{
var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id)
{
Name = createSubOrchestrationAction.Name,
Version = createSubOrchestrationAction.Version,
InstanceId = createSubOrchestrationAction.InstanceId
};
if (includeParameters)
{
historyEvent.Input = createSubOrchestrationAction.Input;
}
runtimeState.AddEvent(historyEvent);
var taskMessage = new TaskMessage();
var startedEvent = new ExecutionStartedEvent(-1, createSubOrchestrationAction.Input)
{
Tags = OrchestrationTags.MergeTags(createSubOrchestrationAction.Tags, runtimeState.Tags),
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = createSubOrchestrationAction.InstanceId,
ExecutionId = Guid.NewGuid().ToString("N")
},
ParentInstance = new ParentInstance
{
OrchestrationInstance = runtimeState.OrchestrationInstance,
Name = runtimeState.Name,
Version = runtimeState.Version,
TaskScheduleId = createSubOrchestrationAction.Id
},
Name = createSubOrchestrationAction.Name,
Version = createSubOrchestrationAction.Version
};
this.logHelper.SchedulingOrchestration(startedEvent);
taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance;
taskMessage.Event = startedEvent;
return(taskMessage);
}
/// <summary>
/// Create a new orchestration of the specified name and version
/// </summary>
/// <param name="name">Name of the orchestration as specified by the ObjectCreator</param>
/// <param name="version">Name of the orchestration as specified by the ObjectCreator</param>
/// <param name="instanceId">Instance id for the orchestration to be created, must be unique across the Task Hub</param>
/// <param name="input">Input parameter to the specified TaskOrchestration</param>
/// <param name="tags">Dictionary of key/value tags associated with this instance</param>
/// <returns>OrchestrationInstance that represents the orchestration that was created</returns>
public async Task <OrchestrationInstance> CreateOrchestrationInstanceAsync(string name, string version,
string instanceId,
object input, IDictionary <string, string> tags)
{
if (string.IsNullOrWhiteSpace(instanceId))
{
instanceId = Guid.NewGuid().ToString("N");
}
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = instanceId,
ExecutionId = Guid.NewGuid().ToString("N"),
};
string serializedInput = defaultConverter.Serialize(input);
string serializedtags = tags != null?defaultConverter.Serialize(tags) : null;
var startedEvent = new ExecutionStartedEvent(-1, serializedInput)
{
Tags = serializedtags,
Name = name,
Version = version,
OrchestrationInstance = orchestrationInstance
};
var taskMessage = new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
};
BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(taskMessage,
settings.MessageCompressionSettings);
brokeredMessage.SessionId = instanceId;
MessageSender sender =
await messagingFactory.CreateMessageSenderAsync(orchestratorEntityName).ConfigureAwait(false);
await sender.SendAsync(brokeredMessage).ConfigureAwait(false);
await sender.CloseAsync().ConfigureAwait(false);
return(orchestrationInstance);
}
public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage)
{
if (creationMessage.Event is ExecutionStartedEvent executionStarted && executionStarted.ScheduledStartTime.HasValue)
{
throw new NotSupportedException("Service Fabric storage provider for Durable Tasks currently does not support scheduled starts");
}
creationMessage.OrchestrationInstance.InstanceId.EnsureValidInstanceId();
ExecutionStartedEvent startEvent = creationMessage.Event as ExecutionStartedEvent;
if (startEvent == null)
{
await this.SendTaskOrchestrationMessageAsync(creationMessage);
return;
}
var instance = creationMessage.OrchestrationInstance;
var added = await RetryHelper.ExecuteWithRetryOnTransient <bool>(async() =>
{
using (var tx = this.stateManager.CreateTransaction())
{
if (await this.orchestrationProvider.TryAddSession(tx, new TaskMessageItem(creationMessage)))
{
await WriteExecutionStartedEventToInstanceStore(tx, startEvent);
await tx.CommitAsync();
return(true);
}
return(false);
}
}, uniqueActionIdentifier : $"Orchestration = '{instance}', Action = '{nameof(CreateTaskOrchestrationAsync)}'");
if (added)
{
string message = string.Format("Orchestration with instanceId : '{0}' and executionId : '{1}' is Created.", instance.InstanceId, instance.ExecutionId);
ServiceFabricProviderEventSource.Tracing.LogOrchestrationInformation(instance.InstanceId, instance.ExecutionId, message);
this.orchestrationProvider.TryEnqueueSession(creationMessage.OrchestrationInstance);
}
else
{
throw new OrchestrationAlreadyExistsException($"An orchestration with id '{creationMessage.OrchestrationInstance.InstanceId}' is already running.");
}
}
protected async Task JumpStartOrchestrationAsync(OrchestrationJumpStartInstanceEntity jumpStartEntity)
{
var instance = jumpStartEntity.State.OrchestrationInstance;
OrchestrationStateInstanceEntity stateEntity = (await this.service.InstanceStore.GetEntitiesAsync(instance.InstanceId, instance.ExecutionId))?.FirstOrDefault();
if (stateEntity != null)
{
// It seems orchestration started, delete entity from JumpStart table
await this.service.InstanceStore.DeleteJumpStartEntitiesAsync(new[] { jumpStartEntity });
}
else if (!jumpStartEntity.JumpStartTime.IsSet() &&
jumpStartEntity.State.CreatedTime + this.ignoreWindow < DateTime.UtcNow)
{
// JumpStart orchestration
var startedEvent = new ExecutionStartedEvent(-1, jumpStartEntity.State.Input)
{
Tags = jumpStartEntity.State.Tags,
Name = jumpStartEntity.State.Name,
Version = jumpStartEntity.State.Version,
OrchestrationInstance = jumpStartEntity.State.OrchestrationInstance
};
var taskMessage = new TaskMessage
{
OrchestrationInstance = jumpStartEntity.State.OrchestrationInstance,
Event = startedEvent
};
await this.service.SendTaskOrchestrationMessageAsync(taskMessage);
TraceHelper.Trace(
TraceEventType.Information,
"JumpStartManager-SendTaskOrchestrationMessage",
$"JumpStartManager: SendTaskOrchestrationMessageAsync({instance.InstanceId}, {instance.ExecutionId}) success!");
// Now update the JumpStart table
jumpStartEntity.JumpStartTime = DateTime.UtcNow;
await this.service.InstanceStore.WriteJumpStartEntitiesAsync(new[] { jumpStartEntity });
TraceHelper.Trace(
TraceEventType.Information,
"JumpStartManager-WriteJumpStartEntities",
$"JumpStartManager: WriteJumpStartEntitiesAsync({instance.InstanceId}, {instance.ExecutionId}) success!");
}
}
public override async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
{
using SqlConnection connection = await this.GetAndOpenConnectionAsync();
using SqlCommand command = this.GetSprocCommand(connection, "dt.CreateInstance");
ExecutionStartedEvent startEvent = (ExecutionStartedEvent)creationMessage.Event;
OrchestrationInstance instance = startEvent.OrchestrationInstance;
command.Parameters.Add("@Name", SqlDbType.VarChar, size: 300).Value = startEvent.Name;
command.Parameters.Add("@Version", SqlDbType.VarChar, size: 100).Value = startEvent.Version;
command.Parameters.Add("@InstanceID", SqlDbType.VarChar, size: 100).Value = instance.InstanceId;
command.Parameters.Add("@ExecutionID", SqlDbType.VarChar, size: 50).Value = instance.ExecutionId;
command.Parameters.Add("@InputText", SqlDbType.VarChar).Value = startEvent.Input;
command.Parameters.Add("@StartTime", SqlDbType.DateTime2).Value = startEvent.ScheduledStartTime;
await SqlUtils.ExecuteNonQueryAsync(command, this.traceHelper, instance.InstanceId);
}
void SetMarkerEvents(HistoryEvent historyEvent)
{
if (historyEvent is ExecutionStartedEvent)
{
if (ExecutionStartedEvent != null)
{
throw new InvalidOperationException(
"Multiple ExecutionStartedEvent found, potential corruption in state storage");
}
ExecutionStartedEvent = (ExecutionStartedEvent)historyEvent;
}
else if (historyEvent is ExecutionCompletedEvent)
{
if (ExecutionCompletedEvent != null)
{
throw new InvalidOperationException(
"Multiple ExecutionCompletedEvent found, potential corruption in state storage");
}
ExecutionCompletedEvent = (ExecutionCompletedEvent)historyEvent;
}
}
/******************************/
// client methods
/******************************/
/// <inheritdoc />
public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage)
{
ExecutionStartedEvent ee = creationMessage.Event as ExecutionStartedEvent;
if (ee == null)
{
throw new InvalidOperationException("Invalid creation task message");
}
lock (this.thisLock)
{
this.orchestratorQueue.SendMessage(creationMessage);
Dictionary <string, OrchestrationState> ed;
if (!this.instanceStore.TryGetValue(creationMessage.OrchestrationInstance.InstanceId, out ed))
{
ed = new Dictionary <string, OrchestrationState>();
this.instanceStore[creationMessage.OrchestrationInstance.InstanceId] = ed;
}
OrchestrationState newState = new OrchestrationState()
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = creationMessage.OrchestrationInstance.InstanceId,
ExecutionId = creationMessage.OrchestrationInstance.ExecutionId,
},
CreatedTime = DateTime.UtcNow,
OrchestrationStatus = OrchestrationStatus.Pending,
Version = ee.Version,
Name = ee.Name,
Input = ee.Input,
};
ed.Add(creationMessage.OrchestrationInstance.ExecutionId, newState);
}
return(Task.FromResult <object>(null));
}
static TaskMessage ProcessCreateSubOrchestrationInstanceDecision(
CreateSubOrchestrationAction createSubOrchestrationAction,
OrchestrationRuntimeState runtimeState, bool includeParameters)
{
var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id);
historyEvent.Name = createSubOrchestrationAction.Name;
historyEvent.Version = createSubOrchestrationAction.Version;
historyEvent.InstanceId = createSubOrchestrationAction.InstanceId;
if (includeParameters)
{
historyEvent.Input = createSubOrchestrationAction.Input;
}
runtimeState.AddEvent(historyEvent);
var taskMessage = new TaskMessage();
var startedEvent = new ExecutionStartedEvent(-1, createSubOrchestrationAction.Input);
startedEvent.Tags = runtimeState.Tags;
startedEvent.OrchestrationInstance = new OrchestrationInstance
{
InstanceId = createSubOrchestrationAction.InstanceId,
ExecutionId = Guid.NewGuid().ToString("N")
};
startedEvent.ParentInstance = new ParentInstance
{
OrchestrationInstance = runtimeState.OrchestrationInstance,
Name = runtimeState.Name,
Version = runtimeState.Version,
TaskScheduleId = createSubOrchestrationAction.Id
};
startedEvent.Name = createSubOrchestrationAction.Name;
startedEvent.Version = createSubOrchestrationAction.Version;
taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance;
taskMessage.Event = startedEvent;
return(taskMessage);
}
Task WriteExecutionStartedEventToInstanceStore(ITransaction tx, ExecutionStartedEvent startEvent)
{
var createdTime = DateTime.UtcNow;
var initialState = new OrchestrationState()
{
Name = startEvent.Name,
Version = startEvent.Version,
OrchestrationInstance = startEvent.OrchestrationInstance,
OrchestrationStatus = OrchestrationStatus.Pending,
Input = startEvent.Input,
Tags = startEvent.Tags,
CreatedTime = createdTime,
LastUpdatedTime = createdTime
};
return(this.instanceStore.WriteEntitiesAsync(tx, new InstanceEntityBase[]
{
new OrchestrationStateInstanceEntity()
{
State = initialState
}
}));
}
/// <inheritdoc />
public override async Task SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent)
{
var orchestrationState = new OrchestrationState()
{
Name = executionStartedEvent.Name,
Version = executionStartedEvent.Version,
OrchestrationInstance = executionStartedEvent.OrchestrationInstance,
OrchestrationStatus = OrchestrationStatus.Pending,
Input = executionStartedEvent.Input,
Tags = executionStartedEvent.Tags,
CreatedTime = executionStartedEvent.Timestamp,
LastUpdatedTime = DateTime.UtcNow,
CompletedTime = Core.Common.DateTimeUtils.MinDateTime
};
var orchestrationStateEntity = new OrchestrationStateInstanceEntity()
{
State = orchestrationState,
SequenceNumber = 0
};
await this.instanceStore.WriteEntitiesAsync(new[] { orchestrationStateEntity });
}
HistoryEvent GenerateAbridgedEvent(HistoryEvent evt)
{
HistoryEvent returnedEvent = evt;
if (evt is TaskScheduledEvent)
{
var sourceEvent = (TaskScheduledEvent) evt;
returnedEvent = new TaskScheduledEvent(sourceEvent.EventId)
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
Name = sourceEvent.Name,
Version = sourceEvent.Version,
Input = "[..snipped..]",
};
}
else if (evt is TaskCompletedEvent)
{
var sourceEvent = (TaskCompletedEvent) evt;
returnedEvent = new TaskCompletedEvent(sourceEvent.EventId, sourceEvent.TaskScheduledId, "[..snipped..]")
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
else if (evt is SubOrchestrationInstanceCreatedEvent)
{
var sourceEvent = (SubOrchestrationInstanceCreatedEvent) evt;
returnedEvent = new SubOrchestrationInstanceCreatedEvent(sourceEvent.EventId)
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
Name = sourceEvent.Name,
Version = sourceEvent.Version,
Input = "[..snipped..]",
};
}
else if (evt is SubOrchestrationInstanceCompletedEvent)
{
var sourceEvent = (SubOrchestrationInstanceCompletedEvent) evt;
returnedEvent = new SubOrchestrationInstanceCompletedEvent(sourceEvent.EventId,
sourceEvent.TaskScheduledId, "[..snipped..]")
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
else if (evt is TaskFailedEvent)
{
var sourceEvent = (TaskFailedEvent) evt;
returnedEvent = new TaskFailedEvent(sourceEvent.EventId,
sourceEvent.TaskScheduledId, sourceEvent.Reason, "[..snipped..]")
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
else if (evt is SubOrchestrationInstanceFailedEvent)
{
var sourceEvent = (SubOrchestrationInstanceFailedEvent) evt;
returnedEvent = new SubOrchestrationInstanceFailedEvent(sourceEvent.EventId,
sourceEvent.TaskScheduledId, sourceEvent.Reason, "[..snipped..]")
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
else if (evt is ExecutionStartedEvent)
{
var sourceEvent = (ExecutionStartedEvent) evt;
returnedEvent = new ExecutionStartedEvent(sourceEvent.EventId, "[..snipped..]")
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
else if (evt is ExecutionCompletedEvent)
{
var sourceEvent = (ExecutionCompletedEvent) evt;
returnedEvent = new ExecutionCompletedEvent(sourceEvent.EventId, "[..snipped..]",
sourceEvent.OrchestrationStatus)
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
else if (evt is ExecutionTerminatedEvent)
{
var sourceEvent = (ExecutionTerminatedEvent) evt;
returnedEvent = new ExecutionTerminatedEvent(sourceEvent.EventId, "[..snipped..]")
{
Timestamp = sourceEvent.Timestamp,
IsPlayed = sourceEvent.IsPlayed,
};
}
// ContinueAsNewEvent is covered by the ExecutionCompletedEvent block
return returnedEvent;
}
void SetMarkerEvents(HistoryEvent historyEvent)
{
if (historyEvent is ExecutionStartedEvent)
{
if (ExecutionStartedEvent != null)
{
throw new InvalidOperationException(
"Multiple ExecutionStartedEvent found, potential corruption in state storage");
}
ExecutionStartedEvent = (ExecutionStartedEvent) historyEvent;
}
else if (historyEvent is ExecutionCompletedEvent)
{
if (ExecutionCompletedEvent != null)
{
throw new InvalidOperationException(
"Multiple ExecutionCompletedEvent found, potential corruption in state storage");
}
ExecutionCompletedEvent = (ExecutionCompletedEvent) historyEvent;
}
}
//выполняется в отдельном потоке
public void ExecuteBatchConnectionList(object batchConnectionListObject)
{
List <BatchConnection> batchConnectionList = (List <BatchConnection>)batchConnectionListObject;
if (batchConnectionList.Count == 0)
{
return;
}
ExecutionStartedEvent.Invoke(null, new EventArgs());
for (int i = 0; i < batchConnectionList.Count; i++)
{
if (StopFlag)
{
ExecutionEnded_EventHandler(null, new EventArgs());
return;
}
BatchConnection bConn = batchConnectionList[i];
string tmpCmd = bConn.Command;
if (!bConn.ExistsRDS)
{
PrintMsg(String.Format("{1}: файл 'rdslib.exe, необходимый для подключения, не найден. Дирректория поиска: {0}\n\n", bConn.FileNameRDSLib, bConn.SerialNumber));
ExecutionEnded_EventHandler(null, new EventArgs());
return;
}
this.Invoke((MethodInvoker) delegate()
{
timerProgressBar.Maximum = 60;
timerProgressBar.Value = 0;
});
bool fileExecutionResult = sayaniKombik.ExecuteBatchConnection(bConn);
if (fileExecutionResult)
{
if (loadDumpFile(bConn.FileNameDump, false))
{
if (tmpMeterInfo.sayaniMeterTypeInt != (int)SayaniMeterTypes.RMDImpulse2Channel)
{
this.Invoke((MethodInvoker) delegate()
{
PrintLastRecord((int)numericUpDown1.Value, false, dt, i + 1);
PrintMsg("\n");
});
}
else
{
loadDumpFile(bConn.FileNameDump, true);
}
}
if (fStreamDump != null)
{
fStreamDump.Close();
}
sayaniKombik.DeleteDumpFileAndLogs(bConn.FileNameDump);
}
else
{
PrintMsg(String.Format("{0}: не найден файл дампа {1}, ошибка чтения.\n\n", bConn.SerialNumberDec2Bytes, bConn.FileNameDump));
}
BatchFileTriedEvent.Invoke(null, new EventArgs());
}
ExecutionEndedEvent.Invoke(null, new EventArgs());
}
static TaskMessage ProcessWorkflowCompletedTaskDecision(
OrchestrationCompleteOrchestratorAction completeOrchestratorAction,
OrchestrationRuntimeState runtimeState,
bool includeDetails,
out bool continuedAsNew)
{
ExecutionCompletedEvent executionCompletedEvent;
continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew);
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result);
}
else
{
executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result,
completeOrchestratorAction.OrchestrationStatus);
}
runtimeState.AddEvent(executionCompletedEvent);
TraceHelper.TraceInstance(
runtimeState.OrchestrationStatus == OrchestrationStatus.Failed ? TraceEventType.Warning : TraceEventType.Information,
"TaskOrchestrationDispatcher-InstanceCompleted",
runtimeState.OrchestrationInstance,
"Instance Id '{0}' completed in state {1} with result: {2}",
runtimeState.OrchestrationInstance,
runtimeState.OrchestrationStatus,
completeOrchestratorAction.Result);
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-InstanceCompletionEvents",
runtimeState.OrchestrationInstance,
() => Utils.EscapeJson(DataConverter.Serialize(runtimeState.Events, true)));
// Check to see if we need to start a new execution
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
var taskMessage = new TaskMessage();
var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result)
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = runtimeState.OrchestrationInstance.InstanceId,
ExecutionId = Guid.NewGuid().ToString("N")
},
Tags = runtimeState.Tags,
ParentInstance = runtimeState.ParentInstance,
Name = runtimeState.Name,
Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version
};
taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance;
taskMessage.Event = startedEvent;
return(taskMessage);
}
// If this is a Sub Orchestration, and not tagged as fire-and-forget,
// then notify the parent by sending a complete message
if (runtimeState.ParentInstance != null &&
!OrchestrationTags.IsTaggedAsFireAndForget(runtimeState.Tags))
{
var taskMessage = new TaskMessage();
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed)
{
var subOrchestrationCompletedEvent =
new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId,
completeOrchestratorAction.Result);
taskMessage.Event = subOrchestrationCompletedEvent;
}
else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed ||
completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated)
{
var subOrchestrationFailedEvent =
new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId,
completeOrchestratorAction.Result,
includeDetails ? completeOrchestratorAction.Details : null);
taskMessage.Event = subOrchestrationFailedEvent;
}
if (taskMessage.Event != null)
{
taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance;
return(taskMessage);
}
}
return(null);
}
/// <summary>
/// Method to process a new work item
/// </summary>
/// <param name="workItem">The work item to process</param>
protected async Task <bool> OnProcessWorkItemAsync(TaskOrchestrationWorkItem workItem)
{
var messagesToSend = new List <TaskMessage>();
var timerMessages = new List <TaskMessage>();
var orchestratorMessages = new List <TaskMessage>();
var isCompleted = false;
var continuedAsNew = false;
var isInterrupted = false;
// correlation
CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext);
ExecutionStartedEvent continueAsNewExecutionStarted = null;
TaskMessage continuedAsNewMessage = null;
IList <HistoryEvent> carryOverEvents = null;
string carryOverStatus = null;
OrchestrationRuntimeState runtimeState = workItem.OrchestrationRuntimeState;
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
OrchestrationRuntimeState originalOrchestrationRuntimeState = runtimeState;
OrchestrationState instanceState = null;
if (!ReconcileMessagesWithState(workItem))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(
TraceEventType.Error,
"TaskOrchestrationDispatcher-DeletedOrchestration",
runtimeState.OrchestrationInstance?.InstanceId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
do
{
continuedAsNew = false;
continuedAsNewMessage = null;
TraceHelper.TraceInstance(
TraceEventType.Verbose,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-Begin",
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
DataConverter.Serialize(runtimeState.GetOrchestrationRuntimeStateDump(), true));
if (workItem.Cursor == null)
{
workItem.Cursor = await ExecuteOrchestrationAsync(runtimeState);
}
else
{
await ResumeOrchestrationAsync(workItem.Cursor);
}
IReadOnlyList <OrchestratorAction> decisions = workItem.Cursor.LatestDecisions.ToList();
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ExecuteUserOrchestration-End",
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count,
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-ProcessOrchestratorAction",
runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}",
decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
messagesToSend.Add(
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters));
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
timerMessages.Add(ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState));
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
orchestratorMessages.Add(
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters));
break;
case OrchestratorActionType.SendEvent:
var sendEventAction = (SendEventOrchestratorAction)decision;
orchestratorMessages.Add(
ProcessSendEventDecision(sendEventAction, runtimeState));
break;
case OrchestratorActionType.OrchestrationComplete:
OrchestrationCompleteOrchestratorAction completeDecision = (OrchestrationCompleteOrchestratorAction)decision;
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision(completeDecision, runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
// Store the event so we can rebuild the state
carryOverEvents = null;
if (continuedAsNew)
{
continuedAsNewMessage = workflowInstanceCompletedMessage;
continueAsNewExecutionStarted = workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
if (completeDecision.CarryoverEvents.Any())
{
carryOverEvents = completeDecision.CarryoverEvents.ToList();
completeDecision.CarryoverEvents.Clear();
}
}
else
{
orchestratorMessages.Add(workflowInstanceCompletedMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(
TraceEventType.Error,
"TaskOrchestrationDispatcher-UnsupportedDecisionType",
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// Underlying orchestration service provider may have a limit of messages per call, to avoid the situation
// we keep on asking the provider if message count is ok and stop processing new decisions if not.
//
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + orchestratorMessages.Count + timerMessages.Count;
if (this.orchestrationService.IsMaxMessageCountExceeded(totalMessages, runtimeState))
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-MaxMessageCountReached",
runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(
TraceEventType.Information,
"TaskOrchestrationDispatcher-OrchestrationAlreadyCompleted",
runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
timerMessages.Add(ProcessCreateTimerDecision(dummyTimer, runtimeState));
isInterrupted = true;
break;
}
}
// finish up processing of the work item
if (!continuedAsNew && runtimeState.Events.Last().EventType != EventType.OrchestratorCompleted)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, "TaskOrchestrationDispatcher-DeletingSessionState", workItem.InstanceId, "Deleting session state");
if (runtimeState.ExecutionStartedEvent != null)
{
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
}
else
{
if (continuedAsNew)
{
TraceHelper.TraceSession(
TraceEventType.Information,
"TaskOrchestrationDispatcher-UpdatingStateForContinuation",
workItem.InstanceId,
"Updating state for continuation");
runtimeState = new OrchestrationRuntimeState();
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
runtimeState.AddEvent(continueAsNewExecutionStarted);
runtimeState.Status = workItem.OrchestrationRuntimeState.Status ?? carryOverStatus;
carryOverStatus = workItem.OrchestrationRuntimeState.Status;
if (carryOverEvents != null)
{
foreach (var historyEvent in carryOverEvents)
{
runtimeState.AddEvent(historyEvent);
}
}
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
workItem.OrchestrationRuntimeState = runtimeState;
TaskOrchestration newOrchestration = this.objectManager.GetObject(
runtimeState.Name,
continueAsNewExecutionStarted.Version);
workItem.Cursor = new OrchestrationExecutionCursor(
runtimeState,
newOrchestration,
new TaskOrchestrationExecutor(
runtimeState,
newOrchestration,
orchestrationService.EventBehaviourForContinueAsNew),
latestDecisions: null);
if (workItem.LockedUntilUtc < DateTime.UtcNow.AddMinutes(1))
{
await orchestrationService.RenewTaskOrchestrationWorkItemLockAsync(workItem);
}
}
instanceState = Utils.BuildOrchestrationState(runtimeState);
}
} while (continuedAsNew);
}
workItem.OrchestrationRuntimeState = originalOrchestrationRuntimeState;
runtimeState.Status = runtimeState.Status ?? carryOverStatus;
await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync(
workItem,
runtimeState,
continuedAsNew?null : messagesToSend,
orchestratorMessages,
continuedAsNew?null : timerMessages,
continuedAsNewMessage,
instanceState);
workItem.OrchestrationRuntimeState = runtimeState;
return(isCompleted || continuedAsNew || isInterrupted);
}
async Task <OrchestrationInstance> InternalCreateOrchestrationInstanceWithRaisedEventAsync(
string orchestrationName,
string orchestrationVersion,
string orchestrationInstanceId,
object orchestrationInput,
IDictionary <string, string> orchestrationTags,
OrchestrationStatus[] dedupeStatuses,
string eventName,
object eventData)
{
if (string.IsNullOrWhiteSpace(orchestrationInstanceId))
{
orchestrationInstanceId = Guid.NewGuid().ToString("N");
}
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = orchestrationInstanceId,
ExecutionId = Guid.NewGuid().ToString("N"),
};
string serializedOrchestrationData = defaultConverter.Serialize(orchestrationInput);
var startedEvent = new ExecutionStartedEvent(-1, serializedOrchestrationData)
{
Tags = orchestrationTags,
Name = orchestrationName,
Version = orchestrationVersion,
OrchestrationInstance = orchestrationInstance
};
var taskMessages = new List <TaskMessage>
{
new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
}
};
if (eventData != null)
{
string serializedEventData = defaultConverter.Serialize(eventData);
taskMessages.Add(new TaskMessage
{
OrchestrationInstance = new OrchestrationInstance
{
InstanceId = orchestrationInstanceId,
// to ensure that the event gets raised on the running
// orchestration instance, null the execution id
// so that it will find out which execution
// it should use for processing
ExecutionId = null
},
Event = new EventRaisedEvent(-1, serializedEventData)
{
Name = eventName
}
});
}
// Raised events and create orchestration calls use different methods so get handled separately
await Task.WhenAll(taskMessages.Where(t => !(t.Event is EventRaisedEvent)).Select(sEvent => this.serviceClient.CreateTaskOrchestrationAsync(sEvent, dedupeStatuses)));
await this.serviceClient.SendTaskOrchestrationMessageBatchAsync(taskMessages.Where(t => (t.Event is EventRaisedEvent)).ToArray());
return(orchestrationInstance);
}
/// <inheritdoc />
public override async Task UpdateStateAsync(OrchestrationRuntimeState runtimeState, string instanceId, string executionId)
{
IList <HistoryEvent> newEvents = runtimeState.NewEvents;
IList <HistoryEvent> allEvents = runtimeState.Events;
var newEventListBuffer = new StringBuilder(4000);
var historyEventBatch = new TableBatchOperation();
EventType?orchestratorEventType = null;
string eTagValue = this.GetETagValue(instanceId);
DynamicTableEntity orchestrationInstanceUpdate = new DynamicTableEntity(instanceId, "")
{
Properties =
{
["CustomStatus"] = new EntityProperty(runtimeState.Status),
["ExecutionId"] = new EntityProperty(executionId),
["LastUpdatedTime"] = new EntityProperty(newEvents.Last().Timestamp),
}
};
for (int i = 0; i < newEvents.Count; i++)
{
HistoryEvent historyEvent = newEvents[i];
DynamicTableEntity entity = this.tableEntityConverter.ConvertToTableEntity(historyEvent);
await this.CompressLargeMessageAsync(entity);
newEventListBuffer.Append(historyEvent.EventType.ToString()).Append(',');
// The row key is the sequence number, which represents the chronological ordinal of the event.
long sequenceNumber = i + (allEvents.Count - newEvents.Count);
entity.RowKey = sequenceNumber.ToString("X16");
entity.PartitionKey = instanceId;
entity.Properties["ExecutionId"] = new EntityProperty(executionId);
// Replacement can happen if the orchestration episode gets replayed due to a commit failure in one of the steps below.
historyEventBatch.InsertOrReplace(entity);
// Table storage only supports inserts of up to 100 entities at a time.
if (historyEventBatch.Count == 99)
{
eTagValue = await this.UploadHistoryBatch(instanceId, executionId, historyEventBatch, newEventListBuffer, newEvents.Count, eTagValue);
// Reset local state for the next batch
newEventListBuffer.Clear();
historyEventBatch.Clear();
}
// Monitor for orchestration instance events
switch (historyEvent.EventType)
{
case EventType.ExecutionStarted:
orchestratorEventType = historyEvent.EventType;
ExecutionStartedEvent executionStartedEvent = (ExecutionStartedEvent)historyEvent;
orchestrationInstanceUpdate.Properties["Name"] = new EntityProperty(executionStartedEvent.Name);
orchestrationInstanceUpdate.Properties["Version"] = new EntityProperty(executionStartedEvent.Version);
orchestrationInstanceUpdate.Properties["CreatedTime"] = new EntityProperty(executionStartedEvent.Timestamp);
orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Running.ToString());
this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, InputProperty, InputProperty, executionStartedEvent.Input);
break;
case EventType.ExecutionCompleted:
orchestratorEventType = historyEvent.EventType;
ExecutionCompletedEvent executionCompleted = (ExecutionCompletedEvent)historyEvent;
this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, ResultProperty, OutputProperty, executionCompleted.Result);
orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(executionCompleted.OrchestrationStatus.ToString());
break;
case EventType.ExecutionTerminated:
orchestratorEventType = historyEvent.EventType;
ExecutionTerminatedEvent executionTerminatedEvent = (ExecutionTerminatedEvent)historyEvent;
this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, InputProperty, OutputProperty, executionTerminatedEvent.Input);
orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.Terminated.ToString());
break;
case EventType.ContinueAsNew:
orchestratorEventType = historyEvent.EventType;
ExecutionCompletedEvent executionCompletedEvent = (ExecutionCompletedEvent)historyEvent;
this.SetTablePropertyForMessage(entity, orchestrationInstanceUpdate, ResultProperty, OutputProperty, executionCompletedEvent.Result);
orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.ContinuedAsNew.ToString());
break;
}
}
// First persistence step is to commit history to the history table. Messages must come after.
if (historyEventBatch.Count > 0)
{
eTagValue = await this.UploadHistoryBatch(instanceId, executionId, historyEventBatch, newEventListBuffer, newEvents.Count, eTagValue);
}
if (orchestratorEventType == EventType.ExecutionCompleted ||
orchestratorEventType == EventType.ExecutionFailed ||
orchestratorEventType == EventType.ExecutionTerminated)
{
this.eTagValues.TryRemove(instanceId, out _);
}
else
{
this.eTagValues[instanceId] = eTagValue;
}
Stopwatch orchestrationInstanceUpdateStopwatch = Stopwatch.StartNew();
await this.instancesTable.ExecuteAsync(TableOperation.InsertOrMerge(orchestrationInstanceUpdate));
this.stats.StorageRequests.Increment();
this.stats.TableEntitiesWritten.Increment();
AnalyticsEventSource.Log.InstanceStatusUpdate(
this.storageAccountName,
this.taskHubName,
instanceId,
executionId,
orchestratorEventType?.ToString() ?? string.Empty,
orchestrationInstanceUpdateStopwatch.ElapsedMilliseconds);
}
/// <summary>
/// Create a new orchestration of the specified name and version
/// </summary>
/// <param name="name">Name of the orchestration as specified by the ObjectCreator</param>
/// <param name="version">Name of the orchestration as specified by the ObjectCreator</param>
/// <param name="instanceId">Instance id for the orchestration to be created, must be unique across the Task Hub</param>
/// <param name="input">Input parameter to the specified TaskOrchestration</param>
/// <param name="tags">Dictionary of key/value tags associated with this instance</param>
/// <returns>OrchestrationInstance that represents the orchestration that was created</returns>
public async Task<OrchestrationInstance> CreateOrchestrationInstanceAsync(string name, string version,
string instanceId,
object input, IDictionary<string, string> tags)
{
if (string.IsNullOrWhiteSpace(instanceId))
{
instanceId = Guid.NewGuid().ToString("N");
}
var orchestrationInstance = new OrchestrationInstance
{
InstanceId = instanceId,
ExecutionId = Guid.NewGuid().ToString("N"),
};
string serializedInput = defaultConverter.Serialize(input);
string serializedtags = tags != null ? defaultConverter.Serialize(tags) : null;
var startedEvent = new ExecutionStartedEvent(-1, serializedInput)
{
Tags = serializedtags,
Name = name,
Version = version,
OrchestrationInstance = orchestrationInstance
};
var taskMessage = new TaskMessage
{
OrchestrationInstance = orchestrationInstance,
Event = startedEvent
};
BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(taskMessage,
settings.MessageCompressionSettings);
brokeredMessage.SessionId = instanceId;
MessageSender sender =
await messagingFactory.CreateMessageSenderAsync(orchestratorEntityName).ConfigureAwait(false);
await sender.SendAsync(brokeredMessage).ConfigureAwait(false);
await sender.CloseAsync().ConfigureAwait(false);
return orchestrationInstance;
}
static TaskMessage ProcessWorkflowCompletedTaskDecision(
OrchestrationCompleteOrchestratorAction completeOrchestratorAction, OrchestrationRuntimeState runtimeState,
bool includeDetails, out bool continuedAsNew)
{
ExecutionCompletedEvent executionCompletedEvent;
continuedAsNew = (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew);
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
executionCompletedEvent = new ContinueAsNewEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result);
}
else
{
executionCompletedEvent = new ExecutionCompletedEvent(completeOrchestratorAction.Id,
completeOrchestratorAction.Result,
completeOrchestratorAction.OrchestrationStatus);
}
runtimeState.AddEvent(executionCompletedEvent);
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Instance Id '{0}' completed in state {1} with result: {2}",
runtimeState.OrchestrationInstance, runtimeState.OrchestrationStatus, completeOrchestratorAction.Result);
string history = JsonConvert.SerializeObject(runtimeState.Events, Formatting.Indented,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Objects
});
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
() => Utils.EscapeJson(history));
// Check to see if we need to start a new execution
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.ContinuedAsNew)
{
var taskMessage = new TaskMessage();
var startedEvent = new ExecutionStartedEvent(-1, completeOrchestratorAction.Result);
startedEvent.OrchestrationInstance = new OrchestrationInstance
{
InstanceId = runtimeState.OrchestrationInstance.InstanceId,
ExecutionId = Guid.NewGuid().ToString("N")
};
startedEvent.Tags = runtimeState.Tags;
startedEvent.ParentInstance = runtimeState.ParentInstance;
startedEvent.Name = runtimeState.Name;
startedEvent.Version = completeOrchestratorAction.NewVersion ?? runtimeState.Version;
taskMessage.OrchestrationInstance = startedEvent.OrchestrationInstance;
taskMessage.Event = startedEvent;
return(taskMessage);
}
// If this is a Sub Orchestration than notify the parent by sending a complete message
if (runtimeState.ParentInstance != null)
{
var taskMessage = new TaskMessage();
if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Completed)
{
var subOrchestrationCompletedEvent =
new SubOrchestrationInstanceCompletedEvent(-1, runtimeState.ParentInstance.TaskScheduleId,
completeOrchestratorAction.Result);
taskMessage.Event = subOrchestrationCompletedEvent;
}
else if (completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Failed ||
completeOrchestratorAction.OrchestrationStatus == OrchestrationStatus.Terminated)
{
var subOrchestrationFailedEvent =
new SubOrchestrationInstanceFailedEvent(-1, runtimeState.ParentInstance.TaskScheduleId,
completeOrchestratorAction.Result,
includeDetails ? completeOrchestratorAction.Details : null);
taskMessage.Event = subOrchestrationFailedEvent;
}
if (taskMessage.Event != null)
{
taskMessage.OrchestrationInstance = runtimeState.ParentInstance.OrchestrationInstance;
return(taskMessage);
}
}
return(null);
}
/// <inheritdoc /> public abstract Task <bool> SetNewExecutionAsync(ExecutionStartedEvent executionStartedEvent, bool ignoreExistingInstances, string inputStatusOverride);
protected override async Task OnProcessWorkItem(SessionWorkItem sessionWorkItem)
{
var messagesToSend = new List <BrokeredMessage>();
var timerMessages = new List <BrokeredMessage>();
var subOrchestrationMessages = new List <BrokeredMessage>();
bool isCompleted = false;
bool continuedAsNew = false;
BrokeredMessage continuedAsNewMessage = null;
ExecutionStartedEvent continueAsNewExecutionStarted = null;
MessageSession session = sessionWorkItem.Session;
IEnumerable <BrokeredMessage> newMessages = sessionWorkItem.Messages ?? new List <BrokeredMessage>();
long rawSessionStateSize;
long newSessionStateSize;
bool isEmptySession = false;
OrchestrationRuntimeState runtimeState;
using (Stream rawSessionStream = await session.GetStateAsync())
using (Stream sessionStream = await Utils.GetDecompressedStreamAsync(rawSessionStream))
{
isEmptySession = sessionStream == null;
rawSessionStateSize = isEmptySession ? 0 : rawSessionStream.Length;
newSessionStateSize = isEmptySession ? 0 : sessionStream.Length;
runtimeState = GetOrCreateInstanceState(sessionStream, session.SessionId);
}
TraceHelper.TraceSession(TraceEventType.Information, session.SessionId,
"Size of session state is {0}, compressed {1}", newSessionStateSize, rawSessionStateSize);
runtimeState.AddEvent(new OrchestratorStartedEvent(-1));
if (!(await ReconcileMessagesWithStateAsync(session.SessionId, runtimeState, newMessages)))
{
// TODO : mark an orchestration as faulted if there is data corruption
TraceHelper.TraceSession(TraceEventType.Error, session.SessionId,
"Received result for a deleted orchestration");
isCompleted = true;
}
else
{
TraceHelper.TraceInstance(
TraceEventType.Verbose,
runtimeState.OrchestrationInstance,
"Executing user orchestration: {0}",
JsonConvert.SerializeObject(runtimeState.GetOrchestrationRuntimeStateDump(),
new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto,
Formatting = Formatting.Indented
}));
IEnumerable <OrchestratorAction> decisions = ExecuteOrchestration(runtimeState);
TraceHelper.TraceInstance(TraceEventType.Information,
runtimeState.OrchestrationInstance,
"Executed user orchestration. Received {0} orchestrator actions: {1}",
decisions.Count(),
string.Join(", ", decisions.Select(d => d.Id + ":" + d.OrchestratorActionType)));
foreach (OrchestratorAction decision in decisions)
{
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Processing orchestrator action of type {0}", decision.OrchestratorActionType);
switch (decision.OrchestratorActionType)
{
case OrchestratorActionType.ScheduleOrchestrator:
TaskMessage taskMessage =
ProcessScheduleTaskDecision((ScheduleTaskOrchestratorAction)decision, runtimeState,
IncludeParameters);
BrokeredMessage brokeredMessage = Utils.GetBrokeredMessageFromObject(
taskMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance,
"ScheduleTask");
brokeredMessage.SessionId = session.SessionId;
messagesToSend.Add(brokeredMessage);
break;
case OrchestratorActionType.CreateTimer:
var timerOrchestratorAction = (CreateTimerOrchestratorAction)decision;
TaskMessage timerMessage = ProcessCreateTimerDecision(timerOrchestratorAction, runtimeState);
BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject(
timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance,
"Timer");
brokeredTimerMessage.ScheduledEnqueueTimeUtc = timerOrchestratorAction.FireAt;
brokeredTimerMessage.SessionId = session.SessionId;
timerMessages.Add(brokeredTimerMessage);
break;
case OrchestratorActionType.CreateSubOrchestration:
var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision;
TaskMessage createSubOrchestrationInstanceMessage =
ProcessCreateSubOrchestrationInstanceDecision(createSubOrchestrationAction,
runtimeState, IncludeParameters);
BrokeredMessage createSubOrchestrationMessage = Utils.GetBrokeredMessageFromObject(
createSubOrchestrationInstanceMessage, settings.MessageCompressionSettings,
runtimeState.OrchestrationInstance, "Schedule Suborchestration");
createSubOrchestrationMessage.SessionId =
createSubOrchestrationInstanceMessage.OrchestrationInstance.InstanceId;
subOrchestrationMessages.Add(createSubOrchestrationMessage);
break;
case OrchestratorActionType.OrchestrationComplete:
TaskMessage workflowInstanceCompletedMessage =
ProcessWorkflowCompletedTaskDecision((OrchestrationCompleteOrchestratorAction)decision,
runtimeState, IncludeDetails, out continuedAsNew);
if (workflowInstanceCompletedMessage != null)
{
// Send complete message to parent workflow or to itself to start a new execution
BrokeredMessage workflowCompletedBrokeredMessage = Utils.GetBrokeredMessageFromObject(
workflowInstanceCompletedMessage, settings.MessageCompressionSettings,
runtimeState.OrchestrationInstance, "Complete Suborchestration");
workflowCompletedBrokeredMessage.SessionId =
workflowInstanceCompletedMessage.OrchestrationInstance.InstanceId;
// Store the event so we can rebuild the state
if (continuedAsNew)
{
continuedAsNewMessage = workflowCompletedBrokeredMessage;
continueAsNewExecutionStarted =
workflowInstanceCompletedMessage.Event as ExecutionStartedEvent;
}
else
{
subOrchestrationMessages.Add(workflowCompletedBrokeredMessage);
}
}
isCompleted = !continuedAsNew;
break;
default:
throw TraceHelper.TraceExceptionInstance(TraceEventType.Error,
runtimeState.OrchestrationInstance,
new NotSupportedException("decision type not supported"));
}
// We cannot send more than 100 messages within a transaction, to avoid the situation
// we keep on checking the message count and stop processing the new decisions.
// We also put in a fake timer to force next orchestration task for remaining messages
int totalMessages = messagesToSend.Count + subOrchestrationMessages.Count + timerMessages.Count;
// Also add tracking messages as they contribute to total messages within transaction
if (isTrackingEnabled)
{
totalMessages += runtimeState.NewEvents.Count;
}
if (totalMessages > MaxMessageCount)
{
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"MaxMessageCount reached. Adding timer to process remaining events in next attempt.");
if (isCompleted || continuedAsNew)
{
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Orchestration already completed. Skip adding timer for splitting messages.");
break;
}
var dummyTimer = new CreateTimerOrchestratorAction
{
Id = FrameworkConstants.FakeTimerIdToSplitDecision,
FireAt = DateTime.UtcNow
};
TaskMessage timerMessage = ProcessCreateTimerDecision(dummyTimer, runtimeState);
BrokeredMessage brokeredTimerMessage = Utils.GetBrokeredMessageFromObject(
timerMessage, settings.MessageCompressionSettings, runtimeState.OrchestrationInstance,
"MaxMessageCount Timer");
brokeredTimerMessage.ScheduledEnqueueTimeUtc = dummyTimer.FireAt;
brokeredTimerMessage.SessionId = session.SessionId;
timerMessages.Add(brokeredTimerMessage);
break;
}
}
}
// TODO : make async, transactions are a bit tricky
using (var ts = new TransactionScope())
{
bool isSessionSizeThresholdExceeded = false;
if (!continuedAsNew)
{
runtimeState.AddEvent(new OrchestratorCompletedEvent(-1));
}
using (var ms = new MemoryStream())
{
if (isCompleted)
{
TraceHelper.TraceSession(TraceEventType.Information, session.SessionId, "Deleting session state");
// if session was already null and we finished the orchestration then no change is required
if (!isEmptySession)
{
session.SetState(null);
}
runtimeState.Size = 0;
runtimeState.CompressedSize = 0;
}
else
{
if (ms.Position != 0)
{
throw TraceHelper.TraceExceptionInstance(TraceEventType.Error,
runtimeState.OrchestrationInstance,
new ArgumentException("Instance state stream is partially consumed"));
}
IList <HistoryEvent> newState = runtimeState.Events;
if (continuedAsNew)
{
TraceHelper.TraceSession(TraceEventType.Information, session.SessionId,
"Updating state for continuation");
newState = new List <HistoryEvent>();
newState.Add(new OrchestratorStartedEvent(-1));
newState.Add(continueAsNewExecutionStarted);
newState.Add(new OrchestratorCompletedEvent(-1));
}
string serializedState = JsonConvert.SerializeObject(newState,
new JsonSerializerSettings {
TypeNameHandling = TypeNameHandling.Auto
});
var writer = new StreamWriter(ms);
writer.Write(serializedState);
writer.Flush();
ms.Position = 0;
using (Stream compressedState =
settings.TaskOrchestrationDispatcherSettings.CompressOrchestrationState
? Utils.GetCompressedStream(ms)
: ms)
{
runtimeState.Size = ms.Length;
runtimeState.CompressedSize = compressedState.Length;
if (runtimeState.CompressedSize > SessionStreamTerminationThresholdInBytes)
{
// basic idea is to simply enqueue a terminate message just like how we do it from taskhubclient
// it is possible to have other messages in front of the queue and those will get processed before
// the terminate message gets processed. but that is ok since in the worst case scenario we will
// simply land in this if-block again and end up queuing up another terminate message.
//
// the interesting scenario is when the second time we *dont* land in this if-block because e.g.
// the new messages that we processed caused a new generation to be created. in that case
// it is still ok because the worst case scenario is that we will terminate a newly created generation
// which shouldn't have been created at all in the first place
isSessionSizeThresholdExceeded = true;
TraceHelper.TraceSession(TraceEventType.Critical, session.SessionId,
"Size of session state " + runtimeState.CompressedSize +
" has exceeded termination threshold of "
+ SessionStreamTerminationThresholdInBytes);
string reason =
string.Format(
"Session state size of {0} exceeded the termination threshold of {1} bytes",
runtimeState.CompressedSize, SessionStreamTerminationThresholdInBytes);
BrokeredMessage forcedTerminateMessage = CreateForcedTerminateMessage(
runtimeState.OrchestrationInstance.InstanceId, reason);
orchestratorQueueClient.Send(forcedTerminateMessage);
}
else
{
session.SetState(compressedState);
}
}
writer.Close();
}
}
if (!isSessionSizeThresholdExceeded)
{
if (runtimeState.CompressedSize > SessionStreamWarningSizeInBytes)
{
TraceHelper.TraceSession(TraceEventType.Error, session.SessionId,
"Size of session state is nearing session size limit of 256KB");
}
if (!continuedAsNew)
{
if (messagesToSend.Count > 0)
{
messagesToSend.ForEach(m => workerSender.Send(m));
}
if (timerMessages.Count > 0)
{
timerMessages.ForEach(m => orchestratorQueueClient.Send(m));
}
}
if (subOrchestrationMessages.Count > 0)
{
subOrchestrationMessages.ForEach(m => orchestratorQueueClient.Send(m));
}
if (continuedAsNewMessage != null)
{
orchestratorQueueClient.Send(continuedAsNewMessage);
}
if (isTrackingEnabled)
{
List <BrokeredMessage> trackingMessages = CreateTrackingMessages(runtimeState);
TraceHelper.TraceInstance(TraceEventType.Information, runtimeState.OrchestrationInstance,
"Created {0} tracking messages", trackingMessages.Count);
if (trackingMessages.Count > 0)
{
trackingMessages.ForEach(m => trackingSender.Send(m));
}
}
}
IEnumerable <Guid> lockTokens = newMessages.Select(m => m.LockToken);
session.CompleteBatch(lockTokens);
ts.Complete();
}
}
