/// <summary>
/// Launch Worker method in the controlThread.
/// </summary>
/// <param name="stepExecution"></param>
/// <param name="threadWait"></param>
public void Worker(StepExecution stepExecution, AutoResetEvent threadWait)
{
// workerStartedMessage includes stepname and worker id
string workerStartedMessage = stepExecution.StepName + dot + stepExecution.remoteChunking.WorkerID.ToString();
// worker send workerStartedMessage in the workerStarted queue for master job to check
stepExecution.remoteChunking._workerStartedQueue.Send(workerStartedMessage);
int maxMasterWaitWorkerSecond = stepExecution.remoteChunking.MaxMasterWaitWorkerSecond;
int _workerTimerseconds = maxMasterWaitWorkerSecond * 1000;
// start a timer to let master check worker is still alive or not in every defualt seconds
Timer timer = new Timer(WorkerTimerMethod, stepExecution, 0, _workerTimerseconds);
bool Isterminate = false;
while (!Isterminate)
{
// send back signal to the afterStep and stop timer when batch failed
ControlQueue workerCompletedQueue = stepExecution.remoteChunking._workerCompletedQueue;
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
timer.Dispose();
Isterminate = true;
}
// when batch completed send back signal to the afterStep and stop timer after send completedmessage to message queue
else if ("COMPLETED".Equals(stepExecution.ExitStatus.ExitCode))
{
string workerCompletedMessage = stepExecution.JobExecution.JobInstance.JobName + dot + stepExecution.remoteChunking.WorkerID + dot + "COMPLETED";
workerCompletedQueue.Send(workerCompletedMessage);
timer.Dispose();
threadWait.Set();
Isterminate = true;
}
}
}
/// <summary>
/// Launch Master timer method in the controlThread.
/// </summary>
/// <param name="stepExectuionObject"></param>
private void MasterTimerMethod(object stepExectuionObject)
{
StepExecution stepExecution = (StepExecution)stepExectuionObject;
Logger.Debug("-------------------------------------------- Master Timer --------------------------------------------");
string stepName = stepExecution.StepName;
ControlQueue masterLifeLineQueue = stepExecution.remoteChunking._masterLifeLineQueue;
string masterNoAliveMessage = stepName + dot + bool.FalseString;
// stop timer when batch failed
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
masterLifeLineQueue.Send(masterNoAliveMessage);
return;
}
else // continue update workerMap in the stepExecution and check workers alive
{
Dictionary <string, bool> workerMap = stepExecution.remoteChunking._workerMap;
List <string> workerIDList = stepExecution.remoteChunking._workerStartedQueue.GetWorkerIDByMasterName(stepExecution.StepName);
foreach (string workerID in workerIDList)
{
if (!workerMap.ContainsKey(workerID))
{
workerMap[workerID] = true;
}
}
ControlQueue workerLifeLineQueue = stepExecution.remoteChunking._workerLifeLineQueue;
List <string> workerIDs = new List <string>(workerMap.Keys);
workerLifeLineQueue.CheckMessageExistAndConsumeAll(workerIDs, workerMap);
}
}
private static async Task ControlDevice(ChatSession session)
{
switch (session.CurrentMessage)
{
case ENCENDER:
case APAGAR:
ControlQueue controlQueueDesactivar = new ControlQueue()
{
ControlId = session.SelectedControl.ControlId,
InsertedAt = DateTime.UtcNow,
Value = session.CurrentMessage == ENCENDER ? 1 : 0
};
await DataService.ControlQueues.InsertControlQueue(controlQueueDesactivar);
await TelegramBotManager.SendMessage(session.ChatId, $"El control ({session.SelectedControl.KeyItem}) ha sido {(session.CurrentMessage == ENCENDER ? "encendido" : "apagado")}, Que deseas realizar?", ControlMenuKeyboard);
session.Status = ChatStatus.CONTROL_SELECTED;
break;
case REGRESAR:
await TelegramBotManager.SendMessage(session.ChatId, $"Control seleccionado ({session.SelectedControl.KeyItem})", ControlMenuKeyboard);
session.Status = ChatStatus.CONTROL_SELECTED;
break;
default:
await TelegramBotManager.SendMessageSameKeyboard(session.ChatId, "Entrada no valida");
break;
}
}
async Task DequeueLoop(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken)
{
this.settings.Logger.PartitionManagerInfo(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Started listening for messages on queue {controlQueue.Name}.");
while (!controlQueue.IsReleased)
{
try
{
// Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together.
// Both the dequeue traces and the processing traces will share the same "related" trace activity ID.
Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false);
// This will block until either new messages arrive or the queue is released.
IReadOnlyList <MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken);
if (messages.Count > 0)
{
// De-dupe any execution started messages
IEnumerable <MessageData> filteredMessages = await this.DedupeExecutionStartedMessagesAsync(
controlQueue,
messages,
traceActivityId,
cancellationToken);
this.AddMessageToPendingOrchestration(controlQueue, filteredMessages, traceActivityId, cancellationToken);
}
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// shutting down
break;
}
catch (Exception e)
{
this.settings.Logger.PartitionManagerWarning(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Exception in the dequeue loop for control queue {controlQueue.Name}. Exception: {e}");
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
this.settings.Logger.PartitionManagerInfo(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Stopped listening for messages on queue {controlQueue.Name}.");
}
private static async Task <int> GetControlByControlId(int controlId)
{
ControlQueue controlQueue = await DataService.ControlQueues.FirstControlQueue(controlId);
if (controlQueue == null)
{
return(-1);
}
await DataService.ControlQueues.DeleteControlQueue(controlQueue);
return(controlQueue.Value);
}
public bool ResumeListeningIfOwnQueue(string partitionId, ControlQueue controlQueue, CancellationToken shutdownToken)
{
if (this.ownedControlQueues.TryGetValue(partitionId, out ControlQueue ownedControlQueue))
{
if (ownedControlQueue.IsReleased)
{
// The easiest way to resume listening is to re-add a new queue that has not been released.
this.RemoveQueue(partitionId, null, "OrchestrationSessionManager ResumeListeningIfOwnQueue");
this.AddQueue(partitionId, controlQueue, shutdownToken);
}
}
return(false);
}
public void AddQueue(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken)
{
if (this.ownedControlQueues.TryAdd(partitionId, controlQueue))
{
Task.Run(() => this.DequeueLoop(partitionId, controlQueue, cancellationToken));
}
else
{
this.settings.Logger.PartitionManagerWarning(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Attempted to add a control queue {controlQueue.Name} multiple times!");
}
}
IReadOnlyList <MessageData> FilterOutExecutionStartedForDedupeValidation(
ControlQueue controlQueue,
IReadOnlyList <MessageData> messages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
List <MessageData>?executionStartedMessages = null;
List <MessageData> otherMessages = new List <MessageData>(messages.Count);
foreach (MessageData message in messages)
{
// We do de-duplication when creating top-level orchestrations but not for sub-orchestrations.
// De-dupe protection for sub-orchestrations is not implemented and will require a bit more work.
if (message.TaskMessage.Event is ExecutionStartedEvent startEvent &&
startEvent.ParentInstance == null)
{
executionStartedMessages ??= new List <MessageData>(messages.Count);
executionStartedMessages.Add(message);
}
private static ControlQueue GetControlQueue(string queuename, string hostname, string username, string password)
{
try
{
QueueConnectionProvider queueConnectionProvider = new QueueConnectionProvider();
queueConnectionProvider.UserName = username;
queueConnectionProvider.PassWord = password;
queueConnectionProvider.HostName = hostname;
ControlQueue _controlQueue = new ControlQueue();
_controlQueue.ConnectionProvider = queueConnectionProvider;
_controlQueue.QueueName = queuename;
_controlQueue.CreateQueue();
return(_controlQueue);
}
catch (Exception e)
{
throw e.InnerException;
}
}
async Task DequeueLoop(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken)
{
this.settings.Logger.PartitionManagerInfo(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Started listening for messages on queue {controlQueue.Name}.");
try
{
while (!controlQueue.IsReleased)
{
// Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together.
// Both the dequeue traces and the processing traces will share the same "related" trace activity ID.
Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false);
// This will block until either new messages arrive or the queue is released.
IReadOnlyList <MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken);
// De-dupe any execution started messages
messages = this.FilterOutExecutionStartedForDedupeValidation(controlQueue, messages, traceActivityId, cancellationToken);
if (messages.Count > 0)
{
this.AddMessageToPendingOrchestration(controlQueue, messages, traceActivityId, cancellationToken);
}
}
}
finally
{
this.settings.Logger.PartitionManagerInfo(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Stopped listening for messages on queue {controlQueue.Name}.");
}
}
async void DequeueLoop(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken)
{
AnalyticsEventSource.Log.PartitionManagerInfo(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Started listening for messages on queue {controlQueue.Name}.",
Utils.ExtensionVersion);
try
{
while (!controlQueue.IsReleased)
{
// Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together.
// Both the dequeue traces and the processing traces will share the same "related" trace activity ID.
Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope();
// This will block until either new messages arrive or the queue is released.
IReadOnlyList <MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken);
if (messages.Count > 0)
{
this.AddMessageToPendingOrchestration(controlQueue, messages, traceActivityId, cancellationToken);
}
}
}
finally
{
AnalyticsEventSource.Log.PartitionManagerInfo(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Stopped listening for messages on queue {controlQueue.Name}.",
Utils.ExtensionVersion);
}
}
/// <summary>
/// Launch Worker timer method in the controlThread.
/// </summary>
/// <param name="stepExectuionObject"></param>
private void WorkerTimerMethod(object stepExectuionObject)
{
StepExecution stepExecution = (StepExecution)stepExectuionObject;
string workerIdMessage = stepExecution.StepName + dot + stepExecution.remoteChunking.WorkerID.ToString();
Logger.Info("-------------------------------------------- Worker Timer --------------------------------------------");
ControlQueue workerLifeLineQueue = stepExecution.remoteChunking._workerLifeLineQueue;
string workerAliveMessage = workerIdMessage + dot + bool.TrueString + dot + DateTime.Now.ToString();
string workerNoAliveMessage = workerIdMessage + dot + bool.FalseString + dot + DateTime.Now.ToString();
// stop timer when batch failed
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
workerLifeLineQueue.Send(workerNoAliveMessage);
Logger.Debug(workerIdMessage + "-----------NoAlive------------------");
return;
}
else // continue send workerAliveMessage to the workerLifeLineQueue
{
Logger.Debug(workerIdMessage + "-----------Alive------------------");
workerLifeLineQueue.Send(workerAliveMessage);
}
}
public bool ResumeListeningIfOwnQueue(string partitionId, ControlQueue controlQueue, CancellationToken shutdownToken)
{
if (this.ownedControlQueues.TryGetValue(partitionId, out ControlQueue ownedControlQueue))
{
if (ownedControlQueue.IsReleased)
{
// The easiest way to resume listening is to re-add a new queue that has not been released.
this.RemoveQueue(partitionId);
this.AddQueue(partitionId, controlQueue, shutdownToken);
}
else
{
this.settings.Logger.PartitionManagerWarning(
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
partitionId,
$"Attempted to resume listening on control queue {controlQueue.Name}, which wasn't released!");
}
}
return(false);
}
async Task <IEnumerable <MessageData> > DedupeExecutionStartedMessagesAsync(
ControlQueue controlQueue,
IReadOnlyList <MessageData> messages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
if (this.settings.DisableExecutionStartedDeduplication)
{
// Allow opting out in case there is an issue
return(messages);
}
List <MessageData>?executionStartedMessages = null;
foreach (MessageData message in messages)
{
// We do de-duplication when creating top-level orchestrations but not for sub-orchestrations.
// De-dupe protection for sub-orchestrations is not implemented and will require a bit more work.
if (message.TaskMessage.Event is ExecutionStartedEvent startEvent &&
startEvent.ParentInstance == null)
{
executionStartedMessages ??= new List <MessageData>(messages.Count);
executionStartedMessages.Add(message);
}
}
if (executionStartedMessages == null)
{
// Nothing to filter
return(messages);
}
List <MessageData>?messagesToDefer = null;
List <MessageData>?messagesToDiscard = null;
IEnumerable <string> instanceIds = executionStartedMessages
.Select(msg => msg.TaskMessage.OrchestrationInstance.InstanceId)
.Distinct(StringComparer.OrdinalIgnoreCase);
// Terminology:
// "Local" -> the instance ID info comes from the local copy of the message we're examining
// "Remote" -> the instance ID info comes from the Instances table that we're querying
IList <OrchestrationState> instances = await this.trackingStore.GetStateAsync(instanceIds);
IDictionary <string, OrchestrationState> remoteOrchestrationsById =
instances.ToDictionary(o => o.OrchestrationInstance.InstanceId);
foreach (MessageData message in executionStartedMessages)
{
OrchestrationInstance localInstance = message.TaskMessage.OrchestrationInstance;
if (remoteOrchestrationsById.TryGetValue(localInstance.InstanceId, out OrchestrationState remoteInstance) &&
string.Equals(localInstance.ExecutionId, remoteInstance.OrchestrationInstance.ExecutionId, StringComparison.OrdinalIgnoreCase))
{
// Happy path: The message matches the table status. Allow it to run.
}
else if (this.IsScheduledAfterInstanceUpdate(message, remoteInstance))
{
// The message was scheduled after the Instances table was updated with the orchestration info.
// We know almost certainly that this is a redundant message and can be safely discarded because
// messages are *always* scheduled *before* the Instances table is inserted (or updated).
messagesToDiscard ??= new List <MessageData>(executionStartedMessages.Count);
messagesToDiscard.Add(message);
}
else if (message.OriginalQueueMessage.DequeueCount >= 10)
{
// We've tried and failed repeatedly to process this start message. Most likely it will never succeed, possibly because
// the client failed to update the Instances table after enqueuing this message. In such a case, the client would
// have observed a failure and will know to retry with a new message. Discard this one.
messagesToDiscard ??= new List <MessageData>(executionStartedMessages.Count);
messagesToDiscard.Add(message);
}
else
{
// This message does not match the record in the Instances table, but we don't yet know for sure if it's invalid.
// Defer it in hopes that by the time we dequeue it next we'll be more confident about whether it's valid or not.
messagesToDefer ??= new List <MessageData>(executionStartedMessages.Count);
messagesToDefer.Add(message);
}
}
// Filter out the deferred and discarded messages, making sure to preserve the original order.
IEnumerable <MessageData> filteredMessages = messages;
if (messagesToDefer?.Count > 0)
{
filteredMessages = filteredMessages.Except(messagesToDefer);
// Defer messages on a background thread to avoid blocking the dequeue loop
_ = Task.Run(() => messagesToDefer.ParallelForEachAsync(msg => controlQueue.AbandonMessageAsync(msg, session: null)));
}
if (messagesToDiscard?.Count > 0)
{
filteredMessages = filteredMessages.Except(messagesToDiscard);
// Discard messages on a background thread to avoid blocking the dequeue loop
_ = Task.Run(() => messagesToDiscard.ParallelForEachAsync(msg =>
{
this.settings.Logger.DuplicateMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
msg.TaskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(msg.TaskMessage.Event),
msg.OriginalQueueMessage.Id,
msg.TaskMessage.OrchestrationInstance.InstanceId,
msg.TaskMessage.OrchestrationInstance.ExecutionId,
controlQueue.Name,
msg.OriginalQueueMessage.DequeueCount);
return(controlQueue.DeleteMessageAsync(msg, session: null));
}));
}
return(filteredMessages);
}
public static JobExecution WorkerStart(string xmlJobFile, UnityLoader loader, string hostName, string username = "******", string password = "******", int workerUpdateTimeInterval = 15)
{
ControlQueue _controlQueue = GetControlQueue(controlQueueName, hostName, username, password);
int messageCount = _controlQueue.GetMessageCount();
XmlJob job = null;
TimeSpan WorkerUpdatetimeInterval = TimeSpan.FromSeconds(workerUpdateTimeInterval);
if (messageCount != 0)
{
for (int i = 0; i < messageCount; i++)
{
string fileName = Path.GetFileName(xmlJobFile);
string message = _controlQueue.Receive(fileName);
if (ValidateMessage(message)) // for 3 items
{
Tuple <XmlJob, string, int, bool> tuple = ValidateFileName(message, xmlJobFile);
if (tuple.Item4)
{
// Resend the message to the controlQueue
if (tuple.Item3 > 0)
{
_controlQueue.Send(tuple.Item2);
}
job = tuple.Item1;
Guid guid = Guid.NewGuid();
foreach (XmlStep step in job.JobElements)
{
step.RemoteChunking = new XmlRemoteChunking();
step.RemoteChunking.HostName = hostName;
step.RemoteChunking.Master = false;
step.RemoteChunking.WorkerID = guid.ToString();
}
break;
}
}
}
}
else
{
do
{
messageCount = _controlQueue.GetMessageCount();
if (messageCount != 0)
{
for (int i = 0; i < messageCount; i++)
{
string fileName = Path.GetFileName(xmlJobFile);
string message = _controlQueue.Receive(fileName);
if (ValidateMessage(message)) // for 3 items
{
Tuple <XmlJob, string, int, bool> tuple = ValidateFileName(message, xmlJobFile);
if (tuple.Item4)
{
// Resend the message to the controlQueue
if (tuple.Item3 > 0)
{
_controlQueue.Send(tuple.Item2);
}
job = tuple.Item1;
Guid guid = Guid.NewGuid();
foreach (XmlStep step in job.JobElements)
{
step.RemoteChunking = new XmlRemoteChunking();
step.RemoteChunking.HostName = hostName;
step.RemoteChunking.Master = false;
step.RemoteChunking.WorkerID = guid.ToString();
}
break;
}
}
}
break;
}
else
{
Logger.Info("No master job provided. Wait for worker {0} seconds.", WorkerUpdatetimeInterval.TotalSeconds);
Thread.Sleep(WorkerUpdatetimeInterval);
//throw new JobExecutionException("No master job provided");
}
} while (messageCount == 0);
}
_controlQueue.Requeue();
loader.Job = job;
var jobOperator = (SimpleJobOperator)BatchRuntime.GetJobOperator(loader);
var executionId = jobOperator.StartNextInstance(job.Id);
return(jobOperator.JobExplorer.GetJobExecution((long)executionId));
}
public static async Task InsertControlQueue(ControlQueue control)
{
control.ControlQueueId = await MySQLService.Insert(control, ControlQueue.InsertQuery);
}
public PendingMessageBatch(ControlQueue controlQueue, string instanceId, string executionId)
{
this.ControlQueue = controlQueue ?? throw new ArgumentNullException(nameof(controlQueue));
this.OrchestrationInstanceId = instanceId ?? throw new ArgumentNullException(nameof(instanceId));
this.OrchestrationExecutionId = executionId; // null is expected in some cases
}
internal void AddMessageToPendingOrchestration(
ControlQueue controlQueue,
IEnumerable <MessageData> queueMessages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
// Conditions to consider:
// 1. Do we need to create a new orchestration session or does one already exist?
// 2. Do we already have a copy of this message?
// 3. Do we need to add messages to a currently executing orchestration?
lock (this.messageAndSessionLock)
{
LinkedListNode <PendingMessageBatch> node;
var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >();
foreach (MessageData data in queueMessages)
{
string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId;
string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId;
if (this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session))
{
// If the target orchestration is already running we add the message to the session
// directly rather than adding it to the linked list. A null executionId value
// means that this is a management operation, like RaiseEvent or Terminate, which
// should be delivered to the current session.
if (executionId == null || session.Instance.ExecutionId == executionId)
{
List <MessageData> pendingMessages;
if (!existingSessionMessages.TryGetValue(session, out pendingMessages))
{
pendingMessages = new List <MessageData>();
existingSessionMessages.Add(session, pendingMessages);
}
pendingMessages.Add(data);
}
else if (data.TaskMessage.Event.Timestamp < session.RuntimeState.CreatedTime)
{
// This message was created for a previous generation of this instance.
// This is common for canceled timer fired events in ContinueAsNew scenarios.
session.DiscardMessage(data);
}
else
{
// Most likely this message was created for a new generation of the current
// instance. This can happen if a ContinueAsNew message arrives before the current
// session finished unloading. Defer the message so that it can be processed later.
session.DeferMessage(data);
}
continue;
}
// Walk backwards through the list of batches until we find one with a matching Instance ID.
// This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups.
PendingMessageBatch targetBatch = null;
node = this.pendingOrchestrationMessageBatches.Last;
while (node != null)
{
PendingMessageBatch batch = node.Value;
if (batch.OrchestrationInstanceId == instanceId)
{
if (executionId == null || batch.OrchestrationExecutionId == executionId)
{
targetBatch = batch;
break;
}
else if (batch.OrchestrationExecutionId == null)
{
targetBatch = batch;
batch.OrchestrationExecutionId = executionId;
break;
}
}
node = node.Previous;
}
if (targetBatch == null)
{
targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId);
node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch);
// Before the batch of messages can be processed, we need to download the latest execution state.
// This is done beforehand in the background as a performance optimization.
this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken);
}
// New messages are added; duplicate messages are replaced
targetBatch.Messages.AddOrReplace(data);
}
// The session might be waiting for more messages. If it is, signal them.
foreach (var pair in existingSessionMessages)
{
OrchestrationSession session = pair.Key;
List <MessageData> newMessages = pair.Value;
// New messages are added; duplicate messages are replaced
session.AddOrReplaceMessages(newMessages);
}
}
}
public T Read()
{
T item = null;
string data = null;
bool firstTimeout = true;
_maxNumberOfPolls = MaxNumberOfPolls;
while (_maxNumberOfPolls > 0)
{
// Read Message from the dataQueue
BasicGetResult result = _dataQueue.Channel.BasicGet(_dataQueue.QueueName, true);
if (result != null)
{
try
{
// Deserialize message to business object
item = SerializationUtils.DeserializeObject <T>(result.Body.ToArray());
break;
}
catch (Exception ex)
{
_logger.Debug("Json Serialize failed. Reason: " + ex.StackTrace);
throw;
}
}
else
{
// If there is no data to poll, wait for master step
_logger.Debug("Wait for {0} second for dataQueue, check for master part is completed or not", PollingTimeOut);
Thread.Sleep(TimeSpan.FromSeconds(PollingTimeOut));
_logger.Debug("Polling of number : {0}", _maxNumberOfPolls);
// For the first timeout , check master step is completed
if (!string.IsNullOrWhiteSpace(MasterName))
{
if (_masterqueue == null)
{
_masterqueue = new ControlQueue();
QueueConnectionProvider queueConnectionProvider = new QueueConnectionProvider();
queueConnectionProvider.UserName = _dataQueue.UserName;
queueConnectionProvider.PassWord = _dataQueue.PassWord;
queueConnectionProvider.HostName = _dataQueue.HostName;
_masterqueue.ConnectionProvider = queueConnectionProvider;
_masterqueue.QueueName = "master";
_masterqueue.CreateQueue();
}
string MasterCompletedMessage = "master" + dot + MasterName + dot + "COMPLETED";
if (firstTimeout)
{
_logger.Debug("First Timeout need to check the master process is completed. ");
firstTimeout = false;
if (_masterqueue.CheckMessageExistAndConsume(MasterCompletedMessage))
{
_logger.Debug("There is no more data to read for worker.");
break;
}
_logger.Debug("Need to wait for master to completed.");
}
}
_maxNumberOfPolls--;
}
}
return(item);
}
void AddMessageToPendingOrchestration(
ControlQueue controlQueue,
IEnumerable <MessageData> queueMessages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
// Conditions to consider:
// 1. Do we need to create a new orchestration session or does one already exist?
// 2. Do we already have a copy of this message?
// 3. Do we need to add messages to a currently executing orchestration?
lock (this.messageAndSessionLock)
{
LinkedListNode <PendingMessageBatch> node;
var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >();
foreach (MessageData data in queueMessages)
{
string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId;
string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId;
// If the target orchestration already running we add the message to the session
// directly rather than adding it to the linked list. A null executionId value
// means that this is a management operation, like RaiseEvent or Terminate, which
// should be delivered to the current session.
if (this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session) &&
(executionId == null || session.Instance.ExecutionId == executionId))
{
List <MessageData> pendingMessages;
if (!existingSessionMessages.TryGetValue(session, out pendingMessages))
{
pendingMessages = new List <MessageData>();
existingSessionMessages.Add(session, pendingMessages);
}
pendingMessages.Add(data);
continue;
}
// Walk backwards through the list of batches until we find one with a matching Instance ID.
// This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups.
PendingMessageBatch targetBatch = null;
node = this.pendingOrchestrationMessageBatches.Last;
while (node != null)
{
PendingMessageBatch batch = node.Value;
if (batch.OrchestrationInstanceId == instanceId &&
(executionId == null || batch.OrchestrationExecutionId == executionId))
{
targetBatch = batch;
break;
}
node = node.Previous;
}
if (targetBatch == null)
{
targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId);
node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch);
// Before the batch of messages can be processed, we need to download the latest execution state.
// This is done on another background thread so that it won't slow down dequeuing.
this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken);
}
if (targetBatch.Messages.AddOrReplace(data))
{
// Added. This is the normal path.
this.stats.PendingOrchestratorMessages.Increment();
}
else
{
// Replaced. This happens if the visibility timeout of a message expires while it
// is still sitting here in memory.
AnalyticsEventSource.Log.DuplicateMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
data.OriginalQueueMessage.Id,
instanceId,
executionId,
controlQueue.Name,
data.OriginalQueueMessage.DequeueCount,
Utils.ExtensionVersion);
}
}
// The session might be waiting for more messages. If it is, signal them.
foreach (var pair in existingSessionMessages)
{
OrchestrationSession session = pair.Key;
List <MessageData> newMessages = pair.Value;
IEnumerable <MessageData> replacements = session.AddOrReplaceMessages(newMessages);
foreach (MessageData replacementMessage in replacements)
{
AnalyticsEventSource.Log.DuplicateMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
replacementMessage.OriginalQueueMessage.Id,
session.Instance.InstanceId,
session.Instance.ExecutionId,
controlQueue.Name,
replacementMessage.OriginalQueueMessage.DequeueCount,
Utils.ExtensionVersion);
}
}
}
}
/// <summary>
/// Launch Master method in the controlThread.
/// </summary>
/// <param name="stepExecution"></param>
/// <param name="threadWait"></param>
public void Master(StepExecution stepExecution, AutoResetEvent threadWait)
{
// clean up all message queues
stepExecution.remoteChunking.CleanAllQueue();
// get workerxml name
string WorkerXml = stepExecution.remoteChunking.WorkerFileName;
// get worker max numbers
int workerMaxNumber = stepExecution.remoteChunking.WorkerMaxNumber;
// configuration information includes stepname , workerxml name, and worker max numbers
string message = stepExecution.StepName + semicolon + WorkerXml + semicolon + workerMaxNumber.ToString();
// master send configuration information in the control queue for worker job to execute
stepExecution.remoteChunking._controlQueue.Send(message);
int maxMasterWaitWorkerSecond = stepExecution.remoteChunking.MaxMasterWaitWorkerSecond;
int maxMasterWaitWorkerRetry = stepExecution.remoteChunking.MaxMasterWaitWorkerRetry;
TimeSpan _MasterWaitWorkerTimeout = TimeSpan.FromSeconds(maxMasterWaitWorkerSecond);
// check at least one worker started
if (WaitForAtLeastWorkerStarted(stepExecution, maxMasterWaitWorkerRetry, _MasterWaitWorkerTimeout))
{
// send back signal to the beforeStep
threadWait.Set();
bool Isterminate = false;
int _masterTimerseconds = maxMasterWaitWorkerSecond * 1000;
// start a timer to check worker is still alive or not in every defualt seconds
Timer timer = new Timer(MasterTimerMethod, stepExecution, 0, _masterTimerseconds);
while (!Isterminate)
{
ControlQueue masterQueue = stepExecution.remoteChunking._masterQueue;
// send back signal to the afterStep and stop timer when batch failed
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
Isterminate = true;
timer.Dispose();
threadWait.Set();
}
// when batch completed send back signal to the afterStep and stop timer after send completedmessage to message queue and check all workers completed
else if ("COMPLETED".Equals(stepExecution.ExitStatus.ExitCode))
{
string masterCompletedMessage = "master" + dot + stepExecution.StepName + dot + stepExecution.ExitStatus.ExitCode;
int maxWorkerCompleteMessageNeeded = workerMaxNumber;
while (maxWorkerCompleteMessageNeeded > 0)
{
masterQueue.Send(masterCompletedMessage);
maxWorkerCompleteMessageNeeded--;
}
List <string> failList = WaitForWorkerCompleted(stepExecution, workerMaxNumber, maxMasterWaitWorkerSecond, _MasterWaitWorkerTimeout);
timer.Dispose();
if (failList.Count > 0)
{
stepExecution.ExitStatus = ExitStatus.Failed;
}
threadWait.Set();
Isterminate = true;
}
}
}
}
public static Task DeleteControlQueue(ControlQueue control)
{
return(MySQLService.NonQuery(new[] { control }, ControlQueue.DeleteQuery));
}
internal void OnEntityCreate(MyEntity myEntity)
{
try
{
if (!Inited)
{
lock (InitObj) Init();
}
var planet = myEntity as MyPlanet;
if (planet != null)
{
PlanetMap.TryAdd(planet.EntityId, planet);
}
var grid = myEntity as MyCubeGrid;
if (grid != null)
{
grid.AddedToScene += GridAddedToScene;
}
if (!PbApiInited && myEntity is IMyProgrammableBlock)
{
PbActivate = true;
}
var placer = myEntity as IMyBlockPlacerBase;
if (placer != null && Placer == null)
{
Placer = placer;
}
var cube = myEntity as MyCubeBlock;
var sorter = cube as MyConveyorSorter;
var turret = cube as IMyLargeTurretBase;
var controllableGun = cube as IMyUserControllableGun;
var decoy = cube as IMyDecoy;
var camera = cube as MyCameraBlock;
if (sorter != null || turret != null || controllableGun != null)
{
if (!(ReplaceVanilla && VanillaIds.ContainsKey(cube.BlockDefinition.Id)) && !WeaponPlatforms.ContainsKey(cube.BlockDefinition.Id))
{
return;
}
lock (InitObj)
{
if (!SorterDetected && myEntity is MyConveyorSorter)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMyConveyorSorter>(this));
if (!EarlyInitOver)
{
ControlQueue.Enqueue(typeof(IMyConveyorSorter));
}
SorterDetected = true;
}
else if (!TurretDetected && turret != null)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMyLargeTurretBase>(this));
if (!EarlyInitOver)
{
ControlQueue.Enqueue(typeof(IMyLargeTurretBase));
}
TurretDetected = true;
}
else if (!FixedMissileReloadDetected && controllableGun is IMySmallMissileLauncherReload)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMySmallMissileLauncherReload>(this));
if (!EarlyInitOver)
{
ControlQueue.Enqueue(typeof(IMySmallMissileLauncherReload));
}
FixedMissileReloadDetected = true;
}
else if (!FixedMissileDetected && controllableGun is IMySmallMissileLauncher)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMySmallMissileLauncher>(this));
if (!EarlyInitOver)
{
ControlQueue.Enqueue(typeof(IMySmallMissileLauncher));
}
FixedMissileDetected = true;
}
else if (!FixedGunDetected && controllableGun is IMySmallGatlingGun)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMySmallGatlingGun>(this));
if (!EarlyInitOver)
{
ControlQueue.Enqueue(typeof(IMySmallGatlingGun));
}
FixedGunDetected = true;
}
}
InitComp(cube);
}
else if (decoy != null)
{
if (!DecoyDetected)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateDecoyTerminalUi <IMyDecoy>(this));
DecoyDetected = true;
}
cube.AddedToScene += DecoyAddedToScene;
}
else if (camera != null)
{
if (!CameraDetected)
{
MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateCameraTerminalUi <IMyCameraBlock>(this));
CameraDetected = true;
}
cube.AddedToScene += CameraAddedToScene;
}
}
catch (Exception ex) { Log.Line($"Exception in OnEntityCreate: {ex}"); }
}
internal void AddMessageToPendingOrchestration(
ControlQueue controlQueue,
IEnumerable <MessageData> queueMessages,
Guid traceActivityId,
CancellationToken cancellationToken)
{
// Conditions to consider:
// 1. Do we need to create a new orchestration session or does one already exist?
// 2. Do we already have a copy of this message?
// 3. Do we need to add messages to a currently executing orchestration?
lock (this.messageAndSessionLock)
{
var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >();
foreach (MessageData data in queueMessages)
{
// The instanceID identifies the orchestration across replays and ContinueAsNew generations.
// The executionID identifies a generation of an orchestration instance, doesn't change across replays.
string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId;
string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId;
// If the target orchestration is already in memory, we can potentially add the message to the session directly
// rather than adding it to the pending list. This behavior applies primarily when extended sessions are enabled.
// We can't do this for ExecutionStarted messages - those must *always* go to the pending list since they are for
// creating entirely new orchestration instances.
if (data.TaskMessage.Event.EventType != EventType.ExecutionStarted &&
this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session))
{
// A null executionId value means that this is a management operation, like RaiseEvent or Terminate, which
// should be delivered to the current session.
if (executionId == null || session.Instance.ExecutionId == executionId)
{
List <MessageData> pendingMessages;
if (!existingSessionMessages.TryGetValue(session, out pendingMessages))
{
pendingMessages = new List <MessageData>();
existingSessionMessages.Add(session, pendingMessages);
}
pendingMessages.Add(data);
continue;
}
// Looks like this message is for another generation of the active orchestration. Let it fall
// into the pending list below. If it's a message for an older generation, it will be eventually
// discarded after we discover that we have no state associated with its execution ID. This is
// most common in scenarios involving durable timers and ContinueAsNew. Otherwise, this message
// will be processed after the current session unloads.
}
PendingMessageBatch?targetBatch = null; // batch for the current instanceID-executionID pair
// Unless the message is an ExecutionStarted event, we attempt to assign the current message to an
// existing batch by walking backwards through the list of batches until we find one with a matching InstanceID.
// This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups.
LinkedListNode <PendingMessageBatch> node = this.pendingOrchestrationMessageBatches.Last;
while (node != null && data.TaskMessage.Event.EventType != EventType.ExecutionStarted)
{
PendingMessageBatch batch = node.Value;
if (batch.OrchestrationInstanceId == instanceId)
{
if (executionId == null || batch.OrchestrationExecutionId == executionId)
{
targetBatch = batch;
break;
}
else if (batch.OrchestrationExecutionId == null)
{
targetBatch = batch;
batch.OrchestrationExecutionId = executionId;
break;
}
}
node = node.Previous;
}
// If there is no batch for this instanceID-executionID pair, create one
if (targetBatch == null)
{
targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId);
node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch);
// Before the batch of messages can be processed, we need to download the latest execution state.
// This is done beforehand in the background as a performance optimization.
Task.Run(() => this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken));
}
// New messages are added; duplicate messages are replaced
targetBatch.Messages.AddOrReplace(data);
}
// The session might be waiting for more messages. If it is, signal them.
foreach (var pair in existingSessionMessages)
{
OrchestrationSession session = pair.Key;
List <MessageData> newMessages = pair.Value;
// New messages are added; duplicate messages are replaced
session.AddOrReplaceMessages(newMessages);
}
}
}
