/// <summary> /// Launch Worker method in the controlThread. /// </summary> /// <param name="stepExecution"></param> /// <param name="threadWait"></param> public void Worker(StepExecution stepExecution, AutoResetEvent threadWait) { // workerStartedMessage includes stepname and worker id string workerStartedMessage = stepExecution.StepName + dot + stepExecution.remoteChunking.WorkerID.ToString(); // worker send workerStartedMessage in the workerStarted queue for master job to check stepExecution.remoteChunking._workerStartedQueue.Send(workerStartedMessage); int maxMasterWaitWorkerSecond = stepExecution.remoteChunking.MaxMasterWaitWorkerSecond; int _workerTimerseconds = maxMasterWaitWorkerSecond * 1000; // start a timer to let master check worker is still alive or not in every defualt seconds Timer timer = new Timer(WorkerTimerMethod, stepExecution, 0, _workerTimerseconds); bool Isterminate = false; while (!Isterminate) { // send back signal to the afterStep and stop timer when batch failed ControlQueue workerCompletedQueue = stepExecution.remoteChunking._workerCompletedQueue; if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode)) { timer.Dispose(); Isterminate = true; } // when batch completed send back signal to the afterStep and stop timer after send completedmessage to message queue else if ("COMPLETED".Equals(stepExecution.ExitStatus.ExitCode)) { string workerCompletedMessage = stepExecution.JobExecution.JobInstance.JobName + dot + stepExecution.remoteChunking.WorkerID + dot + "COMPLETED"; workerCompletedQueue.Send(workerCompletedMessage); timer.Dispose(); threadWait.Set(); Isterminate = true; } } }
/// <summary> /// Launch Master timer method in the controlThread. /// </summary> /// <param name="stepExectuionObject"></param> private void MasterTimerMethod(object stepExectuionObject) { StepExecution stepExecution = (StepExecution)stepExectuionObject; Logger.Debug("-------------------------------------------- Master Timer --------------------------------------------"); string stepName = stepExecution.StepName; ControlQueue masterLifeLineQueue = stepExecution.remoteChunking._masterLifeLineQueue; string masterNoAliveMessage = stepName + dot + bool.FalseString; // stop timer when batch failed if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode)) { masterLifeLineQueue.Send(masterNoAliveMessage); return; } else // continue update workerMap in the stepExecution and check workers alive { Dictionary <string, bool> workerMap = stepExecution.remoteChunking._workerMap; List <string> workerIDList = stepExecution.remoteChunking._workerStartedQueue.GetWorkerIDByMasterName(stepExecution.StepName); foreach (string workerID in workerIDList) { if (!workerMap.ContainsKey(workerID)) { workerMap[workerID] = true; } } ControlQueue workerLifeLineQueue = stepExecution.remoteChunking._workerLifeLineQueue; List <string> workerIDs = new List <string>(workerMap.Keys); workerLifeLineQueue.CheckMessageExistAndConsumeAll(workerIDs, workerMap); } }
private static async Task ControlDevice(ChatSession session) { switch (session.CurrentMessage) { case ENCENDER: case APAGAR: ControlQueue controlQueueDesactivar = new ControlQueue() { ControlId = session.SelectedControl.ControlId, InsertedAt = DateTime.UtcNow, Value = session.CurrentMessage == ENCENDER ? 1 : 0 }; await DataService.ControlQueues.InsertControlQueue(controlQueueDesactivar); await TelegramBotManager.SendMessage(session.ChatId, $"El control ({session.SelectedControl.KeyItem}) ha sido {(session.CurrentMessage == ENCENDER ? "encendido" : "apagado")}, Que deseas realizar?", ControlMenuKeyboard); session.Status = ChatStatus.CONTROL_SELECTED; break; case REGRESAR: await TelegramBotManager.SendMessage(session.ChatId, $"Control seleccionado ({session.SelectedControl.KeyItem})", ControlMenuKeyboard); session.Status = ChatStatus.CONTROL_SELECTED; break; default: await TelegramBotManager.SendMessageSameKeyboard(session.ChatId, "Entrada no valida"); break; } }
async Task DequeueLoop(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) { this.settings.Logger.PartitionManagerInfo( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Started listening for messages on queue {controlQueue.Name}."); while (!controlQueue.IsReleased) { try { // Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together. // Both the dequeue traces and the processing traces will share the same "related" trace activity ID. Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false); // This will block until either new messages arrive or the queue is released. IReadOnlyList <MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken); if (messages.Count > 0) { // De-dupe any execution started messages IEnumerable <MessageData> filteredMessages = await this.DedupeExecutionStartedMessagesAsync( controlQueue, messages, traceActivityId, cancellationToken); this.AddMessageToPendingOrchestration(controlQueue, filteredMessages, traceActivityId, cancellationToken); } } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // shutting down break; } catch (Exception e) { this.settings.Logger.PartitionManagerWarning( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Exception in the dequeue loop for control queue {controlQueue.Name}. Exception: {e}"); Thread.Sleep(TimeSpan.FromSeconds(1)); } } this.settings.Logger.PartitionManagerInfo( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Stopped listening for messages on queue {controlQueue.Name}."); }
private static async Task <int> GetControlByControlId(int controlId) { ControlQueue controlQueue = await DataService.ControlQueues.FirstControlQueue(controlId); if (controlQueue == null) { return(-1); } await DataService.ControlQueues.DeleteControlQueue(controlQueue); return(controlQueue.Value); }
public bool ResumeListeningIfOwnQueue(string partitionId, ControlQueue controlQueue, CancellationToken shutdownToken) { if (this.ownedControlQueues.TryGetValue(partitionId, out ControlQueue ownedControlQueue)) { if (ownedControlQueue.IsReleased) { // The easiest way to resume listening is to re-add a new queue that has not been released. this.RemoveQueue(partitionId, null, "OrchestrationSessionManager ResumeListeningIfOwnQueue"); this.AddQueue(partitionId, controlQueue, shutdownToken); } } return(false); }
public void AddQueue(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) { if (this.ownedControlQueues.TryAdd(partitionId, controlQueue)) { Task.Run(() => this.DequeueLoop(partitionId, controlQueue, cancellationToken)); } else { this.settings.Logger.PartitionManagerWarning( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Attempted to add a control queue {controlQueue.Name} multiple times!"); } }
IReadOnlyList <MessageData> FilterOutExecutionStartedForDedupeValidation( ControlQueue controlQueue, IReadOnlyList <MessageData> messages, Guid traceActivityId, CancellationToken cancellationToken) { List <MessageData>?executionStartedMessages = null; List <MessageData> otherMessages = new List <MessageData>(messages.Count); foreach (MessageData message in messages) { // We do de-duplication when creating top-level orchestrations but not for sub-orchestrations. // De-dupe protection for sub-orchestrations is not implemented and will require a bit more work. if (message.TaskMessage.Event is ExecutionStartedEvent startEvent && startEvent.ParentInstance == null) { executionStartedMessages ??= new List <MessageData>(messages.Count); executionStartedMessages.Add(message); }
private static ControlQueue GetControlQueue(string queuename, string hostname, string username, string password) { try { QueueConnectionProvider queueConnectionProvider = new QueueConnectionProvider(); queueConnectionProvider.UserName = username; queueConnectionProvider.PassWord = password; queueConnectionProvider.HostName = hostname; ControlQueue _controlQueue = new ControlQueue(); _controlQueue.ConnectionProvider = queueConnectionProvider; _controlQueue.QueueName = queuename; _controlQueue.CreateQueue(); return(_controlQueue); } catch (Exception e) { throw e.InnerException; } }
async Task DequeueLoop(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) { this.settings.Logger.PartitionManagerInfo( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Started listening for messages on queue {controlQueue.Name}."); try { while (!controlQueue.IsReleased) { // Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together. // Both the dequeue traces and the processing traces will share the same "related" trace activity ID. Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(useExisting: false); // This will block until either new messages arrive or the queue is released. IReadOnlyList <MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken); // De-dupe any execution started messages messages = this.FilterOutExecutionStartedForDedupeValidation(controlQueue, messages, traceActivityId, cancellationToken); if (messages.Count > 0) { this.AddMessageToPendingOrchestration(controlQueue, messages, traceActivityId, cancellationToken); } } } finally { this.settings.Logger.PartitionManagerInfo( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Stopped listening for messages on queue {controlQueue.Name}."); } }
async void DequeueLoop(string partitionId, ControlQueue controlQueue, CancellationToken cancellationToken) { AnalyticsEventSource.Log.PartitionManagerInfo( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Started listening for messages on queue {controlQueue.Name}.", Utils.ExtensionVersion); try { while (!controlQueue.IsReleased) { // Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together. // Both the dequeue traces and the processing traces will share the same "related" trace activity ID. Guid traceActivityId = AzureStorageOrchestrationService.StartNewLogicalTraceScope(); // This will block until either new messages arrive or the queue is released. IReadOnlyList <MessageData> messages = await controlQueue.GetMessagesAsync(cancellationToken); if (messages.Count > 0) { this.AddMessageToPendingOrchestration(controlQueue, messages, traceActivityId, cancellationToken); } } } finally { AnalyticsEventSource.Log.PartitionManagerInfo( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Stopped listening for messages on queue {controlQueue.Name}.", Utils.ExtensionVersion); } }
/// <summary> /// Launch Worker timer method in the controlThread. /// </summary> /// <param name="stepExectuionObject"></param> private void WorkerTimerMethod(object stepExectuionObject) { StepExecution stepExecution = (StepExecution)stepExectuionObject; string workerIdMessage = stepExecution.StepName + dot + stepExecution.remoteChunking.WorkerID.ToString(); Logger.Info("-------------------------------------------- Worker Timer --------------------------------------------"); ControlQueue workerLifeLineQueue = stepExecution.remoteChunking._workerLifeLineQueue; string workerAliveMessage = workerIdMessage + dot + bool.TrueString + dot + DateTime.Now.ToString(); string workerNoAliveMessage = workerIdMessage + dot + bool.FalseString + dot + DateTime.Now.ToString(); // stop timer when batch failed if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode)) { workerLifeLineQueue.Send(workerNoAliveMessage); Logger.Debug(workerIdMessage + "-----------NoAlive------------------"); return; } else // continue send workerAliveMessage to the workerLifeLineQueue { Logger.Debug(workerIdMessage + "-----------Alive------------------"); workerLifeLineQueue.Send(workerAliveMessage); } }
public bool ResumeListeningIfOwnQueue(string partitionId, ControlQueue controlQueue, CancellationToken shutdownToken) { if (this.ownedControlQueues.TryGetValue(partitionId, out ControlQueue ownedControlQueue)) { if (ownedControlQueue.IsReleased) { // The easiest way to resume listening is to re-add a new queue that has not been released. this.RemoveQueue(partitionId); this.AddQueue(partitionId, controlQueue, shutdownToken); } else { this.settings.Logger.PartitionManagerWarning( this.storageAccountName, this.settings.TaskHubName, this.settings.WorkerId, partitionId, $"Attempted to resume listening on control queue {controlQueue.Name}, which wasn't released!"); } } return(false); }
async Task <IEnumerable <MessageData> > DedupeExecutionStartedMessagesAsync( ControlQueue controlQueue, IReadOnlyList <MessageData> messages, Guid traceActivityId, CancellationToken cancellationToken) { if (this.settings.DisableExecutionStartedDeduplication) { // Allow opting out in case there is an issue return(messages); } List <MessageData>?executionStartedMessages = null; foreach (MessageData message in messages) { // We do de-duplication when creating top-level orchestrations but not for sub-orchestrations. // De-dupe protection for sub-orchestrations is not implemented and will require a bit more work. if (message.TaskMessage.Event is ExecutionStartedEvent startEvent && startEvent.ParentInstance == null) { executionStartedMessages ??= new List <MessageData>(messages.Count); executionStartedMessages.Add(message); } } if (executionStartedMessages == null) { // Nothing to filter return(messages); } List <MessageData>?messagesToDefer = null; List <MessageData>?messagesToDiscard = null; IEnumerable <string> instanceIds = executionStartedMessages .Select(msg => msg.TaskMessage.OrchestrationInstance.InstanceId) .Distinct(StringComparer.OrdinalIgnoreCase); // Terminology: // "Local" -> the instance ID info comes from the local copy of the message we're examining // "Remote" -> the instance ID info comes from the Instances table that we're querying IList <OrchestrationState> instances = await this.trackingStore.GetStateAsync(instanceIds); IDictionary <string, OrchestrationState> remoteOrchestrationsById = instances.ToDictionary(o => o.OrchestrationInstance.InstanceId); foreach (MessageData message in executionStartedMessages) { OrchestrationInstance localInstance = message.TaskMessage.OrchestrationInstance; if (remoteOrchestrationsById.TryGetValue(localInstance.InstanceId, out OrchestrationState remoteInstance) && string.Equals(localInstance.ExecutionId, remoteInstance.OrchestrationInstance.ExecutionId, StringComparison.OrdinalIgnoreCase)) { // Happy path: The message matches the table status. Allow it to run. } else if (this.IsScheduledAfterInstanceUpdate(message, remoteInstance)) { // The message was scheduled after the Instances table was updated with the orchestration info. // We know almost certainly that this is a redundant message and can be safely discarded because // messages are *always* scheduled *before* the Instances table is inserted (or updated). messagesToDiscard ??= new List <MessageData>(executionStartedMessages.Count); messagesToDiscard.Add(message); } else if (message.OriginalQueueMessage.DequeueCount >= 10) { // We've tried and failed repeatedly to process this start message. Most likely it will never succeed, possibly because // the client failed to update the Instances table after enqueuing this message. In such a case, the client would // have observed a failure and will know to retry with a new message. Discard this one. messagesToDiscard ??= new List <MessageData>(executionStartedMessages.Count); messagesToDiscard.Add(message); } else { // This message does not match the record in the Instances table, but we don't yet know for sure if it's invalid. // Defer it in hopes that by the time we dequeue it next we'll be more confident about whether it's valid or not. messagesToDefer ??= new List <MessageData>(executionStartedMessages.Count); messagesToDefer.Add(message); } } // Filter out the deferred and discarded messages, making sure to preserve the original order. IEnumerable <MessageData> filteredMessages = messages; if (messagesToDefer?.Count > 0) { filteredMessages = filteredMessages.Except(messagesToDefer); // Defer messages on a background thread to avoid blocking the dequeue loop _ = Task.Run(() => messagesToDefer.ParallelForEachAsync(msg => controlQueue.AbandonMessageAsync(msg, session: null))); } if (messagesToDiscard?.Count > 0) { filteredMessages = filteredMessages.Except(messagesToDiscard); // Discard messages on a background thread to avoid blocking the dequeue loop _ = Task.Run(() => messagesToDiscard.ParallelForEachAsync(msg => { this.settings.Logger.DuplicateMessageDetected( this.storageAccountName, this.settings.TaskHubName, msg.TaskMessage.Event.EventType.ToString(), Utils.GetTaskEventId(msg.TaskMessage.Event), msg.OriginalQueueMessage.Id, msg.TaskMessage.OrchestrationInstance.InstanceId, msg.TaskMessage.OrchestrationInstance.ExecutionId, controlQueue.Name, msg.OriginalQueueMessage.DequeueCount); return(controlQueue.DeleteMessageAsync(msg, session: null)); })); } return(filteredMessages); }
public static JobExecution WorkerStart(string xmlJobFile, UnityLoader loader, string hostName, string username = "******", string password = "******", int workerUpdateTimeInterval = 15) { ControlQueue _controlQueue = GetControlQueue(controlQueueName, hostName, username, password); int messageCount = _controlQueue.GetMessageCount(); XmlJob job = null; TimeSpan WorkerUpdatetimeInterval = TimeSpan.FromSeconds(workerUpdateTimeInterval); if (messageCount != 0) { for (int i = 0; i < messageCount; i++) { string fileName = Path.GetFileName(xmlJobFile); string message = _controlQueue.Receive(fileName); if (ValidateMessage(message)) // for 3 items { Tuple <XmlJob, string, int, bool> tuple = ValidateFileName(message, xmlJobFile); if (tuple.Item4) { // Resend the message to the controlQueue if (tuple.Item3 > 0) { _controlQueue.Send(tuple.Item2); } job = tuple.Item1; Guid guid = Guid.NewGuid(); foreach (XmlStep step in job.JobElements) { step.RemoteChunking = new XmlRemoteChunking(); step.RemoteChunking.HostName = hostName; step.RemoteChunking.Master = false; step.RemoteChunking.WorkerID = guid.ToString(); } break; } } } } else { do { messageCount = _controlQueue.GetMessageCount(); if (messageCount != 0) { for (int i = 0; i < messageCount; i++) { string fileName = Path.GetFileName(xmlJobFile); string message = _controlQueue.Receive(fileName); if (ValidateMessage(message)) // for 3 items { Tuple <XmlJob, string, int, bool> tuple = ValidateFileName(message, xmlJobFile); if (tuple.Item4) { // Resend the message to the controlQueue if (tuple.Item3 > 0) { _controlQueue.Send(tuple.Item2); } job = tuple.Item1; Guid guid = Guid.NewGuid(); foreach (XmlStep step in job.JobElements) { step.RemoteChunking = new XmlRemoteChunking(); step.RemoteChunking.HostName = hostName; step.RemoteChunking.Master = false; step.RemoteChunking.WorkerID = guid.ToString(); } break; } } } break; } else { Logger.Info("No master job provided. Wait for worker {0} seconds.", WorkerUpdatetimeInterval.TotalSeconds); Thread.Sleep(WorkerUpdatetimeInterval); //throw new JobExecutionException("No master job provided"); } } while (messageCount == 0); } _controlQueue.Requeue(); loader.Job = job; var jobOperator = (SimpleJobOperator)BatchRuntime.GetJobOperator(loader); var executionId = jobOperator.StartNextInstance(job.Id); return(jobOperator.JobExplorer.GetJobExecution((long)executionId)); }
public static async Task InsertControlQueue(ControlQueue control) { control.ControlQueueId = await MySQLService.Insert(control, ControlQueue.InsertQuery); }
public PendingMessageBatch(ControlQueue controlQueue, string instanceId, string executionId) { this.ControlQueue = controlQueue ?? throw new ArgumentNullException(nameof(controlQueue)); this.OrchestrationInstanceId = instanceId ?? throw new ArgumentNullException(nameof(instanceId)); this.OrchestrationExecutionId = executionId; // null is expected in some cases }
internal void AddMessageToPendingOrchestration( ControlQueue controlQueue, IEnumerable <MessageData> queueMessages, Guid traceActivityId, CancellationToken cancellationToken) { // Conditions to consider: // 1. Do we need to create a new orchestration session or does one already exist? // 2. Do we already have a copy of this message? // 3. Do we need to add messages to a currently executing orchestration? lock (this.messageAndSessionLock) { LinkedListNode <PendingMessageBatch> node; var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >(); foreach (MessageData data in queueMessages) { string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId; string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId; if (this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session)) { // If the target orchestration is already running we add the message to the session // directly rather than adding it to the linked list. A null executionId value // means that this is a management operation, like RaiseEvent or Terminate, which // should be delivered to the current session. if (executionId == null || session.Instance.ExecutionId == executionId) { List <MessageData> pendingMessages; if (!existingSessionMessages.TryGetValue(session, out pendingMessages)) { pendingMessages = new List <MessageData>(); existingSessionMessages.Add(session, pendingMessages); } pendingMessages.Add(data); } else if (data.TaskMessage.Event.Timestamp < session.RuntimeState.CreatedTime) { // This message was created for a previous generation of this instance. // This is common for canceled timer fired events in ContinueAsNew scenarios. session.DiscardMessage(data); } else { // Most likely this message was created for a new generation of the current // instance. This can happen if a ContinueAsNew message arrives before the current // session finished unloading. Defer the message so that it can be processed later. session.DeferMessage(data); } continue; } // Walk backwards through the list of batches until we find one with a matching Instance ID. // This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups. PendingMessageBatch targetBatch = null; node = this.pendingOrchestrationMessageBatches.Last; while (node != null) { PendingMessageBatch batch = node.Value; if (batch.OrchestrationInstanceId == instanceId) { if (executionId == null || batch.OrchestrationExecutionId == executionId) { targetBatch = batch; break; } else if (batch.OrchestrationExecutionId == null) { targetBatch = batch; batch.OrchestrationExecutionId = executionId; break; } } node = node.Previous; } if (targetBatch == null) { targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId); node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch); // Before the batch of messages can be processed, we need to download the latest execution state. // This is done beforehand in the background as a performance optimization. this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken); } // New messages are added; duplicate messages are replaced targetBatch.Messages.AddOrReplace(data); } // The session might be waiting for more messages. If it is, signal them. foreach (var pair in existingSessionMessages) { OrchestrationSession session = pair.Key; List <MessageData> newMessages = pair.Value; // New messages are added; duplicate messages are replaced session.AddOrReplaceMessages(newMessages); } } }
public T Read() { T item = null; string data = null; bool firstTimeout = true; _maxNumberOfPolls = MaxNumberOfPolls; while (_maxNumberOfPolls > 0) { // Read Message from the dataQueue BasicGetResult result = _dataQueue.Channel.BasicGet(_dataQueue.QueueName, true); if (result != null) { try { // Deserialize message to business object item = SerializationUtils.DeserializeObject <T>(result.Body.ToArray()); break; } catch (Exception ex) { _logger.Debug("Json Serialize failed. Reason: " + ex.StackTrace); throw; } } else { // If there is no data to poll, wait for master step _logger.Debug("Wait for {0} second for dataQueue, check for master part is completed or not", PollingTimeOut); Thread.Sleep(TimeSpan.FromSeconds(PollingTimeOut)); _logger.Debug("Polling of number : {0}", _maxNumberOfPolls); // For the first timeout , check master step is completed if (!string.IsNullOrWhiteSpace(MasterName)) { if (_masterqueue == null) { _masterqueue = new ControlQueue(); QueueConnectionProvider queueConnectionProvider = new QueueConnectionProvider(); queueConnectionProvider.UserName = _dataQueue.UserName; queueConnectionProvider.PassWord = _dataQueue.PassWord; queueConnectionProvider.HostName = _dataQueue.HostName; _masterqueue.ConnectionProvider = queueConnectionProvider; _masterqueue.QueueName = "master"; _masterqueue.CreateQueue(); } string MasterCompletedMessage = "master" + dot + MasterName + dot + "COMPLETED"; if (firstTimeout) { _logger.Debug("First Timeout need to check the master process is completed. "); firstTimeout = false; if (_masterqueue.CheckMessageExistAndConsume(MasterCompletedMessage)) { _logger.Debug("There is no more data to read for worker."); break; } _logger.Debug("Need to wait for master to completed."); } } _maxNumberOfPolls--; } } return(item); }
void AddMessageToPendingOrchestration( ControlQueue controlQueue, IEnumerable <MessageData> queueMessages, Guid traceActivityId, CancellationToken cancellationToken) { // Conditions to consider: // 1. Do we need to create a new orchestration session or does one already exist? // 2. Do we already have a copy of this message? // 3. Do we need to add messages to a currently executing orchestration? lock (this.messageAndSessionLock) { LinkedListNode <PendingMessageBatch> node; var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >(); foreach (MessageData data in queueMessages) { string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId; string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId; // If the target orchestration already running we add the message to the session // directly rather than adding it to the linked list. A null executionId value // means that this is a management operation, like RaiseEvent or Terminate, which // should be delivered to the current session. if (this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session) && (executionId == null || session.Instance.ExecutionId == executionId)) { List <MessageData> pendingMessages; if (!existingSessionMessages.TryGetValue(session, out pendingMessages)) { pendingMessages = new List <MessageData>(); existingSessionMessages.Add(session, pendingMessages); } pendingMessages.Add(data); continue; } // Walk backwards through the list of batches until we find one with a matching Instance ID. // This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups. PendingMessageBatch targetBatch = null; node = this.pendingOrchestrationMessageBatches.Last; while (node != null) { PendingMessageBatch batch = node.Value; if (batch.OrchestrationInstanceId == instanceId && (executionId == null || batch.OrchestrationExecutionId == executionId)) { targetBatch = batch; break; } node = node.Previous; } if (targetBatch == null) { targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId); node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch); // Before the batch of messages can be processed, we need to download the latest execution state. // This is done on another background thread so that it won't slow down dequeuing. this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken); } if (targetBatch.Messages.AddOrReplace(data)) { // Added. This is the normal path. this.stats.PendingOrchestratorMessages.Increment(); } else { // Replaced. This happens if the visibility timeout of a message expires while it // is still sitting here in memory. AnalyticsEventSource.Log.DuplicateMessageDetected( this.storageAccountName, this.settings.TaskHubName, data.OriginalQueueMessage.Id, instanceId, executionId, controlQueue.Name, data.OriginalQueueMessage.DequeueCount, Utils.ExtensionVersion); } } // The session might be waiting for more messages. If it is, signal them. foreach (var pair in existingSessionMessages) { OrchestrationSession session = pair.Key; List <MessageData> newMessages = pair.Value; IEnumerable <MessageData> replacements = session.AddOrReplaceMessages(newMessages); foreach (MessageData replacementMessage in replacements) { AnalyticsEventSource.Log.DuplicateMessageDetected( this.storageAccountName, this.settings.TaskHubName, replacementMessage.OriginalQueueMessage.Id, session.Instance.InstanceId, session.Instance.ExecutionId, controlQueue.Name, replacementMessage.OriginalQueueMessage.DequeueCount, Utils.ExtensionVersion); } } } }
/// <summary> /// Launch Master method in the controlThread. /// </summary> /// <param name="stepExecution"></param> /// <param name="threadWait"></param> public void Master(StepExecution stepExecution, AutoResetEvent threadWait) { // clean up all message queues stepExecution.remoteChunking.CleanAllQueue(); // get workerxml name string WorkerXml = stepExecution.remoteChunking.WorkerFileName; // get worker max numbers int workerMaxNumber = stepExecution.remoteChunking.WorkerMaxNumber; // configuration information includes stepname , workerxml name, and worker max numbers string message = stepExecution.StepName + semicolon + WorkerXml + semicolon + workerMaxNumber.ToString(); // master send configuration information in the control queue for worker job to execute stepExecution.remoteChunking._controlQueue.Send(message); int maxMasterWaitWorkerSecond = stepExecution.remoteChunking.MaxMasterWaitWorkerSecond; int maxMasterWaitWorkerRetry = stepExecution.remoteChunking.MaxMasterWaitWorkerRetry; TimeSpan _MasterWaitWorkerTimeout = TimeSpan.FromSeconds(maxMasterWaitWorkerSecond); // check at least one worker started if (WaitForAtLeastWorkerStarted(stepExecution, maxMasterWaitWorkerRetry, _MasterWaitWorkerTimeout)) { // send back signal to the beforeStep threadWait.Set(); bool Isterminate = false; int _masterTimerseconds = maxMasterWaitWorkerSecond * 1000; // start a timer to check worker is still alive or not in every defualt seconds Timer timer = new Timer(MasterTimerMethod, stepExecution, 0, _masterTimerseconds); while (!Isterminate) { ControlQueue masterQueue = stepExecution.remoteChunking._masterQueue; // send back signal to the afterStep and stop timer when batch failed if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode)) { Isterminate = true; timer.Dispose(); threadWait.Set(); } // when batch completed send back signal to the afterStep and stop timer after send completedmessage to message queue and check all workers completed else if ("COMPLETED".Equals(stepExecution.ExitStatus.ExitCode)) { string masterCompletedMessage = "master" + dot + stepExecution.StepName + dot + stepExecution.ExitStatus.ExitCode; int maxWorkerCompleteMessageNeeded = workerMaxNumber; while (maxWorkerCompleteMessageNeeded > 0) { masterQueue.Send(masterCompletedMessage); maxWorkerCompleteMessageNeeded--; } List <string> failList = WaitForWorkerCompleted(stepExecution, workerMaxNumber, maxMasterWaitWorkerSecond, _MasterWaitWorkerTimeout); timer.Dispose(); if (failList.Count > 0) { stepExecution.ExitStatus = ExitStatus.Failed; } threadWait.Set(); Isterminate = true; } } } }
public static Task DeleteControlQueue(ControlQueue control) { return(MySQLService.NonQuery(new[] { control }, ControlQueue.DeleteQuery)); }
internal void OnEntityCreate(MyEntity myEntity) { try { if (!Inited) { lock (InitObj) Init(); } var planet = myEntity as MyPlanet; if (planet != null) { PlanetMap.TryAdd(planet.EntityId, planet); } var grid = myEntity as MyCubeGrid; if (grid != null) { grid.AddedToScene += GridAddedToScene; } if (!PbApiInited && myEntity is IMyProgrammableBlock) { PbActivate = true; } var placer = myEntity as IMyBlockPlacerBase; if (placer != null && Placer == null) { Placer = placer; } var cube = myEntity as MyCubeBlock; var sorter = cube as MyConveyorSorter; var turret = cube as IMyLargeTurretBase; var controllableGun = cube as IMyUserControllableGun; var decoy = cube as IMyDecoy; var camera = cube as MyCameraBlock; if (sorter != null || turret != null || controllableGun != null) { if (!(ReplaceVanilla && VanillaIds.ContainsKey(cube.BlockDefinition.Id)) && !WeaponPlatforms.ContainsKey(cube.BlockDefinition.Id)) { return; } lock (InitObj) { if (!SorterDetected && myEntity is MyConveyorSorter) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMyConveyorSorter>(this)); if (!EarlyInitOver) { ControlQueue.Enqueue(typeof(IMyConveyorSorter)); } SorterDetected = true; } else if (!TurretDetected && turret != null) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMyLargeTurretBase>(this)); if (!EarlyInitOver) { ControlQueue.Enqueue(typeof(IMyLargeTurretBase)); } TurretDetected = true; } else if (!FixedMissileReloadDetected && controllableGun is IMySmallMissileLauncherReload) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMySmallMissileLauncherReload>(this)); if (!EarlyInitOver) { ControlQueue.Enqueue(typeof(IMySmallMissileLauncherReload)); } FixedMissileReloadDetected = true; } else if (!FixedMissileDetected && controllableGun is IMySmallMissileLauncher) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMySmallMissileLauncher>(this)); if (!EarlyInitOver) { ControlQueue.Enqueue(typeof(IMySmallMissileLauncher)); } FixedMissileDetected = true; } else if (!FixedGunDetected && controllableGun is IMySmallGatlingGun) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateTerminalUi <IMySmallGatlingGun>(this)); if (!EarlyInitOver) { ControlQueue.Enqueue(typeof(IMySmallGatlingGun)); } FixedGunDetected = true; } } InitComp(cube); } else if (decoy != null) { if (!DecoyDetected) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateDecoyTerminalUi <IMyDecoy>(this)); DecoyDetected = true; } cube.AddedToScene += DecoyAddedToScene; } else if (camera != null) { if (!CameraDetected) { MyAPIGateway.Utilities.InvokeOnGameThread(() => CreateCameraTerminalUi <IMyCameraBlock>(this)); CameraDetected = true; } cube.AddedToScene += CameraAddedToScene; } } catch (Exception ex) { Log.Line($"Exception in OnEntityCreate: {ex}"); } }
internal void AddMessageToPendingOrchestration( ControlQueue controlQueue, IEnumerable <MessageData> queueMessages, Guid traceActivityId, CancellationToken cancellationToken) { // Conditions to consider: // 1. Do we need to create a new orchestration session or does one already exist? // 2. Do we already have a copy of this message? // 3. Do we need to add messages to a currently executing orchestration? lock (this.messageAndSessionLock) { var existingSessionMessages = new Dictionary <OrchestrationSession, List <MessageData> >(); foreach (MessageData data in queueMessages) { // The instanceID identifies the orchestration across replays and ContinueAsNew generations. // The executionID identifies a generation of an orchestration instance, doesn't change across replays. string instanceId = data.TaskMessage.OrchestrationInstance.InstanceId; string executionId = data.TaskMessage.OrchestrationInstance.ExecutionId; // If the target orchestration is already in memory, we can potentially add the message to the session directly // rather than adding it to the pending list. This behavior applies primarily when extended sessions are enabled. // We can't do this for ExecutionStarted messages - those must *always* go to the pending list since they are for // creating entirely new orchestration instances. if (data.TaskMessage.Event.EventType != EventType.ExecutionStarted && this.activeOrchestrationSessions.TryGetValue(instanceId, out OrchestrationSession session)) { // A null executionId value means that this is a management operation, like RaiseEvent or Terminate, which // should be delivered to the current session. if (executionId == null || session.Instance.ExecutionId == executionId) { List <MessageData> pendingMessages; if (!existingSessionMessages.TryGetValue(session, out pendingMessages)) { pendingMessages = new List <MessageData>(); existingSessionMessages.Add(session, pendingMessages); } pendingMessages.Add(data); continue; } // Looks like this message is for another generation of the active orchestration. Let it fall // into the pending list below. If it's a message for an older generation, it will be eventually // discarded after we discover that we have no state associated with its execution ID. This is // most common in scenarios involving durable timers and ContinueAsNew. Otherwise, this message // will be processed after the current session unloads. } PendingMessageBatch?targetBatch = null; // batch for the current instanceID-executionID pair // Unless the message is an ExecutionStarted event, we attempt to assign the current message to an // existing batch by walking backwards through the list of batches until we find one with a matching InstanceID. // This is assumed to be more efficient than walking forward if most messages arrive in the queue in groups. LinkedListNode <PendingMessageBatch> node = this.pendingOrchestrationMessageBatches.Last; while (node != null && data.TaskMessage.Event.EventType != EventType.ExecutionStarted) { PendingMessageBatch batch = node.Value; if (batch.OrchestrationInstanceId == instanceId) { if (executionId == null || batch.OrchestrationExecutionId == executionId) { targetBatch = batch; break; } else if (batch.OrchestrationExecutionId == null) { targetBatch = batch; batch.OrchestrationExecutionId = executionId; break; } } node = node.Previous; } // If there is no batch for this instanceID-executionID pair, create one if (targetBatch == null) { targetBatch = new PendingMessageBatch(controlQueue, instanceId, executionId); node = this.pendingOrchestrationMessageBatches.AddLast(targetBatch); // Before the batch of messages can be processed, we need to download the latest execution state. // This is done beforehand in the background as a performance optimization. Task.Run(() => this.ScheduleOrchestrationStatePrefetch(node, traceActivityId, cancellationToken)); } // New messages are added; duplicate messages are replaced targetBatch.Messages.AddOrReplace(data); } // The session might be waiting for more messages. If it is, signal them. foreach (var pair in existingSessionMessages) { OrchestrationSession session = pair.Key; List <MessageData> newMessages = pair.Value; // New messages are added; duplicate messages are replaced session.AddOrReplaceMessages(newMessages); } } }