public void MapValueEnqueues()
        {
            var instance = QueueMapper <MyOptions, int> .Create(opt => opt.Queue);

            var options = new MyOptions();

            instance.MapValue(options, 10);
            options.Queue.Single().ShouldBe(10);
        }
        public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
        {
            using (var dbContext = _dbContextFactory())
            {
                var executionStartedEvent = creationMessage.Event as ExecutionStartedEvent;

                var instanceId  = creationMessage.OrchestrationInstance.InstanceId;
                var executionId = creationMessage.OrchestrationInstance.ExecutionId;

                var instance = await dbContext.Instances
                               .Include(i => i.LastExecution)
                               .FirstOrDefaultAsync(i => i.InstanceId == instanceId);

                if (instance != null)
                {
                    if (dedupeStatuses != null && dedupeStatuses.Contains(instance.LastExecution.Status))
                    {
                        return;
                    }

                    if (!IsFinalInstanceStatus(instance.LastExecution.Status))
                    {
                        throw new Exception("Orchestration has an active execution");
                    }
                }

                var runtimeState = new OrchestrationRuntimeState(new[] { executionStartedEvent });

                if (instance == null)
                {
                    instance = _instanceMapper.CreateInstance(executionStartedEvent);
                    await dbContext.Instances.AddAsync(instance);
                }
                else
                {
                    _instanceMapper.UpdateInstance(instance, runtimeState);
                }

                var execution = _executionMapper.CreateExecution(runtimeState);
                await dbContext.Executions.AddAsync(execution);

                var knownQueues = new Dictionary <string, string>
                {
                    [instance.InstanceId] = QueueMapper.ToQueueName(runtimeState.Name, runtimeState.Version)
                };
                var orchestrationWorkItem = await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(
                    creationMessage,
                    0,
                    dbContext,
                    knownQueues
                    );

                await dbContext.OrchestrationMessages.AddAsync(orchestrationWorkItem);

                await dbContext.SaveChangesAsync();
            }
        }
예제 #3
0
        public override void Consumer_ReceiveMessage(IBasicGetResult message)
        {
            base.Consumer_ReceiveMessage(message);
            if (!ContinueProcessing)
            {
                return;
            }

            var request = Message;

            Log.Information("Processing CorrectCodelineRequest '{@request}', '{@correlationId}'", request, CorrelationId);

            try
            {
                //Mapping queue table
                var queue = QueueMapper.Map(request);
                queue.CorrelationId = CorrelationId;
                queue.RoutingKey    = RoutingKey;

                //Mapping voucher fields
                var vouchers      = VoucherMapper.Map(request).ToList();
                var jobIdentifier = CorrelationId;
                var batchNumber   = string.IsNullOrEmpty(request.voucherBatch.scannedBatchNumber)
                    ? queue.S_BATCH
                    : request.voucherBatch.scannedBatchNumber;
                var processingDate = request.voucher.First().processingDate;

                //Peform image merge for Dips
                var imageMergeHelper = new ImageMergeHelper(Configuration);
                imageMergeHelper.EnsureMergedImageFilesExist(jobIdentifier, batchNumber, processingDate);
                imageMergeHelper.PopulateMergedImageInfo(jobIdentifier, batchNumber, vouchers);

                //Mapping index fields
                var dbIndexes = DbIndexMapper.Map(request);

                using (var dbConnection = new SqlConnection(Configuration.SqlConnectionString))
                {
                    using (var dipsDbContext = new DipsDbContext(dbConnection))
                    {
                        using (var tx = dipsDbContext.BeginTransaction())
                        {
                            try
                            {
                                //Adding to queue table
                                dipsDbContext.Queues.Add(queue);
                                Log.Verbose("Adding new queue {@batchNumber} to the database", queue.S_BATCH);

                                dipsDbContext.SaveChanges();

                                //Adding to voucher table
                                foreach (var voucher in vouchers)
                                {
                                    dipsDbContext.NabChqPods.Add(voucher);
                                    Log.Verbose(
                                        "Adding new voucher {@batchNumber} - {@sequenceNumber} to the database",
                                        voucher.S_BATCH,
                                        voucher.S_SEQUENCE);
                                }

                                dipsDbContext.SaveChanges();

                                //Adding to index table
                                foreach (var dbIndex in dbIndexes)
                                {
                                    dipsDbContext.DbIndexes.Add(dbIndex);
                                    Log.Verbose(
                                        "Adding new db index {@batchNumber} - {@sequenceNumber} to the database",
                                        dbIndex.BATCH, dbIndex.SEQUENCE);
                                }

                                dipsDbContext.SaveChanges();

                                tx.Commit();
                                InvalidExchange.SendMessage(message.Body, RecoverableRoutingKey, CorrelationId);

                                Log.Information(
                                    "Successfully processed CorrectCodelineRequest '{@batchNumber}', '{@jobIdentifier}'",
                                    batchNumber, jobIdentifier);
                            }
                            catch (OptimisticConcurrencyException)
                            {
                                //this is to handle the race condition where more than instance of this service is running at the same time and tries to update the row.

                                //basically ignore the message by loggin a warning and rolling back.
                                //if this row was not included by mistake (e.g. it should be included), it will just come in in the next batch run.
                                Log.Warning(
                                    "Could not create a CorrectCodelineRequest '{@CorrectionCodelineRequest}', '{@jobIdentifier}' because the DIPS database row was updated by another connection",
                                    request, jobIdentifier);

                                tx.Rollback();
                                InvalidExchange.SendMessage(message.Body, RecoverableRoutingKey, CorrelationId);
                            }
                            catch (Exception ex)
                            {
                                Log.Error(
                                    ex,
                                    "Could not complete and create a CorrectCodelineRequest '{@CorrectionCodelineRequest}', '{@jobIdentifier}'",
                                    request, jobIdentifier);
                                tx.Rollback();
                                InvalidExchange.SendMessage(message.Body, RecoverableRoutingKey, CorrelationId);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Log.Error(ex, "Error processing CorrectionCodelineRequest {@CorrectionCodelineRequest}", request);
                InvalidExchange.SendMessage(message.Body, InvalidRoutingKey, CorrelationId);
            }
        }
예제 #4
0
 /// <summary>
 /// Maps one or more argument values to the queue specified by the expression.
 /// </summary>
 /// <param name="expression">Expression that identifies the collection.</param>
 /// <returns>Configuration.</returns>
 public MultiValueArgumentConfiguration <TOptions, TValue> ToQueue(
     Expression <Func <TOptions, Queue <TValue> > > expression)
 {
     Using(QueueMapper <TOptions, TValue> .Create(expression));
     return(Configuration);
 }
        public async Task CompleteTaskOrchestrationWorkItemAsync(
            TaskOrchestrationWorkItem workItem,
            OrchestrationRuntimeState newOrchestrationRuntimeState,
            IList <TaskMessage> outboundMessages,
            IList <TaskMessage> orchestratorMessages,
            IList <TaskMessage> timerMessages,
            TaskMessage continuedAsNewMessage,
            OrchestrationState orchestrationState)
        {
            using (var dbContext = _dbContextFactory())
            {
                var session = workItem.Session as EFCoreOrchestrationSession;

                // Create child orchestrations
                foreach (var executionStartedEvent in orchestratorMessages.Select(m => m.Event).OfType <ExecutionStartedEvent>())
                {
                    var childInstance = _instanceMapper.CreateInstance(executionStartedEvent);
                    await dbContext.Instances.AddAsync(childInstance);

                    var childRuntimeState = new OrchestrationRuntimeState(new[] { executionStartedEvent });
                    var childExecution    = _executionMapper.CreateExecution(childRuntimeState);
                    await dbContext.Executions.AddAsync(childExecution);
                }

                var orchestrationQueueName = QueueMapper.ToQueueName(orchestrationState.Name, orchestrationState.Version);

                var knownQueues = new Dictionary <string, string>
                {
                    [orchestrationState.OrchestrationInstance.InstanceId] = orchestrationQueueName
                };

                if (orchestrationState.ParentInstance != null)
                {
                    knownQueues[orchestrationState.ParentInstance.OrchestrationInstance.InstanceId] = QueueMapper.ToQueueName(orchestrationState.ParentInstance.Name, orchestrationState.ParentInstance.Version);
                }

                // Write messages
                var activityMessages = outboundMessages
                                       .Select(m => _activityMessageMapper.CreateActivityMessage(m, orchestrationQueueName))
                                       .ToArray();
                var orchestatorMessages = await orchestratorMessages
                                          .Select((m, i) => _orchestrationMessageMapper.CreateOrchestrationMessageAsync(m, i, dbContext, knownQueues))
                                          .WhenAllSerial();

                var timerOrchestrationMessages = await timerMessages
                                                 .Select((m, i) => _orchestrationMessageMapper.CreateOrchestrationMessageAsync(m, i, dbContext, knownQueues))
                                                 .WhenAllSerial();

                var continuedAsNewOrchestrationMessage = continuedAsNewMessage != null
                    ? await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(continuedAsNewMessage, 0, dbContext, knownQueues)
                    : null;

                await dbContext.ActivityMessages.AddRangeAsync(activityMessages);

                await dbContext.OrchestrationMessages.AddRangeAsync(orchestatorMessages);

                await dbContext.OrchestrationMessages.AddRangeAsync(timerOrchestrationMessages);

                if (continuedAsNewOrchestrationMessage != null)
                {
                    await dbContext.OrchestrationMessages.AddAsync(continuedAsNewOrchestrationMessage);
                }

                // Remove executed messages
                dbContext.AttachRange(session.Messages);
                dbContext.OrchestrationMessages.RemoveRange(session.Messages);

                // Update instance
                var instance = session.Instance;
                dbContext.Instances.Attach(instance);
                _instanceMapper.UpdateInstance(instance, newOrchestrationRuntimeState);

                // Update current execution
                EnrichNewEventsInput(workItem.OrchestrationRuntimeState, outboundMessages, orchestratorMessages);
                session.Execution = await SaveExecutionAsync(dbContext, workItem.OrchestrationRuntimeState, session.Execution);

                // Update new execution
                if (newOrchestrationRuntimeState != workItem.OrchestrationRuntimeState)
                {
                    EnrichNewEventsInput(newOrchestrationRuntimeState, outboundMessages, orchestratorMessages);
                    session.Execution = await SaveExecutionAsync(dbContext, newOrchestrationRuntimeState);
                }

                await dbContext.SaveChangesAsync();

                session.RuntimeState = newOrchestrationRuntimeState;
                session.ClearMessages();
            }
        }
 public void CreateReturnsMultivaluedMapper()
 {
     QueueMapper <MyOptions, int> .Create(opt => opt.Queue).MultiValued.ShouldBeTrue();
 }
 public void CreateReturnsInstance()
 {
     QueueMapper <MyOptions, int> .Create(opt => opt.Queue).ShouldNotBeNull();
 }
        private async Task RewindInstanceAsync(OrchestrationDbContext dbContext, string instanceId, string reason)
        {
            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
            // REWIND ALGORITHM:
            // 1. Finds failed execution of specified orchestration instance to rewind
            // 2. Finds failure entities to clear and over-writes them (as well as corresponding trigger events)
            // 3. Identifies sub-orchestration failure(s) from parent instance and calls RewindHistoryAsync recursively on failed sub-orchestration child instance(s)
            // 4. Resets orchestration status of rewound instance in instance store table to prepare it to be restarted
            // 5. Restart that doesn't have failed suborchestrations with a generic event
            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

            var lastExecution = dbContext.Instances
                                .Where(i => i.InstanceId == instanceId)
                                .Select(i => i.LastExecution)
                                .FirstOrDefault();

            var events = await dbContext.Events
                         .Where(e => e.ExecutionId == lastExecution.ExecutionId)
                         .ToArrayAsync();

            var historyEvents = events
                                .ToDictionary(e => _options.DataConverter.Deserialize <HistoryEvent>(e.Content));

            bool hasFailedSubOrchestrations = false;

            foreach (var historyEvent in historyEvents.Keys)
            {
                if (historyEvent is TaskFailedEvent taskFailedEvent)
                {
                    var taskScheduledEvent = historyEvents.Keys.OfType <TaskScheduledEvent>()
                                             .FirstOrDefault(e => e.EventId == taskFailedEvent.TaskScheduledId);

                    var rewoundTaskScheduledData = _options.RewindDataConverter.Serialize(new
                    {
                        taskScheduledEvent.EventType,
                        taskScheduledEvent.Name,
                        taskScheduledEvent.Version,
                        taskScheduledEvent.Input
                    });

                    historyEvents[taskScheduledEvent].Content = _options.DataConverter.Serialize(
                        new GenericEvent(taskScheduledEvent.EventId, $"Rewound: {rewoundTaskScheduledData}")
                    {
                        Timestamp = taskScheduledEvent.Timestamp
                    }
                        );

                    var rewoundTaskFailedData = _options.RewindDataConverter.Serialize(new
                    {
                        taskFailedEvent.EventType,
                        taskFailedEvent.Reason,
                        taskFailedEvent.TaskScheduledId
                    });

                    historyEvents[taskFailedEvent].Content = _options.DataConverter.Serialize(
                        new GenericEvent(taskFailedEvent.EventId, $"Rewound: {rewoundTaskFailedData}")
                    {
                        Timestamp = taskFailedEvent.Timestamp
                    }
                        );
                }
                else if (historyEvent is SubOrchestrationInstanceFailedEvent soFailedEvent)
                {
                    hasFailedSubOrchestrations = true;

                    var soCreatedEvent = historyEvents.Keys.OfType <SubOrchestrationInstanceCreatedEvent>()
                                         .FirstOrDefault(e => e.EventId == soFailedEvent.TaskScheduledId);

                    var rewoundSoCreatedData = _options.RewindDataConverter.Serialize(new
                    {
                        soCreatedEvent.EventType,
                        soCreatedEvent.Name,
                        soCreatedEvent.Version,
                        soCreatedEvent.Input
                    });

                    historyEvents[soCreatedEvent].Content = _options.DataConverter.Serialize(
                        new GenericEvent(soCreatedEvent.EventId, $"Rewound: {rewoundSoCreatedData}")
                    {
                        Timestamp = soCreatedEvent.Timestamp
                    }
                        );

                    var rewoundSoFailedData = _options.RewindDataConverter.Serialize(new
                    {
                        soFailedEvent.EventType,
                        soFailedEvent.Reason,
                        soFailedEvent.TaskScheduledId
                    });

                    historyEvents[soFailedEvent].Content = _options.DataConverter.Serialize(
                        new GenericEvent(soFailedEvent.EventId, $"Rewound: {rewoundSoFailedData}")
                    {
                        Timestamp = soFailedEvent.Timestamp
                    }
                        );

                    // recursive call to clear out failure events on child instances
                    await RewindInstanceAsync(dbContext, soCreatedEvent.InstanceId, reason);
                }
                else if (historyEvent is ExecutionCompletedEvent executionCompletedEvent &&
                         executionCompletedEvent.OrchestrationStatus == OrchestrationStatus.Failed)
                {
                    var rewoundExecutionCompletedData = _options.RewindDataConverter.Serialize(new
                    {
                        executionCompletedEvent.EventType,
                        executionCompletedEvent.Result,
                        executionCompletedEvent.OrchestrationStatus
                    });

                    historyEvents[executionCompletedEvent].Content = _options.DataConverter.Serialize(
                        new GenericEvent(executionCompletedEvent.EventId, $"Rewound: {rewoundExecutionCompletedData}")
                    {
                        Timestamp = executionCompletedEvent.Timestamp
                    }
                        );
                }
            }

            // Reset execution status
            lastExecution.Status          = OrchestrationStatus.Running;
            lastExecution.LastUpdatedTime = DateTime.UtcNow;

            if (!hasFailedSubOrchestrations)
            {
                var orchestrationInstance = new OrchestrationInstance
                {
                    InstanceId = instanceId
                };

                var taskMessage = new TaskMessage
                {
                    OrchestrationInstance = orchestrationInstance,
                    Event = new GenericEvent(-1, reason)
                };

                var knownQueues = new Dictionary <string, string>
                {
                    [lastExecution.InstanceId] = QueueMapper.ToQueueName(lastExecution.Name, lastExecution.Version)
                };
                var orchestrationMessage = await _orchestrationMessageMapper.CreateOrchestrationMessageAsync(taskMessage, 0, dbContext, knownQueues);

                await dbContext.OrchestrationMessages.AddAsync(orchestrationMessage);
            }
        }