Пример #1
0
        public void Process(ClientRequestEventWithPrefetch clientRequestEvent, EffectTracker effects)
        {
            if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read)
            {
                this.Partition.Assert(!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString));

                // Issue a read request that fetches the instance state.
                // We have to buffer this request in the pending list so we can recover it.

                this.PendingPrefetches.Add(clientRequestEvent.EventIdString, clientRequestEvent);

                if (!effects.IsReplaying)
                {
                    this.Partition.SubmitInternalEvent(new InstanceLookup(clientRequestEvent));
                }
            }
            else
            {
                if (this.PendingPrefetches.Remove(clientRequestEvent.EventIdString))
                {
                    if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.ConfirmAndProcess)
                    {
                        effects.Add(clientRequestEvent.Target);
                    }
                }
            }
        }
Пример #2
0
        public void Process(DeletionRequestReceived deletionRequest, EffectTracker effects)
        {
            int numberInstancesDeleted = 0;

            if (this.OrchestrationState != null &&
                (!deletionRequest.CreatedTime.HasValue || deletionRequest.CreatedTime.Value == this.OrchestrationState.CreatedTime))
            {
                this.OrchestrationState = null;
                numberInstancesDeleted++;

                // we also delete this instance's history, and pending operations on it
                effects.Add(TrackedObjectKey.History(this.InstanceId));
                effects.Add(TrackedObjectKey.Sessions);
            }

            if (!effects.IsReplaying)
            {
                this.Partition.Send(new DeletionResponseReceived()
                {
                    ClientId  = deletionRequest.ClientId,
                    RequestId = deletionRequest.RequestId,
                    NumberInstancesDeleted = numberInstancesDeleted,
                });
            }
        }
        void SendBatchOnceEventIsPersisted(PartitionUpdateEvent evt, EffectTracker effects, Batch batch)
        {
            // put the messages in the outbox where they are kept until actually sent
            var commitPosition = evt.NextCommitLogPosition;

            // Update the ready to send timestamp to check the delay caused
            // by non-speculation
            evt.ReadyToSendTimestamp = this.Partition.CurrentTimeMs;

            this.Outbox[commitPosition] = batch;
            batch.Position  = commitPosition;
            batch.Partition = this.Partition;

            if (!effects.IsReplaying)
            {
                if (!this.Partition.Settings.PersistStepsFirst)
                {
                    // we must not send messages until this step has been persisted
                    evt.OutboxBatch = batch;
                    DurabilityListeners.Register(evt, this);
                }
                else
                {
                    // we can send the messages now
                    this.Send(batch);
                }
            }
        }
        public void Process(SendConfirmed evt, EffectTracker _)
        {
            this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Store has sent all outbound messages by event {evt} id={evt.EventIdString}");

            // we no longer need to keep these events around
            this.Outbox.Remove(evt.Position);
        }
Пример #5
0
        public void Process(WaitRequestReceived evt, EffectTracker effects)
        {
            if (WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState))
            {
                if (!effects.IsReplaying)
                {
                    this.Partition.Send(evt.CreateResponse(this.OrchestrationState));
                }
            }
            else
            {
                if (this.Waiters == null)
                {
                    this.Waiters = new List <WaitRequestReceived>();
                }
                else
                {
                    // cull the list of waiters to remove requests that have already timed out
                    this.Waiters = this.Waiters
                                   .Where(request => request.TimeoutUtc > DateTime.UtcNow)
                                   .ToList();
                }

                this.Waiters.Add(evt);
            }
        }
Пример #6
0
        public virtual void Process(PartitionEventFragment e, EffectTracker effects)
        {
            // processing a reassembled event just applies the original event
            dynamic dynamicThis           = this;
            dynamic dynamicPartitionEvent = e.ReassembledEvent;

            dynamicThis.Process(dynamicPartitionEvent, effects);
        }
Пример #7
0
        public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
        {
            OrchestrationState state = this.OrchestrationState;

            if (this.OrchestrationState != null &&
                purgeBatchIssued.InstanceQuery.Matches(this.OrchestrationState))
            {
                this.OrchestrationState = null;
                purgeBatchIssued.Purged.Add(this.InstanceId);
                effects.Add(TrackedObjectKey.History(this.InstanceId));
            }
        }
Пример #8
0
        public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
        {
            var purgeRequest = (PurgeRequestReceived)this.PendingQueries[purgeBatchIssued.QueryEventId];

            purgeRequest.NumberInstancesPurged += purgeBatchIssued.Purged.Count;

            if (!effects.IsReplaying)
            {
                // lets the query that is currently in progress know that this batch is done
                purgeBatchIssued.WhenProcessed.TrySetResult(null);
            }
        }
        public void Process(ActivityCompleted evt, EffectTracker effects)
        {
            var batch = new Batch();

            batch.OutgoingMessages.Add(new RemoteActivityResultReceived()
            {
                PartitionId         = evt.OriginPartitionId,
                Result              = evt.Response,
                ActivityId          = evt.ActivityId,
                ActivitiesQueueSize = evt.ReportedLoad,
            });
            this.SendBatchOnceEventIsPersisted(evt, effects, batch);
        }
Пример #10
0
        public void Process(CreationRequestReceived creationRequestReceived, EffectTracker effects)
        {
            bool filterDuplicate = this.OrchestrationState != null &&
                                   creationRequestReceived.DedupeStatuses != null &&
                                   creationRequestReceived.DedupeStatuses.Contains(this.OrchestrationState.OrchestrationStatus);

            if (!filterDuplicate)
            {
                var ee = creationRequestReceived.ExecutionStartedEvent;

                // set the orchestration state now (before processing the creation in the history)
                // so that this new instance is "on record" immediately - it is guaranteed to replace whatever is in flight
                this.OrchestrationState = new OrchestrationState
                {
                    Name    = ee.Name,
                    Version = ee.Version,
                    OrchestrationInstance = ee.OrchestrationInstance,
                    OrchestrationStatus   = OrchestrationStatus.Pending,
                    ParentInstance        = ee.ParentInstance,
                    Input              = ee.Input,
                    Tags               = ee.Tags,
                    CreatedTime        = ee.Timestamp,
                    LastUpdatedTime    = DateTime.UtcNow,
                    CompletedTime      = Core.Common.DateTimeUtils.MinDateTime,
                    ScheduledStartTime = ee.ScheduledStartTime
                };

                // queue the message in the session, or start a timer if delayed
                if (!ee.ScheduledStartTime.HasValue)
                {
                    effects.Add(TrackedObjectKey.Sessions);
                }
                else
                {
                    effects.Add(TrackedObjectKey.Timers);
                }
            }

            if (!effects.IsReplaying)
            {
                // send response to client
                effects.Partition.Send(new CreationResponseReceived()
                {
                    ClientId  = creationRequestReceived.ClientId,
                    RequestId = creationRequestReceived.RequestId,
                    Succeeded = !filterDuplicate,
                    ExistingInstanceOrchestrationStatus = this.OrchestrationState?.OrchestrationStatus,
                });
            }
        }
Пример #11
0
        public override void DetermineEffects(EffectTracker effects)
        {
            // the last-added effects are processed first
            // so they can set the Purged list to contain only the instance ids that are actually purged

            effects.Add(TrackedObjectKey.Queries);
            effects.Add(TrackedObjectKey.Sessions);

            this.Purged = new List <string>();
            foreach (string instanceId in this.InstanceIds)
            {
                effects.Add(TrackedObjectKey.Instance(instanceId));
            }
        }
        public void Process(BatchProcessed evt, EffectTracker effects)
        {
            // can add events to the history, or replace it with a new history

            // update the stored history
            if (this.History == null || evt.State.OrchestrationInstance.ExecutionId != this.ExecutionId)
            {
                this.History     = new List <HistoryEvent>();
                this.Episode     = 0;
                this.ExecutionId = evt.State.OrchestrationInstance.ExecutionId;
            }

            this.Partition.Assert(!string.IsNullOrEmpty(this.InstanceId) || string.IsNullOrEmpty(this.ExecutionId));

            // add all the new events to the history, and update episode number
            if (evt.NewEvents != null)
            {
                for (int i = 0; i < evt.NewEvents.Count; i++)
                {
                    var historyEvent = evt.NewEvents[i];
                    if (historyEvent.EventType == EventType.OrchestratorStarted)
                    {
                        this.Episode++;
                    }
                    this.History.Add(evt.NewEvents[i]);
                }
            }

            if (!effects.IsReplaying)
            {
                this.Partition.EventTraceHelper.TraceInstanceUpdate(
                    evt.WorkItemId,
                    evt.State.OrchestrationInstance.InstanceId,
                    evt.State.OrchestrationInstance.ExecutionId,
                    this.History.Count,
                    evt.NewEvents, this.Episode);

                // if present, we keep the work item so we can reuse the execution cursor
                this.CachedOrchestrationWorkItem = evt.WorkItemForReuse;

                if (this.CachedOrchestrationWorkItem != null &&
                    this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId != evt.State.OrchestrationInstance.ExecutionId)
                {
                    effects.Partition.EventTraceHelper.TraceEventProcessingWarning($"Dropping bad workitem cache instance={this.InstanceId} expected_executionid={evt.State.OrchestrationInstance.ExecutionId} actual_executionid={this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId}");
                    this.CachedOrchestrationWorkItem = null;
                }
            }
        }
Пример #13
0
        public void Process(BatchProcessed evt, EffectTracker effects)
        {
            // update the state of an orchestration
            this.OrchestrationState = evt.State;

            // if the orchestration is complete, notify clients that are waiting for it
            if (this.Waiters != null && WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState))
            {
                if (!effects.IsReplaying)
                {
                    foreach (var request in this.Waiters)
                    {
                        this.Partition.Send(request.CreateResponse(this.OrchestrationState));
                    }
                }

                this.Waiters = null;
            }
        }
        public override void Process(PartitionEventFragment evt, EffectTracker effects)
        {
            // stores fragments until the last one is received
            var originalEventString = evt.OriginalEventId.ToString();

            if (evt.IsLast)
            {
                evt.ReassembledEvent = FragmentationAndReassembly.Reassemble <PartitionEvent>(this.Fragments[originalEventString], evt);

                this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Reassembled {evt.ReassembledEvent}");

                this.Fragments.Remove(originalEventString);

                switch (evt.ReassembledEvent)
                {
                case PartitionUpdateEvent updateEvent:
                    updateEvent.DetermineEffects(effects);
                    break;

                case PartitionReadEvent readEvent:
                    this.Partition.SubmitInternalEvent(readEvent);
                    break;

                case PartitionQueryEvent queryEvent:
                    this.Partition.SubmitInternalEvent(queryEvent);
                    break;

                default:
                    throw new InvalidCastException("Could not cast to neither PartitionReadEvent nor PartitionUpdateEvent");
                }
            }
            else
            {
                if (!this.Fragments.TryGetValue(originalEventString, out var list))
                {
                    this.Fragments[originalEventString] = list = new List <PartitionEventFragment>();
                }
                list.Add(evt);
            }
        }
        public void Process(BatchProcessed evt, EffectTracker effects)
        {
            var sorted = new Dictionary <uint, TaskMessagesReceived>();

            foreach (var message in evt.RemoteMessages)
            {
                var instanceId  = message.OrchestrationInstance.InstanceId;
                var destination = this.Partition.PartitionFunction(instanceId);
                if (!sorted.TryGetValue(destination, out var outmessage))
                {
                    sorted[destination] = outmessage = new TaskMessagesReceived()
                    {
                        PartitionId = destination,
                        WorkItemId  = evt.WorkItemId,
                    };
                }
                if (Entities.IsDelayedEntityMessage(message, out _))
                {
                    (outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List <TaskMessage>())).Add(message);
                }
                else if (message.Event is ExecutionStartedEvent executionStartedEvent && executionStartedEvent.ScheduledStartTime.HasValue)
                {
                    (outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List <TaskMessage>())).Add(message);
                }
        protected override async Task Process(IList <PartitionEvent> batch)
        {
            try
            {
                var effects = new EffectTracker(
                    this.partition,
                    this.ApplyToStore,
                    () => (this.commitPosition, this.inputQueuePosition)
                    );

                if (batch.Count != 0)
                {
                    foreach (var partitionEvent in batch)
                    {
                        // record the current time, for measuring latency in the event processing pipeline
                        partitionEvent.IssuedTimestamp = this.partition.CurrentTimeMs;

                        try
                        {
                            switch (partitionEvent)
                            {
                            case PartitionUpdateEvent updateEvent:
                                updateEvent.NextCommitLogPosition = this.commitPosition + 1;
                                await effects.ProcessUpdate(updateEvent).ConfigureAwait(false);

                                DurabilityListeners.ConfirmDurable(updateEvent);
                                if (updateEvent.NextCommitLogPosition > 0)
                                {
                                    this.partition.Assert(updateEvent.NextCommitLogPosition > this.commitPosition);
                                    this.commitPosition = updateEvent.NextCommitLogPosition;
                                }
                                break;

                            case PartitionReadEvent readEvent:
                                readEvent.OnReadIssued(this.partition);
                                if (readEvent.Prefetch.HasValue)
                                {
                                    var prefetchTarget = this.GetOrAdd(readEvent.Prefetch.Value);
                                    effects.ProcessReadResult(readEvent, readEvent.Prefetch.Value, prefetchTarget);
                                }
                                var readTarget = this.GetOrAdd(readEvent.ReadTarget);
                                effects.ProcessReadResult(readEvent, readEvent.ReadTarget, readTarget);
                                break;

                            case PartitionQueryEvent queryEvent:
                                var instances      = this.QueryOrchestrationStates(queryEvent.InstanceQuery);
                                var backgroundTask = Task.Run(() => effects.ProcessQueryResultAsync(queryEvent, instances.ToAsyncEnumerable()));
                                break;

                            default:
                                throw new InvalidCastException("could not cast to neither PartitionReadEvent nor PartitionUpdateEvent");
                            }

                            if (partitionEvent.NextInputQueuePosition > 0)
                            {
                                this.partition.Assert(partitionEvent.NextInputQueuePosition > this.inputQueuePosition);
                                this.inputQueuePosition = partitionEvent.NextInputQueuePosition;
                            }
                        }
                        catch (Exception e)
                        {
                            this.partition.ErrorHandler.HandleError(nameof(Process), $"Encountered exception while processing event {partitionEvent}", e, false, false);
                        }
                    }
                }
            }
            catch (Exception e)
            {
                this.logger.LogError("Exception in MemoryQueue BatchWorker: {exception}", e);
            }
        }
 public ValueTask ApplyToStore(TrackedObjectKey key, EffectTracker tracker)
 {
     tracker.ProcessEffectOn(this.GetOrAdd(key));
     return(default);
 public override void DetermineEffects(EffectTracker effects)
 {
     effects.Add(TrackedObjectKey.Reassembly);
 }
 public sealed override void DetermineEffects(EffectTracker effects)
 {
     effects.Add(TrackedObjectKey.Prefetch);
 }
 public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
 {
     this.DeleteHistory();
 }
 public override void DetermineEffects(EffectTracker effects)
 {
     effects.Add(TrackedObjectKey.Dedup);
 }
 public void Process(DeletionRequestReceived deletionRequestReceived, EffectTracker effects)
 {
     this.DeleteHistory();
 }