public void Process(ClientRequestEventWithPrefetch clientRequestEvent, EffectTracker effects) { if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read) { this.Partition.Assert(!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString)); // Issue a read request that fetches the instance state. // We have to buffer this request in the pending list so we can recover it. this.PendingPrefetches.Add(clientRequestEvent.EventIdString, clientRequestEvent); if (!effects.IsReplaying) { this.Partition.SubmitInternalEvent(new InstanceLookup(clientRequestEvent)); } } else { if (this.PendingPrefetches.Remove(clientRequestEvent.EventIdString)) { if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.ConfirmAndProcess) { effects.Add(clientRequestEvent.Target); } } } }
public void Process(DeletionRequestReceived deletionRequest, EffectTracker effects) { int numberInstancesDeleted = 0; if (this.OrchestrationState != null && (!deletionRequest.CreatedTime.HasValue || deletionRequest.CreatedTime.Value == this.OrchestrationState.CreatedTime)) { this.OrchestrationState = null; numberInstancesDeleted++; // we also delete this instance's history, and pending operations on it effects.Add(TrackedObjectKey.History(this.InstanceId)); effects.Add(TrackedObjectKey.Sessions); } if (!effects.IsReplaying) { this.Partition.Send(new DeletionResponseReceived() { ClientId = deletionRequest.ClientId, RequestId = deletionRequest.RequestId, NumberInstancesDeleted = numberInstancesDeleted, }); } }
void SendBatchOnceEventIsPersisted(PartitionUpdateEvent evt, EffectTracker effects, Batch batch) { // put the messages in the outbox where they are kept until actually sent var commitPosition = evt.NextCommitLogPosition; // Update the ready to send timestamp to check the delay caused // by non-speculation evt.ReadyToSendTimestamp = this.Partition.CurrentTimeMs; this.Outbox[commitPosition] = batch; batch.Position = commitPosition; batch.Partition = this.Partition; if (!effects.IsReplaying) { if (!this.Partition.Settings.PersistStepsFirst) { // we must not send messages until this step has been persisted evt.OutboxBatch = batch; DurabilityListeners.Register(evt, this); } else { // we can send the messages now this.Send(batch); } } }
public void Process(SendConfirmed evt, EffectTracker _) { this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Store has sent all outbound messages by event {evt} id={evt.EventIdString}"); // we no longer need to keep these events around this.Outbox.Remove(evt.Position); }
public void Process(WaitRequestReceived evt, EffectTracker effects) { if (WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState)) { if (!effects.IsReplaying) { this.Partition.Send(evt.CreateResponse(this.OrchestrationState)); } } else { if (this.Waiters == null) { this.Waiters = new List <WaitRequestReceived>(); } else { // cull the list of waiters to remove requests that have already timed out this.Waiters = this.Waiters .Where(request => request.TimeoutUtc > DateTime.UtcNow) .ToList(); } this.Waiters.Add(evt); } }
public virtual void Process(PartitionEventFragment e, EffectTracker effects) { // processing a reassembled event just applies the original event dynamic dynamicThis = this; dynamic dynamicPartitionEvent = e.ReassembledEvent; dynamicThis.Process(dynamicPartitionEvent, effects); }
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects) { OrchestrationState state = this.OrchestrationState; if (this.OrchestrationState != null && purgeBatchIssued.InstanceQuery.Matches(this.OrchestrationState)) { this.OrchestrationState = null; purgeBatchIssued.Purged.Add(this.InstanceId); effects.Add(TrackedObjectKey.History(this.InstanceId)); } }
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects) { var purgeRequest = (PurgeRequestReceived)this.PendingQueries[purgeBatchIssued.QueryEventId]; purgeRequest.NumberInstancesPurged += purgeBatchIssued.Purged.Count; if (!effects.IsReplaying) { // lets the query that is currently in progress know that this batch is done purgeBatchIssued.WhenProcessed.TrySetResult(null); } }
public void Process(ActivityCompleted evt, EffectTracker effects) { var batch = new Batch(); batch.OutgoingMessages.Add(new RemoteActivityResultReceived() { PartitionId = evt.OriginPartitionId, Result = evt.Response, ActivityId = evt.ActivityId, ActivitiesQueueSize = evt.ReportedLoad, }); this.SendBatchOnceEventIsPersisted(evt, effects, batch); }
public void Process(CreationRequestReceived creationRequestReceived, EffectTracker effects) { bool filterDuplicate = this.OrchestrationState != null && creationRequestReceived.DedupeStatuses != null && creationRequestReceived.DedupeStatuses.Contains(this.OrchestrationState.OrchestrationStatus); if (!filterDuplicate) { var ee = creationRequestReceived.ExecutionStartedEvent; // set the orchestration state now (before processing the creation in the history) // so that this new instance is "on record" immediately - it is guaranteed to replace whatever is in flight this.OrchestrationState = new OrchestrationState { Name = ee.Name, Version = ee.Version, OrchestrationInstance = ee.OrchestrationInstance, OrchestrationStatus = OrchestrationStatus.Pending, ParentInstance = ee.ParentInstance, Input = ee.Input, Tags = ee.Tags, CreatedTime = ee.Timestamp, LastUpdatedTime = DateTime.UtcNow, CompletedTime = Core.Common.DateTimeUtils.MinDateTime, ScheduledStartTime = ee.ScheduledStartTime }; // queue the message in the session, or start a timer if delayed if (!ee.ScheduledStartTime.HasValue) { effects.Add(TrackedObjectKey.Sessions); } else { effects.Add(TrackedObjectKey.Timers); } } if (!effects.IsReplaying) { // send response to client effects.Partition.Send(new CreationResponseReceived() { ClientId = creationRequestReceived.ClientId, RequestId = creationRequestReceived.RequestId, Succeeded = !filterDuplicate, ExistingInstanceOrchestrationStatus = this.OrchestrationState?.OrchestrationStatus, }); } }
public override void DetermineEffects(EffectTracker effects) { // the last-added effects are processed first // so they can set the Purged list to contain only the instance ids that are actually purged effects.Add(TrackedObjectKey.Queries); effects.Add(TrackedObjectKey.Sessions); this.Purged = new List <string>(); foreach (string instanceId in this.InstanceIds) { effects.Add(TrackedObjectKey.Instance(instanceId)); } }
public void Process(BatchProcessed evt, EffectTracker effects) { // can add events to the history, or replace it with a new history // update the stored history if (this.History == null || evt.State.OrchestrationInstance.ExecutionId != this.ExecutionId) { this.History = new List <HistoryEvent>(); this.Episode = 0; this.ExecutionId = evt.State.OrchestrationInstance.ExecutionId; } this.Partition.Assert(!string.IsNullOrEmpty(this.InstanceId) || string.IsNullOrEmpty(this.ExecutionId)); // add all the new events to the history, and update episode number if (evt.NewEvents != null) { for (int i = 0; i < evt.NewEvents.Count; i++) { var historyEvent = evt.NewEvents[i]; if (historyEvent.EventType == EventType.OrchestratorStarted) { this.Episode++; } this.History.Add(evt.NewEvents[i]); } } if (!effects.IsReplaying) { this.Partition.EventTraceHelper.TraceInstanceUpdate( evt.WorkItemId, evt.State.OrchestrationInstance.InstanceId, evt.State.OrchestrationInstance.ExecutionId, this.History.Count, evt.NewEvents, this.Episode); // if present, we keep the work item so we can reuse the execution cursor this.CachedOrchestrationWorkItem = evt.WorkItemForReuse; if (this.CachedOrchestrationWorkItem != null && this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId != evt.State.OrchestrationInstance.ExecutionId) { effects.Partition.EventTraceHelper.TraceEventProcessingWarning($"Dropping bad workitem cache instance={this.InstanceId} expected_executionid={evt.State.OrchestrationInstance.ExecutionId} actual_executionid={this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId}"); this.CachedOrchestrationWorkItem = null; } } }
public void Process(BatchProcessed evt, EffectTracker effects) { // update the state of an orchestration this.OrchestrationState = evt.State; // if the orchestration is complete, notify clients that are waiting for it if (this.Waiters != null && WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState)) { if (!effects.IsReplaying) { foreach (var request in this.Waiters) { this.Partition.Send(request.CreateResponse(this.OrchestrationState)); } } this.Waiters = null; } }
public override void Process(PartitionEventFragment evt, EffectTracker effects) { // stores fragments until the last one is received var originalEventString = evt.OriginalEventId.ToString(); if (evt.IsLast) { evt.ReassembledEvent = FragmentationAndReassembly.Reassemble <PartitionEvent>(this.Fragments[originalEventString], evt); this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Reassembled {evt.ReassembledEvent}"); this.Fragments.Remove(originalEventString); switch (evt.ReassembledEvent) { case PartitionUpdateEvent updateEvent: updateEvent.DetermineEffects(effects); break; case PartitionReadEvent readEvent: this.Partition.SubmitInternalEvent(readEvent); break; case PartitionQueryEvent queryEvent: this.Partition.SubmitInternalEvent(queryEvent); break; default: throw new InvalidCastException("Could not cast to neither PartitionReadEvent nor PartitionUpdateEvent"); } } else { if (!this.Fragments.TryGetValue(originalEventString, out var list)) { this.Fragments[originalEventString] = list = new List <PartitionEventFragment>(); } list.Add(evt); } }
public void Process(BatchProcessed evt, EffectTracker effects) { var sorted = new Dictionary <uint, TaskMessagesReceived>(); foreach (var message in evt.RemoteMessages) { var instanceId = message.OrchestrationInstance.InstanceId; var destination = this.Partition.PartitionFunction(instanceId); if (!sorted.TryGetValue(destination, out var outmessage)) { sorted[destination] = outmessage = new TaskMessagesReceived() { PartitionId = destination, WorkItemId = evt.WorkItemId, }; } if (Entities.IsDelayedEntityMessage(message, out _)) { (outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List <TaskMessage>())).Add(message); } else if (message.Event is ExecutionStartedEvent executionStartedEvent && executionStartedEvent.ScheduledStartTime.HasValue) { (outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List <TaskMessage>())).Add(message); }
protected override async Task Process(IList <PartitionEvent> batch) { try { var effects = new EffectTracker( this.partition, this.ApplyToStore, () => (this.commitPosition, this.inputQueuePosition) ); if (batch.Count != 0) { foreach (var partitionEvent in batch) { // record the current time, for measuring latency in the event processing pipeline partitionEvent.IssuedTimestamp = this.partition.CurrentTimeMs; try { switch (partitionEvent) { case PartitionUpdateEvent updateEvent: updateEvent.NextCommitLogPosition = this.commitPosition + 1; await effects.ProcessUpdate(updateEvent).ConfigureAwait(false); DurabilityListeners.ConfirmDurable(updateEvent); if (updateEvent.NextCommitLogPosition > 0) { this.partition.Assert(updateEvent.NextCommitLogPosition > this.commitPosition); this.commitPosition = updateEvent.NextCommitLogPosition; } break; case PartitionReadEvent readEvent: readEvent.OnReadIssued(this.partition); if (readEvent.Prefetch.HasValue) { var prefetchTarget = this.GetOrAdd(readEvent.Prefetch.Value); effects.ProcessReadResult(readEvent, readEvent.Prefetch.Value, prefetchTarget); } var readTarget = this.GetOrAdd(readEvent.ReadTarget); effects.ProcessReadResult(readEvent, readEvent.ReadTarget, readTarget); break; case PartitionQueryEvent queryEvent: var instances = this.QueryOrchestrationStates(queryEvent.InstanceQuery); var backgroundTask = Task.Run(() => effects.ProcessQueryResultAsync(queryEvent, instances.ToAsyncEnumerable())); break; default: throw new InvalidCastException("could not cast to neither PartitionReadEvent nor PartitionUpdateEvent"); } if (partitionEvent.NextInputQueuePosition > 0) { this.partition.Assert(partitionEvent.NextInputQueuePosition > this.inputQueuePosition); this.inputQueuePosition = partitionEvent.NextInputQueuePosition; } } catch (Exception e) { this.partition.ErrorHandler.HandleError(nameof(Process), $"Encountered exception while processing event {partitionEvent}", e, false, false); } } } } catch (Exception e) { this.logger.LogError("Exception in MemoryQueue BatchWorker: {exception}", e); } }
public ValueTask ApplyToStore(TrackedObjectKey key, EffectTracker tracker) { tracker.ProcessEffectOn(this.GetOrAdd(key)); return(default);
public override void DetermineEffects(EffectTracker effects) { effects.Add(TrackedObjectKey.Reassembly); }
public sealed override void DetermineEffects(EffectTracker effects) { effects.Add(TrackedObjectKey.Prefetch); }
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects) { this.DeleteHistory(); }
public override void DetermineEffects(EffectTracker effects) { effects.Add(TrackedObjectKey.Dedup); }
public void Process(DeletionRequestReceived deletionRequestReceived, EffectTracker effects) { this.DeleteHistory(); }