public void Process(ClientRequestEventWithPrefetch clientRequestEvent, EffectTracker effects)
{
if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read)
{
this.Partition.Assert(!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString));
// Issue a read request that fetches the instance state.
// We have to buffer this request in the pending list so we can recover it.
this.PendingPrefetches.Add(clientRequestEvent.EventIdString, clientRequestEvent);
if (!effects.IsReplaying)
{
this.Partition.SubmitInternalEvent(new InstanceLookup(clientRequestEvent));
}
}
else
{
if (this.PendingPrefetches.Remove(clientRequestEvent.EventIdString))
{
if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.ConfirmAndProcess)
{
effects.Add(clientRequestEvent.Target);
}
}
}
}
public void Process(DeletionRequestReceived deletionRequest, EffectTracker effects)
{
int numberInstancesDeleted = 0;
if (this.OrchestrationState != null &&
(!deletionRequest.CreatedTime.HasValue || deletionRequest.CreatedTime.Value == this.OrchestrationState.CreatedTime))
{
this.OrchestrationState = null;
numberInstancesDeleted++;
// we also delete this instance's history, and pending operations on it
effects.Add(TrackedObjectKey.History(this.InstanceId));
effects.Add(TrackedObjectKey.Sessions);
}
if (!effects.IsReplaying)
{
this.Partition.Send(new DeletionResponseReceived()
{
ClientId = deletionRequest.ClientId,
RequestId = deletionRequest.RequestId,
NumberInstancesDeleted = numberInstancesDeleted,
});
}
}
void SendBatchOnceEventIsPersisted(PartitionUpdateEvent evt, EffectTracker effects, Batch batch)
{
// put the messages in the outbox where they are kept until actually sent
var commitPosition = evt.NextCommitLogPosition;
// Update the ready to send timestamp to check the delay caused
// by non-speculation
evt.ReadyToSendTimestamp = this.Partition.CurrentTimeMs;
this.Outbox[commitPosition] = batch;
batch.Position = commitPosition;
batch.Partition = this.Partition;
if (!effects.IsReplaying)
{
if (!this.Partition.Settings.PersistStepsFirst)
{
// we must not send messages until this step has been persisted
evt.OutboxBatch = batch;
DurabilityListeners.Register(evt, this);
}
else
{
// we can send the messages now
this.Send(batch);
}
}
}
public void Process(SendConfirmed evt, EffectTracker _)
{
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Store has sent all outbound messages by event {evt} id={evt.EventIdString}");
// we no longer need to keep these events around
this.Outbox.Remove(evt.Position);
}
public void Process(WaitRequestReceived evt, EffectTracker effects)
{
if (WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState))
{
if (!effects.IsReplaying)
{
this.Partition.Send(evt.CreateResponse(this.OrchestrationState));
}
}
else
{
if (this.Waiters == null)
{
this.Waiters = new List <WaitRequestReceived>();
}
else
{
// cull the list of waiters to remove requests that have already timed out
this.Waiters = this.Waiters
.Where(request => request.TimeoutUtc > DateTime.UtcNow)
.ToList();
}
this.Waiters.Add(evt);
}
}
public virtual void Process(PartitionEventFragment e, EffectTracker effects)
{
// processing a reassembled event just applies the original event
dynamic dynamicThis = this;
dynamic dynamicPartitionEvent = e.ReassembledEvent;
dynamicThis.Process(dynamicPartitionEvent, effects);
}
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
{
OrchestrationState state = this.OrchestrationState;
if (this.OrchestrationState != null &&
purgeBatchIssued.InstanceQuery.Matches(this.OrchestrationState))
{
this.OrchestrationState = null;
purgeBatchIssued.Purged.Add(this.InstanceId);
effects.Add(TrackedObjectKey.History(this.InstanceId));
}
}
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
{
var purgeRequest = (PurgeRequestReceived)this.PendingQueries[purgeBatchIssued.QueryEventId];
purgeRequest.NumberInstancesPurged += purgeBatchIssued.Purged.Count;
if (!effects.IsReplaying)
{
// lets the query that is currently in progress know that this batch is done
purgeBatchIssued.WhenProcessed.TrySetResult(null);
}
}
public void Process(ActivityCompleted evt, EffectTracker effects)
{
var batch = new Batch();
batch.OutgoingMessages.Add(new RemoteActivityResultReceived()
{
PartitionId = evt.OriginPartitionId,
Result = evt.Response,
ActivityId = evt.ActivityId,
ActivitiesQueueSize = evt.ReportedLoad,
});
this.SendBatchOnceEventIsPersisted(evt, effects, batch);
}
public void Process(CreationRequestReceived creationRequestReceived, EffectTracker effects)
{
bool filterDuplicate = this.OrchestrationState != null &&
creationRequestReceived.DedupeStatuses != null &&
creationRequestReceived.DedupeStatuses.Contains(this.OrchestrationState.OrchestrationStatus);
if (!filterDuplicate)
{
var ee = creationRequestReceived.ExecutionStartedEvent;
// set the orchestration state now (before processing the creation in the history)
// so that this new instance is "on record" immediately - it is guaranteed to replace whatever is in flight
this.OrchestrationState = new OrchestrationState
{
Name = ee.Name,
Version = ee.Version,
OrchestrationInstance = ee.OrchestrationInstance,
OrchestrationStatus = OrchestrationStatus.Pending,
ParentInstance = ee.ParentInstance,
Input = ee.Input,
Tags = ee.Tags,
CreatedTime = ee.Timestamp,
LastUpdatedTime = DateTime.UtcNow,
CompletedTime = Core.Common.DateTimeUtils.MinDateTime,
ScheduledStartTime = ee.ScheduledStartTime
};
// queue the message in the session, or start a timer if delayed
if (!ee.ScheduledStartTime.HasValue)
{
effects.Add(TrackedObjectKey.Sessions);
}
else
{
effects.Add(TrackedObjectKey.Timers);
}
}
if (!effects.IsReplaying)
{
// send response to client
effects.Partition.Send(new CreationResponseReceived()
{
ClientId = creationRequestReceived.ClientId,
RequestId = creationRequestReceived.RequestId,
Succeeded = !filterDuplicate,
ExistingInstanceOrchestrationStatus = this.OrchestrationState?.OrchestrationStatus,
});
}
}
public override void DetermineEffects(EffectTracker effects)
{
// the last-added effects are processed first
// so they can set the Purged list to contain only the instance ids that are actually purged
effects.Add(TrackedObjectKey.Queries);
effects.Add(TrackedObjectKey.Sessions);
this.Purged = new List <string>();
foreach (string instanceId in this.InstanceIds)
{
effects.Add(TrackedObjectKey.Instance(instanceId));
}
}
public void Process(BatchProcessed evt, EffectTracker effects)
{
// can add events to the history, or replace it with a new history
// update the stored history
if (this.History == null || evt.State.OrchestrationInstance.ExecutionId != this.ExecutionId)
{
this.History = new List <HistoryEvent>();
this.Episode = 0;
this.ExecutionId = evt.State.OrchestrationInstance.ExecutionId;
}
this.Partition.Assert(!string.IsNullOrEmpty(this.InstanceId) || string.IsNullOrEmpty(this.ExecutionId));
// add all the new events to the history, and update episode number
if (evt.NewEvents != null)
{
for (int i = 0; i < evt.NewEvents.Count; i++)
{
var historyEvent = evt.NewEvents[i];
if (historyEvent.EventType == EventType.OrchestratorStarted)
{
this.Episode++;
}
this.History.Add(evt.NewEvents[i]);
}
}
if (!effects.IsReplaying)
{
this.Partition.EventTraceHelper.TraceInstanceUpdate(
evt.WorkItemId,
evt.State.OrchestrationInstance.InstanceId,
evt.State.OrchestrationInstance.ExecutionId,
this.History.Count,
evt.NewEvents, this.Episode);
// if present, we keep the work item so we can reuse the execution cursor
this.CachedOrchestrationWorkItem = evt.WorkItemForReuse;
if (this.CachedOrchestrationWorkItem != null &&
this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId != evt.State.OrchestrationInstance.ExecutionId)
{
effects.Partition.EventTraceHelper.TraceEventProcessingWarning($"Dropping bad workitem cache instance={this.InstanceId} expected_executionid={evt.State.OrchestrationInstance.ExecutionId} actual_executionid={this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId}");
this.CachedOrchestrationWorkItem = null;
}
}
}
public void Process(BatchProcessed evt, EffectTracker effects)
{
// update the state of an orchestration
this.OrchestrationState = evt.State;
// if the orchestration is complete, notify clients that are waiting for it
if (this.Waiters != null && WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState))
{
if (!effects.IsReplaying)
{
foreach (var request in this.Waiters)
{
this.Partition.Send(request.CreateResponse(this.OrchestrationState));
}
}
this.Waiters = null;
}
}
public override void Process(PartitionEventFragment evt, EffectTracker effects)
{
// stores fragments until the last one is received
var originalEventString = evt.OriginalEventId.ToString();
if (evt.IsLast)
{
evt.ReassembledEvent = FragmentationAndReassembly.Reassemble <PartitionEvent>(this.Fragments[originalEventString], evt);
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Reassembled {evt.ReassembledEvent}");
this.Fragments.Remove(originalEventString);
switch (evt.ReassembledEvent)
{
case PartitionUpdateEvent updateEvent:
updateEvent.DetermineEffects(effects);
break;
case PartitionReadEvent readEvent:
this.Partition.SubmitInternalEvent(readEvent);
break;
case PartitionQueryEvent queryEvent:
this.Partition.SubmitInternalEvent(queryEvent);
break;
default:
throw new InvalidCastException("Could not cast to neither PartitionReadEvent nor PartitionUpdateEvent");
}
}
else
{
if (!this.Fragments.TryGetValue(originalEventString, out var list))
{
this.Fragments[originalEventString] = list = new List <PartitionEventFragment>();
}
list.Add(evt);
}
}
public void Process(BatchProcessed evt, EffectTracker effects)
{
var sorted = new Dictionary <uint, TaskMessagesReceived>();
foreach (var message in evt.RemoteMessages)
{
var instanceId = message.OrchestrationInstance.InstanceId;
var destination = this.Partition.PartitionFunction(instanceId);
if (!sorted.TryGetValue(destination, out var outmessage))
{
sorted[destination] = outmessage = new TaskMessagesReceived()
{
PartitionId = destination,
WorkItemId = evt.WorkItemId,
};
}
if (Entities.IsDelayedEntityMessage(message, out _))
{
(outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List <TaskMessage>())).Add(message);
}
else if (message.Event is ExecutionStartedEvent executionStartedEvent && executionStartedEvent.ScheduledStartTime.HasValue)
{
(outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List <TaskMessage>())).Add(message);
}
protected override async Task Process(IList <PartitionEvent> batch)
{
try
{
var effects = new EffectTracker(
this.partition,
this.ApplyToStore,
() => (this.commitPosition, this.inputQueuePosition)
);
if (batch.Count != 0)
{
foreach (var partitionEvent in batch)
{
// record the current time, for measuring latency in the event processing pipeline
partitionEvent.IssuedTimestamp = this.partition.CurrentTimeMs;
try
{
switch (partitionEvent)
{
case PartitionUpdateEvent updateEvent:
updateEvent.NextCommitLogPosition = this.commitPosition + 1;
await effects.ProcessUpdate(updateEvent).ConfigureAwait(false);
DurabilityListeners.ConfirmDurable(updateEvent);
if (updateEvent.NextCommitLogPosition > 0)
{
this.partition.Assert(updateEvent.NextCommitLogPosition > this.commitPosition);
this.commitPosition = updateEvent.NextCommitLogPosition;
}
break;
case PartitionReadEvent readEvent:
readEvent.OnReadIssued(this.partition);
if (readEvent.Prefetch.HasValue)
{
var prefetchTarget = this.GetOrAdd(readEvent.Prefetch.Value);
effects.ProcessReadResult(readEvent, readEvent.Prefetch.Value, prefetchTarget);
}
var readTarget = this.GetOrAdd(readEvent.ReadTarget);
effects.ProcessReadResult(readEvent, readEvent.ReadTarget, readTarget);
break;
case PartitionQueryEvent queryEvent:
var instances = this.QueryOrchestrationStates(queryEvent.InstanceQuery);
var backgroundTask = Task.Run(() => effects.ProcessQueryResultAsync(queryEvent, instances.ToAsyncEnumerable()));
break;
default:
throw new InvalidCastException("could not cast to neither PartitionReadEvent nor PartitionUpdateEvent");
}
if (partitionEvent.NextInputQueuePosition > 0)
{
this.partition.Assert(partitionEvent.NextInputQueuePosition > this.inputQueuePosition);
this.inputQueuePosition = partitionEvent.NextInputQueuePosition;
}
}
catch (Exception e)
{
this.partition.ErrorHandler.HandleError(nameof(Process), $"Encountered exception while processing event {partitionEvent}", e, false, false);
}
}
}
}
catch (Exception e)
{
this.logger.LogError("Exception in MemoryQueue BatchWorker: {exception}", e);
}
}
public ValueTask ApplyToStore(TrackedObjectKey key, EffectTracker tracker)
{
tracker.ProcessEffectOn(this.GetOrAdd(key));
return(default);
public override void DetermineEffects(EffectTracker effects)
{
effects.Add(TrackedObjectKey.Reassembly);
}
public sealed override void DetermineEffects(EffectTracker effects)
{
effects.Add(TrackedObjectKey.Prefetch);
}
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
{
this.DeleteHistory();
}
public override void DetermineEffects(EffectTracker effects)
{
effects.Add(TrackedObjectKey.Dedup);
}
public void Process(DeletionRequestReceived deletionRequestReceived, EffectTracker effects)
{
this.DeleteHistory();
}