protected CoreProjectionCheckpointManager(
IPublisher publisher, Guid projectionCorrelationId, ProjectionConfig projectionConfig, string name,
PositionTagger positionTagger, ProjectionNamesBuilder namingBuilder, bool usePersistentCheckpoints,
bool producesRunningResults)
{
if (publisher == null) throw new ArgumentNullException("publisher");
if (projectionConfig == null) throw new ArgumentNullException("projectionConfig");
if (name == null) throw new ArgumentNullException("name");
if (positionTagger == null) throw new ArgumentNullException("positionTagger");
if (namingBuilder == null) throw new ArgumentNullException("namingBuilder");
if (name == "") throw new ArgumentException("name");
_lastProcessedEventPosition = new PositionTracker(positionTagger);
_zeroTag = positionTagger.MakeZeroCheckpointTag();
_publisher = publisher;
_projectionCorrelationId = projectionCorrelationId;
_projectionConfig = projectionConfig;
_logger = LogManager.GetLoggerFor<CoreProjectionCheckpointManager>();
_namingBuilder = namingBuilder;
_usePersistentCheckpoints = usePersistentCheckpoints;
_producesRunningResults = producesRunningResults;
_requestedCheckpointState = new PartitionState("", null, _zeroTag);
_currentProjectionState = new PartitionState("", null, _zeroTag);
}
public void given()
{
//given
_cache = new PartitionStateCache(CheckpointTag.FromPosition(0, -1));
_cachedAtCheckpointTag = CheckpointTag.FromPosition(1000, 900);
_cache.CacheAndLockPartitionState("partition", new PartitionState("data", null, _cachedAtCheckpointTag), _cachedAtCheckpointTag);
_relockedData = _cache.TryGetAndLockPartitionState("partition", CheckpointTag.FromPosition(2000, 1900));
}
private void AssertCorrect(string state, string result = null)
{
var partitionState = new PartitionState(state, result, CheckpointTag.FromPosition(0, 100, 50));
var serialized = partitionState.Serialize();
var deserialized = PartitionState.Deserialize(serialized, CheckpointTag.FromPosition(0, 100, 50));
Assert.AreEqual(partitionState.State, deserialized.State);
Assert.AreEqual(partitionState.Result, deserialized.Result);
}
protected override void Reply(PartitionState state, CheckpointTag checkpointTag)
{
if (state == null)
_envelope.ReplyWith(
new CoreProjectionManagementMessage.ResultReport(
_correlationId, _projectionId, _partition, null, checkpointTag));
else
_envelope.ReplyWith(
new CoreProjectionManagementMessage.ResultReport(
_correlationId, _projectionId, _partition, state.Result, checkpointTag));
}
public void StateUpdated(string partition, PartitionState state, CheckpointTag basedOn)
{
State stateEntry;
if (_states.TryGetValue(partition, out stateEntry))
{
stateEntry.PartitionState = state;
}
else
{
_states.Add(partition, new State { PartitionState = state, ExpectedTag = basedOn});
}
}
public void CachePartitionState(string partition, PartitionState data)
{
if (partition == null) throw new ArgumentNullException("partition");
if (data == null) throw new ArgumentNullException("data");
if (partition == null) throw new ArgumentException("Root partition must be locked", "partition");
_partitionStates[partition] = Tuple.Create(data, _zeroPosition);
_cachedItemCount = _partitionStates.Count;
_cacheOrder.AddFirst(Tuple.Create(_zeroPosition, partition));
CleanUp();
}
public void CacheAndLockPartitionState(string partition, PartitionState data, CheckpointTag at)
{
if (partition == null) throw new ArgumentNullException("partition");
if (data == null) throw new ArgumentNullException("data");
EnsureCanLockPartitionAt(partition, at);
_partitionStates[partition] = Tuple.Create(data, at);
_cachedItemCount = _partitionStates.Count;
if (!string.IsNullOrEmpty(partition)) // cached forever - for root state
_cacheOrder.AddLast(Tuple.Create(at, partition));
CleanUp();
}
public EventProcessedResult(
string partition, CheckpointTag checkpointTag, PartitionState oldState, PartitionState newState,
EmittedEvent[] emittedEvents, Guid causedBy, string correlationId)
{
if (partition == null) throw new ArgumentNullException("partition");
if (checkpointTag == null) throw new ArgumentNullException("checkpointTag");
_emittedEvents = emittedEvents;
_causedBy = causedBy;
_correlationId = correlationId;
_oldState = oldState;
_newState = newState;
_partition = partition;
_checkpointTag = checkpointTag;
}
protected override void Reply(PartitionState state, CheckpointTag checkpointTag)
{
if (state == null)
_publisher.Publish(
new CoreProjectionStatusMessage.ResultReport(
_correlationId,
_projectionId,
_partition,
null,
checkpointTag));
else
_publisher.Publish(
new CoreProjectionStatusMessage.ResultReport(
_correlationId,
_projectionId,
_partition,
state.Result,
checkpointTag));
}
public virtual void Start(CheckpointTag checkpointTag, PartitionState rootPartitionState)
{
Contract.Requires(_currentCheckpoint == null);
if (_started)
{
throw new InvalidOperationException("Already started");
}
_started = true;
if (rootPartitionState != null)
{
_currentProjectionState = rootPartitionState;
}
_lastProcessedEventPosition.UpdateByCheckpointTagInitial(checkpointTag);
_lastProcessedEventProgress = -1;
_lastCompletedCheckpointPosition = checkpointTag;
_requestedCheckpointPosition = null;
_currentCheckpoint = CreateProjectionCheckpoint(_lastProcessedEventPosition.LastTag);
_currentCheckpoint.Start();
}
public void CacheAndLockPartitionState(string partition, PartitionState data, CheckpointTag at)
{
if (partition == null)
{
throw new ArgumentNullException("partition");
}
if (data == null)
{
throw new ArgumentNullException("data");
}
EnsureCanLockPartitionAt(partition, at);
_partitionStates[partition] = Tuple.Create(data, at);
_cachedItemCount = _partitionStates.Count;
if (!string.IsNullOrEmpty(partition)) // cached forever - for root state
{
_cacheOrder.AddLast(Tuple.Create(at, partition));
}
CleanUp();
}
public void CachePartitionState(string partition, PartitionState data)
{
if (partition == null)
{
throw new ArgumentNullException("partition");
}
if (data == null)
{
throw new ArgumentNullException("data");
}
if (partition == null)
{
throw new ArgumentException("Root partition must be locked", "partition");
}
_partitionStates[partition] = Tuple.Create(data, _zeroPosition);
_cachedItemCount = _partitionStates.Count;
_cacheOrder.AddFirst(Tuple.Create(_zeroPosition, partition));
CleanUp();
}
public void Handle(CoreProjectionProcessingMessage.CheckpointLoaded message)
{
EnsureState(State.LoadStateRequested);
try {
var checkpointTag = message.CheckpointTag;
var phase = checkpointTag == null ? 0 : checkpointTag.Phase;
var projectionProcessingPhase = _projectionProcessingPhases[phase];
if (checkpointTag == null)
{
checkpointTag = projectionProcessingPhase.MakeZeroCheckpointTag();
}
checkpointTag = projectionProcessingPhase.AdjustTag(checkpointTag);
//TODO: initialize projection state here (test it)
//TODO: write test to ensure projection state is correctly loaded from a checkpoint and posted back when enough empty records processed
//TODO: handle errors
_coreProjectionCheckpointWriter.StartFrom(checkpointTag, message.CheckpointEventNumber);
PartitionState rootPartitionState = null;
if (_requiresRootPartition)
{
rootPartitionState = PartitionState.Deserialize(message.CheckpointData, checkpointTag);
_partitionStateCache.CacheAndLockPartitionState("", rootPartitionState, null);
}
BeginPhase(projectionProcessingPhase, checkpointTag, rootPartitionState);
GoToState(State.StateLoaded);
if (_startOnLoad)
{
_projectionProcessingPhase.Subscribe(checkpointTag, fromCheckpoint: true);
}
else
{
GoToState(State.Stopped);
}
} catch (Exception ex) {
SetFaulted(ex);
}
}
protected EventProcessedResult InternalPartitionDeletedProcessed(
string partition, CheckpointTag deletePosition,
PartitionState newPartitionState
)
{
var oldState = _partitionStateCache.GetLockedPartitionState(partition);
var oldSharedState = _isBiState ? _partitionStateCache.GetLockedPartitionState("") : null;
bool changed = oldState.IsChanged(newPartitionState);
PartitionState partitionState = null;
// NOTE: projectionResult cannot change independently unless projection definition has changed
if (changed)
{
var lockPartitionStateAt = partition != "" ? deletePosition : null;
partitionState = newPartitionState;
_partitionStateCache.CacheAndLockPartitionState(partition, partitionState, lockPartitionStateAt);
}
return(new EventProcessedResult(
partition, deletePosition, oldState, partitionState, oldSharedState, null, null, Guid.Empty, null,
isPartitionTombstone: true));
}
protected override void Reply(PartitionState state, CheckpointTag checkpointTag)
{
if (state == null)
{
_publisher.Publish(
new CoreProjectionStatusMessage.ResultReport(
_correlationId,
_projectionId,
_partition,
null,
checkpointTag));
}
else
{
_publisher.Publish(
new CoreProjectionStatusMessage.ResultReport(
_correlationId,
_projectionId,
_partition,
state.Result,
checkpointTag));
}
}
public void StateUpdated(
string partition, PartitionState oldState, PartitionState newState, Guid causedBy, string correlationId)
{
if (_stopped)
{
return;
}
EnsureStarted();
if (_stopping)
{
throw new InvalidOperationException("Stopping");
}
if (oldState.Result != newState.Result)
{
var result = ResultUpdated(partition, oldState, newState);
if (result != null)
{
EventsEmitted(result, causedBy, correlationId);
}
}
if (_emitPartitionCheckpoints && partition != "")
{
CapturePartitionStateUpdated(partition, oldState, newState);
}
if (partition == "" && newState.State == null) // ignore non-root partitions and non-changed states
{
throw new NotSupportedException("Internal check");
}
if (partition == "")
{
_currentProjectionState = newState;
}
}
public EventProcessedResult(
string partition, CheckpointTag checkpointTag, PartitionState oldState, PartitionState newState,
PartitionState oldSharedState, PartitionState newSharedState, EmittedEventEnvelope[] emittedEvents,
Guid causedBy, string correlationId, bool isPartitionTombstone = false)
{
if (partition == null)
{
throw new ArgumentNullException("partition");
}
if (checkpointTag == null)
{
throw new ArgumentNullException("checkpointTag");
}
_emittedEvents = emittedEvents;
_causedBy = causedBy;
_correlationId = correlationId;
_isPartitionTombstone = isPartitionTombstone;
_oldState = oldState;
_newState = newState;
_oldSharedState = oldSharedState;
_newSharedState = newSharedState;
_partition = partition;
_checkpointTag = checkpointTag;
}
private EventProcessedResult InternalProcessCommittedEvent(
string partition, EventReaderSubscriptionMessage.CommittedEventReceived message)
{
string newState;
string projectionResult;
EmittedEventEnvelope[] emittedEvents;
//TODO: support shared state
string newSharedState;
var hasBeenProcessed = SafeProcessEventByHandler(
partition, message, out newState, out newSharedState, out projectionResult, out emittedEvents);
if (hasBeenProcessed)
{
var newPartitionState = new PartitionState(newState, projectionResult, message.CheckpointTag);
var newSharedPartitionState = newSharedState != null
? new PartitionState(newSharedState, null, message.CheckpointTag)
: null;
return(InternalCommittedEventProcessed(
partition, message, emittedEvents, newPartitionState, newSharedPartitionState));
}
return(null);
}
protected override void Reply(PartitionState state, CheckpointTag checkpointTag)
{
_envelope.ReplyWith(
new CoreProjectionManagementMessage.StateReport(
_correlationId, _projectionId, _partition, state.State, checkpointTag));
}
private void LoadCompleted(PartitionState state)
{
NextStage();
}
public bool IsChanged(PartitionState newState)
{
return State != newState.State || Result != newState.Result;
}
protected abstract void CapturePartitionStateUpdated(string partition, PartitionState oldState, PartitionState newState);
public void StateUpdated(
string partition, PartitionState oldState, PartitionState newState)
{
if (_stopped)
return;
EnsureStarted();
if (_stopping)
throw new InvalidOperationException("Stopping");
if (_usePersistentCheckpoints && partition != "")
CapturePartitionStateUpdated(partition, oldState, newState);
if (partition == "" && newState.State == null) // ignore non-root partitions and non-changed states
throw new NotSupportedException("Internal check");
if (partition == "")
_currentProjectionState = newState;
}
protected abstract void Reply(PartitionState state, CheckpointTag checkpointTag);
private void OnLoadPartitionStateReadStreamEventsBackwardCompleted(
ClientMessage.ReadStreamEventsBackwardCompleted message, CheckpointTag requestedStateCheckpointTag,
Action<PartitionState> loadCompleted, string partitionStreamName, string stateEventType)
{
//NOTE: the following remove may do nothing in tests as completed is raised before we return from publish.
_loadStateRequests.Remove(message.CorrelationId);
_readRequestsInProgress--;
if (message.Events.Length == 1)
{
EventRecord @event = message.Events[0].Event;
if (@event.EventType == stateEventType)
{
var parsed = @event.Metadata.ParseCheckpointTagVersionExtraJson(_projectionVersion);
if (parsed.Version.ProjectionId != _projectionVersion.ProjectionId
|| _projectionVersion.Epoch > parsed.Version.Version)
{
var state = new PartitionState("", null, _zeroTag);
loadCompleted(state);
return;
}
else
{
var loadedStateCheckpointTag = parsed.AdjustBy(_positionTagger, _projectionVersion);
// always recovery mode? skip until state before current event
//TODO: skip event processing in case we know i has been already processed
if (loadedStateCheckpointTag < requestedStateCheckpointTag)
{
var state = PartitionState.Deserialize(
Helper.UTF8NoBom.GetString(@event.Data), loadedStateCheckpointTag);
loadCompleted(state);
return;
}
}
}
}
if (message.NextEventNumber == -1)
{
var state = new PartitionState("", null, _zeroTag);
loadCompleted(state);
return;
}
_readRequestsInProgress++;
var corrId = Guid.NewGuid();
var requestId =
_readDispatcher.Publish(
new ClientMessage.ReadStreamEventsBackward(
corrId, corrId, _readDispatcher.Envelope, partitionStreamName, message.NextEventNumber, 1,
resolveLinkTos: false, requireMaster: false, validationStreamVersion: null, user: SystemAccount.Principal),
m =>
OnLoadPartitionStateReadStreamEventsBackwardCompleted(m, requestedStateCheckpointTag, loadCompleted, partitionStreamName, stateEventType));
if (requestId != Guid.Empty)
_loadStateRequests.Add(requestId);
}
public void StateUpdated(string partition, PartitionState oldState, PartitionState newState)
{
throw new NotImplementedException();
}
public void StateUpdated(string partition, PartitionState oldState, PartitionState newState)
{
if (_stopped)
return;
EnsureStarted();
if (_stopping)
throw new InvalidOperationException("Stopping");
if (partition == "" && newState.State == null) // ignore non-root partitions and non-changed states
throw new NotSupportedException("Internal check");
}
private EmittedEvent[] ResultUpdated(string partition, PartitionState oldState, PartitionState newState)
{
return(_resultEmitter.ResultUpdated(partition, newState.Result, newState.CausedBy));
}
protected override void CapturePartitionStateUpdated(string partition, PartitionState oldState, PartitionState newState)
{
if (_partitionStateUpdateManager == null)
{
_partitionStateUpdateManager = new PartitionStateUpdateManager(_namingBuilder);
}
_partitionStateUpdateManager.StateUpdated(partition, newState, oldState.CausedBy);
}
private EmittedEvent[] ResultUpdated(string partition, PartitionState oldState, PartitionState newState)
{
return _resultEmitter.ResultUpdated(partition, newState.Result, newState.CausedBy);
}
protected override void Reply(PartitionState state, CheckpointTag checkpointTag)
{
_envelope.ReplyWith(
new CoreProjectionManagementMessage.ResultReport(
_correlationId, _projectionId, _partition, state.Result, checkpointTag));
}
public override void Start(CheckpointTag checkpointTag, PartitionState rootPartitionState)
{
base.Start(checkpointTag, rootPartitionState);
_orderStream = CreateOrderStream(checkpointTag);
_orderStream.Start();
}
private void CapturePartitionStateUpdated(string partition, PartitionState oldState, PartitionState newState)
{
if (_partitionStateUpdateManager == null)
_partitionStateUpdateManager = new PartitionStateUpdateManager(_namingBuilder);
_partitionStateUpdateManager.StateUpdated(partition, newState, oldState.CausedBy);
}
private void BeginPhase(IProjectionProcessingPhase processingPhase, CheckpointTag startFrom, PartitionState rootPartitionState)
{
_projectionProcessingPhase = processingPhase;
_projectionProcessingPhase.SetProjectionState(PhaseState.Starting);
_checkpointManager = processingPhase.CheckpointManager;
_projectionProcessingPhase.InitializeFromCheckpoint(startFrom);
_checkpointManager.Start(startFrom, rootPartitionState);
}
public void StateUpdated(
string partition, PartitionState oldState, PartitionState newState, Guid causedBy, string correlationId)
{
if (_stopped)
return;
EnsureStarted();
if (_stopping)
throw new InvalidOperationException("Stopping");
if (oldState.Result != newState.Result)
{
var result = ResultUpdated(partition, oldState, newState);
if (result != null)
EventsEmitted(result, causedBy, correlationId);
}
if (_emitPartitionCheckpoints && partition != "")
CapturePartitionStateUpdated(partition, oldState, newState);
if (partition == "" && newState.State == null) // ignore non-root partitions and non-changed states
throw new NotSupportedException("Internal check");
if (partition == "")
_currentProjectionState = newState;
}
public virtual void Initialize()
{
if (_currentCheckpoint != null) _currentCheckpoint.Dispose();
if (_closingCheckpoint != null) _closingCheckpoint.Dispose();
_currentCheckpoint = null;
_closingCheckpoint = null;
_requestedCheckpointPosition = null;
_inCheckpoint = false;
_requestedCheckpointState = new PartitionState("", null, _zeroTag);
_lastCompletedCheckpointPosition = null;
_lastProcessedEventPosition.Initialize();
_lastProcessedEventProgress = -1;
_eventsProcessedAfterRestart = 0;
_started = false;
_stopping = false;
_stopped = false;
_currentProjectionState = new PartitionState("", null, _zeroTag);
}
private void LoadCompleted(PartitionState state)
{
NextStage();
}
/// <returns>true - if checkpoint has been completed in-sync</returns>
private bool StartCheckpoint(PositionTracker lastProcessedEventPosition, PartitionState projectionState)
{
Contract.Requires(_closingCheckpoint == null);
if (projectionState == null) throw new ArgumentNullException("projectionState");
CheckpointTag requestedCheckpointPosition = lastProcessedEventPosition.LastTag;
if (requestedCheckpointPosition == _lastCompletedCheckpointPosition)
return true; // either suggested or requested to stop
if (_usePersistentCheckpoints) // do not emit any events if we do not use persistent checkpoints
EmitPartitionCheckpoints();
_inCheckpoint = true;
_requestedCheckpointPosition = requestedCheckpointPosition;
_requestedCheckpointState = projectionState;
_closingCheckpoint = _currentCheckpoint;
_currentCheckpoint = CreateProjectionCheckpoint(requestedCheckpointPosition);
// checkpoint only after assigning new current checkpoint, as it may call back immediately
_closingCheckpoint.Prepare(requestedCheckpointPosition);
return false; // even if prepare completes in sync it notifies the world by a message
}
private void LoadCompleted(PartitionState obj)
{
_state = obj;
NextStage();
}
protected abstract void CapturePartitionStateUpdated(string partition, PartitionState oldState,
PartitionState newState);
protected abstract void Reply(PartitionState state, CheckpointTag checkpointTag);
private void LoadCompleted(PartitionState obj)
{
_state = obj;
NextStage();
}
public bool IsChanged(PartitionState newState)
{
return(State != newState.State || Result != newState.Result);
}