private static void NotifyEventCommitted(EmittedEvent @event, int eventNumber)
{
if (@event.OnCommitted != null)
{
@event.OnCommitted(eventNumber);
}
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category1, ResolvedEvent data,
out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (data.PositionSequenceNumber != 0)
return false; // not our event
if (data.EventStreamId.StartsWith("$"))
return false;
var lastSlashPos = data.EventStreamId.LastIndexOf(_separator);
if (lastSlashPos < 0)
return true; // handled but not interesting to us
var category = data.EventStreamId.Substring(0, lastSlashPos);
emittedEvents = new[]
{
new EmittedDataEvent(
"$category" + _separator + category, Guid.NewGuid(), "StreamCreated", data.EventStreamId, null,
eventPosition, expectedTag: null)
};
return true;
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category1, ResolvedEvent data,
out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (data.PositionStreamId.StartsWith("$"))
return false;
var lastSlashPos = data.PositionStreamId.LastIndexOf(_separator);
if (lastSlashPos < 0)
return true; // handled but not interesting to us
var category = data.PositionStreamId.Substring(0, lastSlashPos);
string linkTarget;
if (data.EventType == SystemEventTypes.LinkTo)
linkTarget = data.Data;
else
linkTarget = data.EventSequenceNumber + "@" + data.EventStreamId;
emittedEvents = new[]
{
new EmittedLinkToWithRecategorization(
_categoryStreamPrefix + category, Guid.NewGuid(), linkTarget, eventPosition, expectedTag: null,
originalStreamId: data.PositionStreamId)
};
return true;
}
private IEnumerable <KeyValuePair <string, JToken> > MetadataWithCausedByAndCorrelationId(
EmittedEvent emittedEvent)
{
var extraMetaData = emittedEvent.ExtraMetaData();
var correlationIdFound = false;
if (extraMetaData != null)
{
foreach (var valuePair in from pair in extraMetaData
where pair.Key != "$causedBy"
select pair)
{
if (valuePair.Key == "$correlationId")
{
correlationIdFound = true;
}
yield return(new KeyValuePair <string, JToken>(valuePair.Key, new JRaw(valuePair.Value)));
}
}
if (emittedEvent.CausedBy != Guid.Empty)
{
yield return
(new KeyValuePair <string, JToken>(
"$causedBy", JValue.CreateString(emittedEvent.CausedBy.ToString("D"))));
}
if (!correlationIdFound && !string.IsNullOrEmpty(emittedEvent.CorrelationId))
{
yield return
(new KeyValuePair <string, JToken>("$correlationId", JValue.CreateString(emittedEvent.CorrelationId)));
}
}
private InvalidEmittedEventSequenceException CreateSequenceException(
Tuple <CheckpointTag, string, long> committed, EmittedEvent eventToWrite)
{
return(new InvalidEmittedEventSequenceException(
$"An event emitted in recovery for stream {_streamId} differs from the originally emitted event. Existing('{committed.Item2}', '{committed.Item1}'). New('{eventToWrite.EventType}', '{eventToWrite.CausedByTag}')"
));
}
private IValidatedEmittedEvent ValidateEmittedEventInRecoveryMode(EmittedEvent eventToWrite)
{
var topAlreadyCommitted = _alreadyCommittedEvents.Pop();
if (topAlreadyCommitted.Item1 < eventToWrite.CausedByTag)
{
return(new IgnoredEmittedEvent());
}
var failed = topAlreadyCommitted.Item1 != eventToWrite.CausedByTag ||
topAlreadyCommitted.Item2 != eventToWrite.EventType;
if (failed && eventToWrite.EventType.Equals(LinkEventType))
{
// We check if the linked event still exists. If not, we skip that emitted event.
var parts = eventToWrite.Data.Split(LinkToSeparator, 2);
var streamId = parts[1];
if (!long.TryParse(parts[0], out long eventNumber))
{
throw new Exception($"Unexpected exception: Emitted event is an invalid link event: Body ({eventToWrite.Data}) CausedByTag ({eventToWrite.CausedByTag}) StreamId ({eventToWrite.StreamId})");
}
return(new EmittedEventResolutionNeeded(streamId, eventNumber, topAlreadyCommitted));
}
if (failed)
{
var error = CreateSequenceException(topAlreadyCommitted, eventToWrite);
return(new ErroredEmittedEvent(error));
}
return(new ValidEmittedEvent(topAlreadyCommitted.Item1, topAlreadyCommitted.Item2, topAlreadyCommitted.Item3));
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category1, ResolvedEvent data,
out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
return true;
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category1, ResolvedEvent data,
out string newState, out EmittedEvent[] emittedEvents)
{
if (data.EventType == "fail" || _query == "fail")
throw new Exception("failed");
_logger("ProcessEvent(" + "..." + ")");
newState = "{\"data\": " + _state + data + "}";
emittedEvents = null;
return true;
}
public static bool ProcessEvent(
this IProjectionStateHandler self, string partition, CheckpointTag eventPosition, string streamId,
string eventType, string category, Guid eventId, int eventSequenceNumber, string metadata, string data,
out string state, out EmittedEvent[] emittedEvents, bool isJson = true)
{
return self.ProcessEvent(
partition, eventPosition, category,
new ResolvedEvent(
streamId, eventSequenceNumber, streamId, eventSequenceNumber, false, new TFPos(0, -1),
eventId, eventType, isJson, data, metadata), out state, out emittedEvents);
}
public void ValidateOrderAndEmitEvents(EmittedEvent[] events)
{
ValidatePosition(events);
EnsureCheckpointNotRequested();
var groupedEvents = events.GroupBy(v => v.StreamId);
foreach (var eventGroup in groupedEvents)
{
EmitEventsToStream(eventGroup.Key, eventGroup.ToArray());
}
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string streamId, string eventType, string category,
Guid eventid, int sequenceNumber, string metadata, string data, out string newState,
out EmittedEvent[] emittedEvents)
{
if (eventType == "fail" || _query == "fail")
throw new Exception("failed");
_logger("ProcessEvent(" + "..." + ")");
newState = "{\"data\": 1}";
emittedEvents = null;
return true;
}
// Used when we need to resolve a link event to see if it points to an event that no longer exists. If that
// event no longer exists, we skip it and resume the recovery process.
private void OnEmittedLinkEventResolved(bool anyFound, EmittedEvent eventToWrite, ClientMessage.ReadEventCompleted resp)
{
var topAlreadyCommitted = _alreadyCommittedEvents.Pop();
if (resp.Result != ReadEventResult.StreamDeleted && resp.Result != ReadEventResult.NotFound)
{
throw CreateSequenceException(topAlreadyCommitted, eventToWrite);
}
Log.Verbose($"Emitted event ignored after resolution because it links to an event that no longer exists: eventId: {eventToWrite.EventId}, eventType: {eventToWrite.EventId}, checkpoint: {eventToWrite.CorrelationId}, causedBy: {eventToWrite.CausedBy}");
_pendingWrites.Dequeue();
SubmitWriteEventsInRecoveryLoop(anyFound);
}
public bool ProcessEvent(
EventPosition position, CheckpointTag eventPosition, string streamId, string eventType, string category1, Guid eventId,
int sequenceNumber, string metadata, string data, out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (sequenceNumber != 0)
return false; // not our event
emittedEvents = new[] { new EmittedEvent(SystemStreams.StreamsStream, Guid.NewGuid(), SystemEventTypes.LinkTo, sequenceNumber + "@" + streamId, eventPosition, expectedTag: null) };
return true;
}
public bool ProcessEvent(
EventPosition position, string streamId, string eventType, string category1, Guid eventId,
int sequenceNumber, string metadata, string data, out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (sequenceNumber != 0)
return false; // not our event
emittedEvents = new[]
{new EmittedEvent("$streams", Guid.NewGuid(), "$>", sequenceNumber + "@" + streamId)};
return true;
}
public EventProcessedResult(
string partition, CheckpointTag checkpointTag, PartitionState oldState, PartitionState newState,
EmittedEvent[] emittedEvents, Guid causedBy, string correlationId)
{
if (partition == null) throw new ArgumentNullException("partition");
if (checkpointTag == null) throw new ArgumentNullException("checkpointTag");
_emittedEvents = emittedEvents;
_causedBy = causedBy;
_correlationId = correlationId;
_oldState = oldState;
_newState = newState;
_partition = partition;
_checkpointTag = checkpointTag;
}
public void TrackEmittedStream(EmittedEvent[] emittedEvents)
{
if (!_projectionConfig.TrackEmittedStreams) return;
foreach (var emittedEvent in emittedEvents)
{
string streamId;
if (!_streamIdCache.TryGetRecord(emittedEvent.StreamId, out streamId))
{
var trackEvent = new Event(Guid.NewGuid(), ProjectionEventTypes.StreamTracked, false, Helper.UTF8NoBom.GetBytes(emittedEvent.StreamId), null);
lock (_locker)
{
_streamIdCache.PutRecord(emittedEvent.StreamId, emittedEvent.StreamId, false);
}
WriteEvent(trackEvent, MaxRetryCount);
}
}
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category1, ResolvedEvent data,
out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (data.EventSequenceNumber != 0)
return false; // not our event
emittedEvents = new[]
{
new EmittedDataEvent(
SystemStreams.StreamsStream, Guid.NewGuid(), SystemEventTypes.LinkTo,
data.EventSequenceNumber + "@" + data.EventStreamId, null, eventPosition, expectedTag: null)
};
return true;
}
public bool ProcessEvent(
EventPosition position, string streamId, string eventType, string category1, Guid eventId,
int sequenceNumber, string metadata, string data, out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (sequenceNumber != 0)
return false; // not our event
var lastSlashPos = streamId.LastIndexOf(_separator);
if (lastSlashPos < 0)
return true; // handled but not interesting to us
var category = streamId.Substring(0, lastSlashPos);
emittedEvents = new[]
{new EmittedEvent("$category" + _separator + category, Guid.NewGuid(), "StreamCreated", streamId)};
return true;
}
// Used when we need to resolve a link event to see if it points to an event that no longer exists. If that
// event no longer exists, we skip it and resume the recovery process.
private void OnEmittedLinkEventResolved(bool anyFound, EmittedEvent eventToWrite, Tuple <CheckpointTag, string, long> topAlreadyCommitted, ClientMessage.ReadEventCompleted resp)
{
if (resp.Result != ReadEventResult.StreamDeleted && resp.Result != ReadEventResult.NotFound && resp.Result != ReadEventResult.NoStream && resp.Result != ReadEventResult.Success)
{
throw CreateSequenceException(topAlreadyCommitted, eventToWrite);
}
if (resp.Result == ReadEventResult.Success)
{
anyFound = true;
NotifyEventCommitted(eventToWrite, topAlreadyCommitted.Item3);
}
else
{
Log.Verbose($"Emitted event ignored after resolution because it links to an event that no longer exists: eventId: {eventToWrite.EventId}, eventType: {eventToWrite.EventId}, checkpoint: {eventToWrite.CorrelationId}, causedBy: {eventToWrite.CausedBy}");
}
_pendingWrites.Dequeue();
_awaitingLinkToResolution = false;
SubmitWriteEventsInRecoveryLoop(anyFound);
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category1, ResolvedEvent data,
out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (data.EventStreamId != data.PositionStreamId)
return false;
if (data.EventType == "$>")
return false;
emittedEvents = new[]
{
new EmittedDataEvent(
_indexStreamPrefix + data.EventType, Guid.NewGuid(), "$>",
data.EventSequenceNumber + "@" + data.EventStreamId, null, eventPosition, expectedTag: null)
};
return true;
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string streamId, string eventType, string category1,
Guid eventId, int sequenceNumber, string metadata, string data, out string newState,
out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (streamId.StartsWith("$"))
return false;
if (eventType == "$>")
return false;
emittedEvents = new[]
{
new EmittedEvent(
_indexStreamPrefix + eventType, Guid.NewGuid(), "$>", sequenceNumber + "@" + streamId,
eventPosition, expectedTag: null)
};
return true;
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string streamId, string eventType, string category1, Guid eventId, int sequenceNumber, string metadata,
string data, out string newState, out EmittedEvent[] emittedEvents)
{
emittedEvents = null;
newState = null;
if (streamId.StartsWith("$"))
return false;
var lastSlashPos = streamId.LastIndexOf(_separator);
if (lastSlashPos < 0)
return true; // handled but not interesting to us
var category = streamId.Substring(0, lastSlashPos);
emittedEvents = new[]
{
new EmittedEvent(
_categoryStreamPrefix + category, Guid.NewGuid(), "$>", sequenceNumber + "@" + streamId, eventPosition, expectedTag: null)
};
return true;
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category, ResolvedEvent data, out string newState,
out EmittedEvent[] emittedEvents)
{
if (!data.EventStreamId.StartsWith(UserStreamPrefix))
throw new InvalidOperationException(
string.Format(
"Invalid stream name: '{0}' The IndexUsersProjectionHandler cannot handle events from other streams than named after the '$user-' pattern",
data.EventStreamId));
var loginName = data.EventStreamId.Substring(UserStreamPrefix.Length);
var userData = data.Data.ParseJson<UserData>();
if (userData.LoginName != loginName)
throw new InvalidOperationException(
string.Format(
"Invalid $UserCreated event found. '{0}' login name expected, but '{1}' found", loginName,
userData.LoginName));
emittedEvents = new EmittedEvent[]
{new EmittedDataEvent(UsersStream, Guid.NewGuid(), UserEventType, loginName, null, eventPosition, null)};
newState = "";
return true;
}
private Tuple <CheckpointTag, string, int> ValidateEmittedEventInRecoveryMode(EmittedEvent eventsToWrite)
{
var topAlreadyCommitted = _alreadyCommittedEvents.Pop();
if (topAlreadyCommitted.Item1 < eventsToWrite.CausedByTag)
{
return(null);
}
var failed = topAlreadyCommitted.Item1 != eventsToWrite.CausedByTag || topAlreadyCommitted.Item2 != eventsToWrite.EventType;
if (failed)
{
throw new InvalidEmittedEventSequenceExceptioin(
string.Format(
"An event emitted in recovery differ from the originally emitted event. Existing('{0}', '{1}'). New('{2}', '{3}')",
topAlreadyCommitted.Item2, topAlreadyCommitted.Item1, eventsToWrite.EventType, eventsToWrite.CausedByTag));
}
return(topAlreadyCommitted);
}
public bool ProcessEvent(
EventPosition position, string streamId, string eventType, string category, Guid eventId, int sequenceNumber,
string metadata, string data, out string newState, out EmittedEvent[] emittedEvents)
{
if (_failOnProcessEvent)
throw new Exception("PROCESS_EVENT_FAILED");
_lastProcessedStreamId = streamId;
_lastProcessedEventType = eventType;
_lastProcessedEventId = eventId;
_lastProcessedSequencenumber = sequenceNumber;
_lastProcessedMetadata = metadata;
_lastProcessedData = data;
_eventsProcessed++;
switch (eventType)
{
case "skip_this_type":
_loadedState = newState = null;
emittedEvents = null;
return false;
case "handle_this_type":
_loadedState = newState = data;
emittedEvents = null;
return true;
case "append":
_loadedState = newState = _loadedState + data;
emittedEvents = null;
return true;
case "no_state_emit1_type":
_loadedState = newState = "";
emittedEvents = new[]
{new EmittedEvent(_emit1StreamId, Guid.NewGuid(), _emit1EventType, _emit1Data),};
return true;
case "emit1_type":
_loadedState = newState = data;
emittedEvents = new[]
{new EmittedEvent(_emit1StreamId, Guid.NewGuid(), _emit1EventType, _emit1Data),};
return true;
case "emit22_type":
_loadedState = newState = data;
emittedEvents = new[]
{
new EmittedEvent(_emit2StreamId, Guid.NewGuid(), _emit2EventType, _emit1Data),
new EmittedEvent(_emit2StreamId, Guid.NewGuid(), _emit2EventType, _emit2Data),
};
return true;
case "emit212_type":
_loadedState = newState = data;
emittedEvents = new[]
{
new EmittedEvent(_emit2StreamId, Guid.NewGuid(), _emit2EventType, _emit1Data),
new EmittedEvent(_emit1StreamId, Guid.NewGuid(), _emit1EventType, _emit2Data),
new EmittedEvent(_emit2StreamId, Guid.NewGuid(), _emit2EventType, _emit3Data),
};
return true;
case "emit12_type":
_loadedState = newState = data;
emittedEvents = new[]
{
new EmittedEvent(_emit1StreamId, Guid.NewGuid(), _emit1EventType, _emit1Data),
new EmittedEvent(_emit2StreamId, Guid.NewGuid(), _emit2EventType, _emit2Data),
};
return true;
default:
throw new NotSupportedException();
}
}
private void ValidatePosition(EmittedEvent[] events)
{
foreach (var emittedEvent in events)
{
ValidatePosition(emittedEvent.CausedByTag);
_last = emittedEvent.CausedByTag;
}
}
private void EmitEventsToStream(
string streamId, EmittedEvent[] emittedEvents)
{
EmittedStream stream;
if (!_emittedStreams.TryGetValue(streamId, out stream))
{
stream = new EmittedStream(
streamId, _zero, _publisher, this /*_recoveryMode*/, maxWriteBatchLength: _maxWriteBatchLength,
logger: _logger);
if (_started)
stream.Start();
_emittedStreams.Add(streamId, stream);
}
stream.EmitEvents(emittedEvents);
}
private void UpdateLastPosition(EmittedEvent[] events)
{
foreach (var emittedEvent in events)
{
if (emittedEvent.CausedByTag > _last)
_last = emittedEvent.CausedByTag;
}
}
public void ProcessNewCheckpoint(CheckpointTag checkpointPosition, out EmittedEvent[] emittedEvents)
{
emittedEvents = new[]
{
new EmittedDataEvent(
_indexCheckpointStream, Guid.NewGuid(), "$Checkpoint", checkpointPosition.ToJsonString(), null,
checkpointPosition, expectedTag: null)
};
}
public void EventsEmitted(EmittedEvent[] scheduledWrites)
{
EnsureStarted();
if (_stopping)
throw new InvalidOperationException("Stopping");
if (scheduledWrites != null)
_currentCheckpoint.ValidateOrderAndEmitEvents(scheduledWrites);
}
private void EmitEventsToStream(string streamId, EmittedEvent[] emittedEvents)
{
EmittedStream stream;
if (!_emittedStreams.TryGetValue(streamId, out stream))
{
stream = new EmittedStream(
streamId, _projectionVersion, _runAs, _positionTagger, _zero, _from, _readDispatcher,
_writeDispatcher, this /*_recoveryMode*/, maxWriteBatchLength: _maxWriteBatchLength, logger: _logger);
if (_started)
stream.Start();
_emittedStreams.Add(streamId, stream);
}
stream.EmitEvents(emittedEvents);
}
public void EventsEmitted(EmittedEvent[] scheduledWrites, Guid causedBy, string correlationId)
{
if (_stopped)
return;
EnsureStarted();
if (_stopping)
throw new InvalidOperationException("Stopping");
if (scheduledWrites != null)
{
foreach (EmittedEvent @event in scheduledWrites)
{
@event.SetCausedBy(causedBy);
@event.SetCorrelationId(correlationId);
}
_currentCheckpoint.ValidateOrderAndEmitEvents(scheduledWrites);
}
}
public void TrackEmittedStream(EmittedEvent[] emittedEvents)
{
}
public bool ProcessEvent(
EventPosition position, CheckpointTag eventPosition, string streamId, string eventType, string category, Guid eventid,
int sequenceNumber, string metadata, string data, out string newState, out EmittedEvent[] emittedEvents)
{
CheckDisposed();
if (eventType == null)
throw new ArgumentNullException("eventType");
if (streamId == null)
throw new ArgumentNullException("streamId");
_eventPosition = eventPosition;
_emittedEvents = null;
_query.Push(
data.Trim(), // trimming data passed to a JS
new string[] {streamId, eventType, category ?? "", sequenceNumber.ToString(CultureInfo.InvariantCulture), metadata ?? "", position.PreparePosition.ToString()});
newState = _query.GetState();
emittedEvents = _emittedEvents == null ? null : _emittedEvents.ToArray();
return true;
}
private static void ValidateEmittedEventInRecoveryMode(Tuple <CheckpointTag, string, int> topAlreadyCommitted, EmittedEvent eventsToWrite)
{
if (topAlreadyCommitted.Item1 != eventsToWrite.CausedByTag || topAlreadyCommitted.Item2 != eventsToWrite.EventType)
{
throw new InvalidOperationException(
string.Format(
"An event emitted in recovery differ from the originally emitted event. Existing('{0}', '{1}'). New('{2}', '{3}')",
topAlreadyCommitted.Item2, topAlreadyCommitted.Item1, eventsToWrite.EventType, eventsToWrite.CausedByTag));
}
}
public void ValidateOrderAndEmitEvents(EmittedEvent[] events)
{
Writes.Add(events);
}
public void ValidateOrderAndEmitEvents(EmittedEvent[] events)
{
Assert.Fail("Should not write any events");
}
public bool ProcessEvent(
string partition, CheckpointTag eventPosition, string category, ResolvedEvent data, out string newState,
out EmittedEvent[] emittedEvents)
{
CheckDisposed();
if (data == null)
throw new ArgumentNullException("data");
_eventPosition = eventPosition;
_emittedEvents = null;
newState = _query.Push(
data.Data.Trim(), // trimming data passed to a JS
new[]
{
data.IsJson ? "1" : "",
data.EventStreamId, data.EventType, category ?? "", data.EventSequenceNumber.ToString(CultureInfo.InvariantCulture),
data.Metadata, partition
});
emittedEvents = _emittedEvents == null ? null : _emittedEvents.ToArray();
return true;
}
