protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream", "type2", "{}", "{Data: 3}");
GivenInitialIndexState();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
AllStreams = true,
Events = new[] {"type1", "type2"},
Options = new QuerySourcesDefinitionOptions {}
};
_readerStrategy = ReaderStrategy.Create(
"test",
0,
_sourceDefinition,
_timeProvider,
stopOnEof: false,
runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: false,
stopAfterNEvents: null);
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
ExistingEvent("catalog", "$>", null, "0@test-stream");
ExistingEvent("catalog", "$>", null, "0@test-stream2");
ExistingEvent("catalog", "$>", null, "0@test-stream3");
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
CatalogStream = "catalog",
AllEvents = true,
ByStreams = true,
Options = new QuerySourcesDefinitionOptions { }
};
_readerStrategy = ReaderStrategy.Create(0, _sourceDefinition, _timeProvider, stopOnEof: true, runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
public ResolvedEvent(
string positionStreamId, int positionSequenceNumber, string eventStreamId, int eventSequenceNumber,
bool resolvedLinkTo, TFPos position, Guid eventId, string eventType, bool isJson, string data,
string metadata, string positionMetadata = null, string streamMetadata = null)
{
DateTime timestamp = default(DateTime);
if (Guid.Empty == eventId)
throw new ArgumentException("Empty eventId provided.");
if (string.IsNullOrEmpty(eventType))
throw new ArgumentException("Empty eventType provided.");
_positionStreamId = positionStreamId;
_positionSequenceNumber = positionSequenceNumber;
_eventStreamId = eventStreamId;
_eventSequenceNumber = eventSequenceNumber;
_resolvedLinkTo = resolvedLinkTo;
_position = position;
EventId = eventId;
EventType = eventType;
IsJson = isJson;
Timestamp = timestamp;
Data = data;
Metadata = metadata;
PositionMetadata = positionMetadata;
StreamMetadata = streamMetadata;
}
public ResolvedEvent(
string positionStreamId, int positionSequenceNumber, string eventStreamId, int eventSequenceNumber,
bool resolvedLinkTo, TFPos position, TFPos originalPosition, Guid eventId, string eventType, bool isJson, byte[] data,
byte[] metadata, byte[] positionMetadata, DateTime timestamp)
{
if (Guid.Empty == eventId)
throw new ArgumentException("Empty eventId provided.");
if (string.IsNullOrEmpty(eventType))
throw new ArgumentException("Empty eventType provided.");
_positionStreamId = positionStreamId;
_positionSequenceNumber = positionSequenceNumber;
_eventStreamId = eventStreamId;
_eventSequenceNumber = eventSequenceNumber;
_resolvedLinkTo = resolvedLinkTo;
_position = position;
_originalPosition = originalPosition;
EventId = eventId;
EventType = eventType;
IsJson = isJson;
Timestamp = timestamp;
//TODO: handle utf-8 conversion exception
Data = Helper.UTF8NoBom.GetString(data);
Metadata = Helper.UTF8NoBom.GetString(metadata);
PositionMetadata = positionMetadata != null ? Helper.UTF8NoBom.GetString(positionMetadata) : null;
}
public void read_all_backward_from_beginning_of_second_chunk_returns_no_records()
{
var pos = new TFPos(10000, 10000);
var events = ReadIndex.ReadAllEventsBackward(pos, 100).Records.Select(r => r.Event).ToArray();
Assert.AreEqual(1, events.Length);
Assert.AreEqual(_event1, events[0]);
}
protected override void Given()
{
base.Given();
NoOtherStreams();
_message1Position = ExistingEvent("stream1", "message1", null, "{}");
_projectionSource = @"fromAll().when({message1: function(s,e){ throw 1; }});";
}
public void read_all_events_backward_returns_all_events_in_correct_order()
{
var pos = new TFPos(Db.Config.WriterCheckpoint.Read(), Db.Config.WriterCheckpoint.Read());
var records = ReadIndex.ReadAllEventsBackward(pos, 10, false).Records;
Assert.AreEqual(2, records.Count);
Assert.AreEqual(_id1, records[1].Event.EventId);
Assert.AreEqual(_id2, records[0].Event.EventId);
}
public IndexReadAllResult(List<CommitEventRecord> records, TFPos currentPos, TFPos nextPos, TFPos prevPos)
{
Ensure.NotNull(records, "records");
Records = records;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
}
public SubscribeAwake(
IEnvelope envelope, Guid correlationId, string streamId, TFPos @from, Message replyWithMessage)
{
StreamId = streamId;
From = @from;
ReplyWithMessage = replyWithMessage;
Envelope = envelope;
CorrelationId = correlationId;
}
public ReadAllResult(ResolvedEvent[] events, int maxCount, TFPos currentPos, TFPos nextPos, TFPos prevPos, long tfEofPosition)
{
Ensure.NotNull(events, "events");
Events = events;
MaxCount = maxCount;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
TfEofPosition = tfEofPosition;
}
public ReadAllResult(ResolvedEventRecord[] records, int maxCount, TFPos currentPos, TFPos nextPos, TFPos prevPos, long tfEofPosition)
{
Ensure.NotNull(records, "records");
Records = records;
MaxCount = maxCount;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
TfEofPosition = tfEofPosition;
}
public void GetAllBefore(HttpEntity entity, TFPos position, int count)
{
var envelope = new SendToHttpEnvelope(entity,
Format.Atom.ReadAllEventsBackwardCompleted,
Configure.ReadAllEventsBackwardCompleted);
Publish(new ClientMessage.ReadAllEventsBackward(Guid.NewGuid(),
envelope,
position.CommitPosition,
position.PreparePosition,
count,
true));
}
public void all_records_can_be_read_sequentially_page_by_page_in_backward_pass()
{
var recs = new[] { _p5, _p3, _p1, _p4, _p2 }; // in reverse committed order
int count = 0;
var pos = new TFPos(Db.Config.WriterCheckpoint.Read(), Db.Config.WriterCheckpoint.Read());
ReadAllResult result;
while ((result = ReadIndex.ReadAllEventsBackward(pos, 1, false)).Records.Count != 0)
{
Assert.AreEqual(1, result.Records.Count);
Assert.AreEqual(recs[count], result.Records[0].Event);
pos = result.NextPos;
count += 1;
}
Assert.AreEqual(recs.Length, count);
}
public void all_records_can_be_read_sequentially_page_by_page_in_forward_pass()
{
var recs = new[] {_p2, _p4, _p1, _p3, _p5}; // in committed order
int count = 0;
var pos = new TFPos(0, 0);
ReadAllResult result;
while ((result = ReadIndex.ReadAllEventsForward(pos, 1, false)).Records.Count != 0)
{
Assert.AreEqual(1, result.Records.Count);
Assert.AreEqual(recs[count], result.Records[0].Event);
pos = result.NextPos;
count += 1;
}
Assert.AreEqual(recs.Length, count);
}
public IndexReadAllResult(List<CommitEventRecord> records,
int maxCount,
TFPos currentPos,
TFPos nextPos,
TFPos prevPos,
long tfEofPosition)
{
Ensure.NotNull(records, "records");
Records = records;
MaxCount = maxCount;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
TfEofPosition = tfEofPosition;
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("stream-a", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("stream-b", "type1", "{}", "{Data: 2}");
GivenOtherEvents();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
Streams = new []{"stream-a", "stream-b"},
AllEvents = true,
Options = new QuerySourcesDefinitionOptions {ReorderEvents = true, ProcessingLag = 100}
};
_readerStrategy = ReaderStrategy.Create(0, _sourceDefinition, _timeProvider, stopOnEof: false, runAs: null);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: false,
stopAfterNEvents: null);
}
//private TFPos _tfPos2;
//private TFPos _tfPos3;
protected override void Given()
{
base.Given();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
//_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
//_tfPos3 = ExistingEvent("test-stream", "type2", "{}", "{Data: 3}");
ExistingEvent("$et-type1", "$>", TFPosToMetadata(_tfPos1), "0@test-stream");
NoStream("$et-type2");
NoStream("$et");
//NOTE: the following events should be late written
//ExistingEvent("$et-type2", "$>", "", "2@test-stream");
//ExistingEvent("$et-type1", "$>", "", "1@test-stream");
_querySourcesDefinition = new QuerySourcesDefinition
{
AllStreams = true,
Events = new[] {"type1", "type2"},
Options = new QuerySourcesDefinitionOptions {}
};
_fromPosition = CheckpointTag.FromEventTypeIndexPositions(new TFPos(0, -1), new Dictionary<string, int>{{"type1", -1}, {"type2", -1}});
_maxEvents = 1; // reading the first event
}
protected override void Given()
{
base.Given();
AllWritesSucceed();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream2", "type1", "{}", "{Data: 3}");
ExistingEvent("test-stream2", "type1", "{}", "{Data: 4}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 5}");
ExistingEvent("test-stream3", "type1", "{}", "{Data: 6}");
ExistingEvent("test-stream4", "type1", "{}", "{Data: 7}");
ExistingEvent("$$test-stream", "$metadata", "", "{Meta: 1}");
ExistingEvent("$$test-stream2", "$metadata", "", "{Meta: 2}");
ExistingEvent("$$test-stream3", "$metadata", "", "{Meta: 3}");
ExistingEvent("$streams", "$>", null, "0@test-stream");
ExistingEvent("$streams", "$>", null, "0@test-stream2");
ExistingEvent("$streams", "$>", null, "0@test-stream3");
ExistingEvent("$streams", "$>", null, "0@test-stream4");
NoOtherStreams();
_subscriptionId = Guid.NewGuid();
_sourceDefinition = new QuerySourcesDefinition
{
CatalogStream = "$all",
AllEvents = true,
ByStreams = true,
Options = new QuerySourcesDefinitionOptions { }
};
_readerStrategy = new ParallelQueryAllStreamsMasterReaderStrategy(
0, SystemAccount.Principal, _timeProvider);
_readerSubscriptionOptions = new ReaderSubscriptionOptions(
checkpointUnhandledBytesThreshold: 10000, checkpointProcessedEventsThreshold: 100, stopOnEof: true,
stopAfterNEvents: null);
}
public ResolvedEvent(
string positionStreamId, int positionSequenceNumber, string eventStreamId, int eventSequenceNumber,
bool resolvedLinkTo, TFPos position, TFPos originalPosition, Guid eventId, string eventType, bool isJson, byte[] data,
byte[] metadata, byte[] positionMetadata, byte[] streamMetadata, DateTime timestamp)
{
_positionStreamId = positionStreamId;
_positionSequenceNumber = positionSequenceNumber;
_eventStreamId = eventStreamId;
_eventSequenceNumber = eventSequenceNumber;
_resolvedLinkTo = resolvedLinkTo;
_position = position;
_originalPosition = originalPosition;
EventId = eventId;
EventType = eventType;
IsJson = isJson;
Timestamp = timestamp;
//TODO: handle utf-8 conversion exception
Data = data != null ? Helper.UTF8NoBom.GetString(data): null;
Metadata = metadata != null ? Helper.UTF8NoBom.GetString(metadata): null;
PositionMetadata = positionMetadata != null ? Helper.UTF8NoBom.GetString(positionMetadata) : null;
StreamMetadata = streamMetadata != null ? Helper.UTF8NoBom.GetString(streamMetadata) : null;
}
public static CheckpointTag FromEventTypeIndexPositions(TFPos position, IDictionary <string, int> streams)
{
// streams cloned inside
return(new CheckpointTag(streams, position));
}
IndexReadAllResult IReadIndex.ReadAllEventsBackward(TFPos pos, int maxCount)
{
return _allReader.ReadAllEventsBackward(pos, maxCount);
}
/// <summary>
/// Returns event records in the sequence they were committed into TF.
/// Positions is specified as pre-positions (pointer at the beginning of the record).
/// </summary>
IndexReadAllResult IReadIndex.ReadAllEventsForward(TFPos pos, int maxCount)
{
var lastCommitPosition = Interlocked.Read(ref _lastCommitPosition);
var records = new List<CommitEventRecord>();
var nextPos = pos;
// in case we are at position after which there is no commit at all, in that case we have to force
// PreparePosition to int.MaxValue, so if you decide to read backwards from PrevPos,
// you will receive all prepares.
var prevPos = new TFPos(pos.CommitPosition, int.MaxValue);
var count = 0;
bool firstCommit = true;
ITransactionFileSequentialReader seqReader = GetSeqReader();
try
{
long nextCommitPos = pos.CommitPosition;
while (count < maxCount)
{
seqReader.Reposition(nextCommitPos);
SeqReadResult result;
do
{
result = seqReader.TryReadNext();
}
while (result.Success && result.LogRecord.RecordType != LogRecordType.Commit); // skip until commit
if (!result.Success) // no more records in TF
break;
nextCommitPos = result.RecordPostPosition;
var commit = (CommitLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
// for backward pass we want to allow read the same commit and skip read prepares,
// so we put post-position of commit and post-position of prepare as TFPos for backward pass
prevPos = new TFPos(result.RecordPostPosition, pos.PreparePosition);
}
seqReader.Reposition(commit.TransactionPosition);
while (count < maxCount)
{
result = seqReader.TryReadNext();
if (!result.Success) // no more records in TF
break;
// prepare with TransactionEnd could be scavenged already
// so we could reach the same commit record. In that case have to stop
if (result.LogRecord.Position >= commit.Position)
break;
if (result.LogRecord.RecordType != LogRecordType.Prepare)
continue;
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.TransactionPosition != commit.TransactionPosition) // wrong prepare
continue;
// prepare with useful data or delete tombstone
if ((prepare.Flags & (PrepareFlags.Data | PrepareFlags.StreamDelete)) != 0)
{
if (new TFPos(commit.Position, prepare.LogPosition) >= pos)
{
var eventRecord = new EventRecord(commit.FirstEventNumber + prepare.TransactionOffset, prepare);
records.Add(new CommitEventRecord(eventRecord, commit.Position));
count++;
// for forward pass position is inclusive,
// so we put pre-position of commit and post-position of prepare
nextPos = new TFPos(commit.LogPosition, result.RecordPostPosition);
}
}
if ((prepare.Flags & PrepareFlags.TransactionEnd) != 0)
break;
}
}
}
finally
{
ReturnSeqReader(seqReader);
}
return new IndexReadAllResult(records, maxCount, pos, nextPos, prevPos, lastCommitPosition);
}
private static List<List<PTable>> LoadPTables(StreamReader reader, string indexmapFilename, TFPos checkpoints)
{
var tables = new List<List<PTable>>();
// all next lines are PTables sorted by levels
string text;
while ((text = reader.ReadLine()) != null)
{
if (checkpoints.PreparePosition < 0 || checkpoints.CommitPosition < 0)
throw new CorruptIndexException(
string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.", checkpoints));
PTable ptable = null;
var pieces = text.Split(',');
try
{
var level = int.Parse(pieces[0]);
var position = int.Parse(pieces[1]);
var file = pieces[2];
var path = Path.GetDirectoryName(indexmapFilename);
var ptablePath = Path.Combine(path, file);
ptable = PTable.FromFile(ptablePath);
ptable.VerifyFileHash();
CreateIfNeeded(level, tables);
tables[level].Insert(position, ptable);
}
catch (Exception exc)
{
// if PTable file path was correct, but data is corrupted, we still need to dispose opened streams
if (ptable != null)
ptable.Dispose();
// also dispose all previously loaded correct PTables
for (int i=0; i<tables.Count; ++i)
{
for (int j=0; j<tables[i].Count; ++j)
{
tables[i][j].Dispose();
}
}
throw new CorruptIndexException("Error while loading IndexMap.", exc);
}
}
return tables;
}
public void position_returned_for_prev_page_when_traversing_backward_allow_to_traverse_forward_correctly()
{
var recs = new[] { _p5, _p3, _p1, _p4, _p2 }; // in reverse committed order
int count = 0;
var pos = new TFPos(Db.Config.WriterCheckpoint.Read(), Db.Config.WriterCheckpoint.Read());
ReadAllResult result;
while ((result = ReadIndex.ReadAllEventsBackward(pos, 1, false)).Records.Count != 0)
{
Assert.AreEqual(1, result.Records.Count);
Assert.AreEqual(recs[count], result.Records[0].Event);
var localPos = result.PrevPos;
int localCount = 0;
ReadAllResult localResult;
while ((localResult = ReadIndex.ReadAllEventsForward(localPos, 1, false)).Records.Count != 0)
{
Assert.AreEqual(1, localResult.Records.Count);
Assert.AreEqual(recs[count - 1 - localCount], localResult.Records[0].Event);
localPos = localResult.NextPos;
localCount += 1;
}
pos = result.NextPos;
count += 1;
}
Assert.AreEqual(recs.Length, count);
}
private CheckpointTag(TFPos position)
{
Position = position;
Mode_ = CalculateMode();
}
private bool BeforeTheLastKnownIndexCheckpoint(TFPos tfPosition)
{
return(_lastKnownIndexCheckpointPosition != null && tfPosition <= _lastKnownIndexCheckpointPosition);
}
protected TFPos GetBackwardReadPos()
{
var pos = new TFPos(WriterCheckpoint.ReadNonFlushed(), WriterCheckpoint.ReadNonFlushed());
return pos;
}
public ResolvedEvent(EventStore.Core.Data.ResolvedEvent resolvedEvent, byte[] streamMetadata)
{
var positionEvent = resolvedEvent.Link ?? resolvedEvent.Event;
_linkOrEventPosition = resolvedEvent.OriginalPosition.GetValueOrDefault();
var @event = resolvedEvent.Event;
_positionStreamId = positionEvent.EventStreamId;
_positionSequenceNumber = positionEvent.EventNumber;
_eventStreamId = @event != null ? @event.EventStreamId : null;
_eventSequenceNumber = @event != null ? @event.EventNumber : -1;
_resolvedLinkTo = positionEvent != @event;
_position = resolvedEvent.OriginalPosition ?? new TFPos(-1, positionEvent.LogPosition);
EventId = @event != null ? @event.EventId : Guid.Empty;
EventType = @event != null ? @event.EventType : null;
IsJson = @event != null && (@event.Flags & PrepareFlags.IsJson) != 0;
Timestamp = positionEvent.TimeStamp;
//TODO: handle utf-8 conversion exception
Data = @event != null && @event.Data != null?Helper.UTF8NoBom.GetString(@event.Data) : null;
Metadata = @event != null && @event.Metadata != null?Helper.UTF8NoBom.GetString(@event.Metadata) : null;
PositionMetadata = _resolvedLinkTo
? (positionEvent.Metadata != null ? Helper.UTF8NoBom.GetString(positionEvent.Metadata) : null)
: null;
StreamMetadata = streamMetadata != null?Helper.UTF8NoBom.GetString(streamMetadata) : null;
TFPos eventOrLinkTargetPosition;
if (_resolvedLinkTo)
{
Dictionary <string, JToken> extraMetadata = null;
if (positionEvent.Metadata != null && positionEvent.Metadata.Length > 0)
{
//TODO: parse JSON only when unresolved link and just tag otherwise
CheckpointTag tag;
if (resolvedEvent.Link != null && resolvedEvent.Event == null)
{
var checkpointTagJson =
positionEvent.Metadata.ParseCheckpointTagVersionExtraJson(default(ProjectionVersion));
tag = checkpointTagJson.Tag;
extraMetadata = checkpointTagJson.ExtraMetadata;
var parsedPosition = tag.Position;
eventOrLinkTargetPosition = parsedPosition != new TFPos(long.MinValue, long.MinValue)
? parsedPosition
: new TFPos(-1, positionEvent.LogPosition);
}
else
{
tag = positionEvent.Metadata.ParseCheckpointTagJson();
var parsedPosition = tag.Position;
if (parsedPosition == new TFPos(long.MinValue, long.MinValue) && @event.Metadata.IsValidJson())
{
tag = @event.Metadata.ParseCheckpointTagJson();
if (tag != null)
{
parsedPosition = tag.Position;
}
}
eventOrLinkTargetPosition = parsedPosition != new TFPos(long.MinValue, long.MinValue)
? parsedPosition
: new TFPos(-1, resolvedEvent.Event.LogPosition);
}
}
else
{
eventOrLinkTargetPosition = @event != null ? new TFPos(-1, @event.LogPosition) : new TFPos(-1, positionEvent.LogPosition);
}
JToken deletedValue;
IsLinkToDeletedStreamTombstone = extraMetadata != null &&
extraMetadata.TryGetValue("$deleted", out deletedValue);
if (resolvedEvent.ResolveResult == ReadEventResult.NoStream ||
resolvedEvent.ResolveResult == ReadEventResult.StreamDeleted || IsLinkToDeletedStreamTombstone)
{
IsLinkToDeletedStream = true;
var streamId = SystemEventTypes.StreamReferenceEventToStreamId(
SystemEventTypes.LinkTo, resolvedEvent.Link.Data);
_eventStreamId = streamId;
}
}
else
{
// not a link
eventOrLinkTargetPosition = resolvedEvent.OriginalPosition ?? new TFPos(-1, positionEvent.LogPosition);
}
_eventOrLinkTargetPosition = eventOrLinkTargetPosition;
}
public PendingEvent(EventStore.Core.Data.ResolvedEvent resolvedEvent, TFPos tfPosition, float progress)
{
ResolvedEvent = resolvedEvent;
Progress = progress;
TfPosition = tfPosition;
}
public AllSubscriptionFiltered(IPublisher bus,
Position?startPosition,
bool resolveLinks,
IEventFilter eventFilter,
ClaimsPrincipal user,
bool requiresLeader,
IReadIndex readIndex,
uint?maxSearchWindow,
uint checkpointIntervalMultiplier,
Func <Position, Task> checkpointReached,
CancellationToken cancellationToken)
{
if (bus == null)
{
throw new ArgumentNullException(nameof(bus));
}
if (eventFilter == null)
{
throw new ArgumentNullException(nameof(eventFilter));
}
if (readIndex == null)
{
throw new ArgumentNullException(nameof(readIndex));
}
if (checkpointReached == null)
{
throw new ArgumentNullException(nameof(checkpointReached));
}
if (checkpointIntervalMultiplier == 0)
{
throw new ArgumentOutOfRangeException(nameof(checkpointIntervalMultiplier));
}
_subscriptionId = Guid.NewGuid();
_bus = bus;
_resolveLinks = resolveLinks;
_eventFilter = eventFilter;
_user = user;
_requiresLeader = requiresLeader;
_maxSearchWindow = maxSearchWindow ?? ReadBatchSize;
_checkpointReached = checkpointReached;
_cancellationToken = cancellationToken;
_subscriptionStarted = new TaskCompletionSource <bool>();
_channel = Channel.CreateBounded <(ResolvedEvent?, Position?)>(BoundedChannelOptions);
_checkpointInterval = checkpointIntervalMultiplier * _maxSearchWindow;
_semaphore = new SemaphoreSlim(1, 1);
_lastCheckpoint = Position.Start;
SubscriptionId = _subscriptionId.ToString();
var lastIndexedPosition = readIndex.LastIndexedPosition;
var startPositionExclusive = startPosition == Position.End
? Position.FromInt64(lastIndexedPosition, lastIndexedPosition)
: startPosition ?? Position.Start;
_startPositionExclusive = new TFPos((long)startPositionExclusive.CommitPosition,
(long)startPositionExclusive.PreparePosition);
Subscribe(startPositionExclusive, startPosition != Position.End);
}
public IndexReadAllResult FilteredReadAllEventsForward(TFPos pos, int maxCount, int maxSearchWindow,
IEventFilter eventFilter)
{
return(ReadAllEventsForwardInternal(pos, maxCount, maxSearchWindow, eventFilter));
}
public IndexReadAllResult ReadAllEventsForward(TFPos pos, int maxCount)
{
return(ReadAllEventsForwardInternal(pos, maxCount, maxCount, EventFilter.None));
}
private void EmptyCache()
{
_lastMessages.Clear();
_subscribeFromPosition = new TFPos(long.MaxValue, long.MaxValue);
}
public void Handle(ClientMessage.ReadAllEventsForwardCompleted message)
{
if (_disposed)
{
return;
}
if (message.CorrelationId != _pendingRequestCorrelationId)
{
return;
}
if (message.Result == ReadAllResult.AccessDenied)
{
SendNotAuthorized();
return;
}
if (!_tfEventsRequested)
{
throw new InvalidOperationException("TF events has not been requested");
}
if (_reader.Paused)
{
throw new InvalidOperationException("Paused");
}
_reader._lastPosition = message.TfLastCommitPosition;
_tfEventsRequested = false;
switch (message.Result)
{
case ReadAllResult.Success:
var eof = message.Events.Length == 0;
_eof = eof;
var willDispose = _reader._stopOnEof && eof;
_fromTfPosition = message.NextPos;
if (!willDispose)
{
_reader.PauseOrContinueProcessing();
}
if (eof)
{
// the end
//TODO: is it safe to pass NEXT as last commit position here
DeliverLastCommitPosition(message.NextPos);
// allow joining heading distribution
SendIdle();
_reader.SendEof();
}
else
{
foreach (var @event in message.Events)
{
var link = @event.Link;
var data = @event.Event;
var byStream = link != null && _streamToEventType.ContainsKey(link.EventStreamId);
string adjustedPositionStreamId;
var isDeleteStreamEvent =
StreamDeletedHelper.IsStreamDeletedEvent(
@event.OriginalStreamId, @event.OriginalEvent.EventType,
@event.OriginalEvent.Data, out adjustedPositionStreamId);
if (data == null)
{
continue;
}
var eventType = isDeleteStreamEvent ? "$deleted" : data.EventType;
var byEvent = link == null && _eventTypes.Contains(eventType);
var originalTfPosition = @event.OriginalPosition.Value;
if (byStream)
{
// ignore data just update positions
_reader.UpdateNextStreamPosition(link.EventStreamId, link.EventNumber + 1);
// recover unresolved link event
var unresolvedLinkEvent =
EventStore.Core.Data.ResolvedEvent.ForUnresolvedEvent(link,
originalTfPosition.CommitPosition);
DeliverEventRetrievedFromTf(
unresolvedLinkEvent, 100.0f * link.LogPosition / message.TfLastCommitPosition,
originalTfPosition);
}
else if (byEvent)
{
DeliverEventRetrievedFromTf(
@event, 100.0f * data.LogPosition / message.TfLastCommitPosition,
originalTfPosition);
}
}
}
if (_disposed)
{
return;
}
break;
default:
throw new NotSupportedException(
String.Format("ReadEvents result code was not recognized. Code: {0}", message.Result));
}
}
protected override void Given()
{
base.Given();
_tfPos1 = ExistingEvent("test-stream", "type1", "{}", "{Data: 1}");
_tfPos2 = ExistingEvent("test-stream", "type1", "{}", "{Data: 2}");
_tfPos3 = ExistingEvent("test-stream", "type2", "{}", "{Data: 3}");
// writes reordered due to batching or timeouts in system projection
ExistingEvent("$et-type1", "$>", TFPosToMetadata(_tfPos1), "0@test-stream");
ExistingEvent("$et-type2", "$>", TFPosToMetadata(_tfPos3), "2@test-stream");
ExistingEvent("$et-type1", "$>", TFPosToMetadata(_tfPos2), "1@test-stream");
NoStream("$et");
_querySourcesDefinition = new QuerySourcesDefinition
{
AllStreams = true,
Events = new[] {"type1", "type2"},
Options = new QuerySourcesDefinitionOptions {}
};
_fromPosition = CheckpointTag.FromEventTypeIndexPositions(
new TFPos(0, -1), new Dictionary<string, int> {{"type1", -1}, {"type2", -1}});
_maxEvents = 3;
}
private ClientMessage.FilteredReadAllEventsForwardCompleted NoDataForFilteredCommand(
ClientMessage.FilteredReadAllEventsForward msg, FilteredReadAllResult result, TFPos pos,
long lastIndexedPosition, string error = null)
{
return(new ClientMessage.FilteredReadAllEventsForwardCompleted(
msg.CorrelationId, result, error, ResolvedEvent.EmptyArray, null, false,
msg.MaxCount, pos, TFPos.Invalid, TFPos.Invalid, lastIndexedPosition, false));
}
protected Item(TFPos position)
{
Position = position;
}
protected TFPos GetBackwardReadPos()
{
var pos = new TFPos(WriterCheckpoint.ReadNonFlushed(), WriterCheckpoint.ReadNonFlushed());
return(pos);
}
private IndexReadAllResult ReadAllEventsBackwardInternal(TFPos pos, int maxCount, int maxSearchWindow,
IEventFilter eventFilter)
{
var records = new List <CommitEventRecord>();
var nextPos = pos;
// in case we are at position after which there is no commit at all, in that case we have to force
// PreparePosition to 0, so if you decide to read backwards from PrevPos,
// you will receive all prepares.
var prevPos = new TFPos(pos.CommitPosition, 0);
var consideredEventsCount = 0L;
bool firstCommit = true;
var reachedEndOfStream = false;
using (var reader = _backend.BorrowReader()) {
long nextCommitPostPos = pos.CommitPosition;
while (records.Count < maxCount && consideredEventsCount < maxSearchWindow)
{
reader.Reposition(nextCommitPostPos);
SeqReadResult result;
while ((result = reader.TryReadPrev()).Success && !IsCommitAlike(result.LogRecord))
{
// skip until commit
}
if (!result.Success)
{
// no more records in TF
reachedEndOfStream = true;
break;
}
nextCommitPostPos = result.RecordPrePosition;
if (!IsReplicated(nextCommitPostPos))
{
continue;
}
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare: {
var prepare = (PrepareLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
prevPos = new TFPos(result.RecordPostPosition, result.RecordPostPosition);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete) &&
new TFPos(result.RecordPostPosition, result.RecordPostPosition) <= pos)
{
var eventRecord = new EventRecord(prepare.ExpectedVersion + 1 /* EventNumber */,
prepare);
consideredEventsCount++;
if (eventFilter.IsEventAllowed(eventRecord))
{
records.Add(new CommitEventRecord(eventRecord, prepare.LogPosition));
}
// for backward pass we allow read the same commit, but force to skip last read prepare
// so we put post-position of commit and pre-position of prepare
nextPos = new TFPos(result.RecordPrePosition, result.RecordPrePosition);
}
break;
}
case LogRecordType.Commit: {
var commit = (CommitLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
// for forward pass we allow read the same commit and as we have post-positions here
// we can put just prepare post-position as prepare pre-position for forward read
// so we put pre-position of commit and post-position of prepare
prevPos = new TFPos(commit.LogPosition, pos.PreparePosition);
}
var commitPostPos = result.RecordPostPosition;
// as we don't know exact position of the last record of transaction,
// we have to sequentially scan backwards, so no need to reposition
while (consideredEventsCount < maxCount)
{
result = reader.TryReadPrev();
if (!result.Success) // no more records in TF
{
break;
}
// prepare with TransactionBegin could be scavenged already
// so we could reach beyond the start of transaction. In that case we have to stop.
if (result.LogRecord.LogPosition < commit.TransactionPosition)
{
break;
}
if (result.LogRecord.RecordType != LogRecordType.Prepare)
{
continue;
}
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.TransactionPosition != commit.TransactionPosition) // wrong prepare
{
continue;
}
// prepare with useful data or delete tombstone
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete) &&
new TFPos(commitPostPos, result.RecordPostPosition) <= pos)
{
var eventRecord =
new EventRecord(commit.FirstEventNumber + prepare.TransactionOffset, prepare);
consideredEventsCount++;
if (eventFilter.IsEventAllowed(eventRecord))
{
records.Add(new CommitEventRecord(eventRecord, commit.LogPosition));
}
// for backward pass we allow read the same commit, but force to skip last read prepare
// so we put post-position of commit and pre-position of prepare
nextPos = new TFPos(commitPostPos, prepare.LogPosition);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionBegin))
{
break;
}
}
break;
}
default:
throw new Exception(string.Format("Unexpected log record type: {0}.",
result.LogRecord.RecordType));
}
}
return(new IndexReadAllResult(records, pos, nextPos, prevPos, reachedEndOfStream));
}
}
public ReadAllResult(ResolvedEventRecord[] records, int maxCount, TFPos currentPos, TFPos nextPos, TFPos prevPos, long tfEofPosition)
{
Ensure.NotNull(records, "records");
Records = records;
MaxCount = maxCount;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
TfEofPosition = tfEofPosition;
}
private CheckpointTag(long preparePosition)
{
Position = new TFPos(long.MinValue, preparePosition);
Mode_ = CalculateMode();
}
public IndexReadAllResult(List <CommitEventRecord> records, TFPos currentPos, TFPos nextPos, TFPos prevPos,
bool isEndOfStream, long consideredEventsCount)
{
Ensure.NotNull(records, "records");
Records = records;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
IsEndOfStream = isEndOfStream;
ConsideredEventsCount = consideredEventsCount;
}
/// <summary>
/// Returns event records in the reverse sequence they were committed into TF.
/// Positions is specified as post-positions (pointer after the end of record).
/// </summary>
IndexReadAllResult IReadIndex.ReadAllEventsBackward(TFPos pos, int maxCount)
{
var lastCommitPosition = Interlocked.Read(ref _lastCommitPosition);
var records = new List<CommitEventRecord>();
var nextPos = pos;
// in case we are at position after which there is no commit at all, in that case we have to force
// PreparePosition to 0, so if you decide to read backwards from PrevPos,
// you will receive all prepares.
var prevPos = new TFPos(pos.CommitPosition, 0);
var count = 0;
bool firstCommit = true;
ITransactionFileSequentialReader seqReader = GetSeqReader();
try
{
long nextCommitPostPos = pos.CommitPosition;
while (count < maxCount)
{
seqReader.Reposition(nextCommitPostPos);
SeqReadResult result;
do
{
result = seqReader.TryReadPrev();
}
while (result.Success && result.LogRecord.RecordType != LogRecordType.Commit); // skip until commit
if (!result.Success) // no more records in TF
break;
var commitPostPos = result.RecordPostPosition;
nextCommitPostPos = result.RecordPrePosition;
var commit = (CommitLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
// for forward pass we allow read the same commit and as we have post-positions here
// we can put just prepare post-position as prepare pre-position for forward read
// so we put pre-position of commit and post-position of prepare
prevPos = new TFPos(commit.LogPosition, pos.PreparePosition);
}
// as we don't know exact position of the last record of transaction,
// we have to sequentially scan backwards, so no need to reposition
//seqReader.Reposition(commitLogRecord.TransactionPosition);
while (count < maxCount)
{
result = seqReader.TryReadPrev();
if (!result.Success) // no more records in TF
break;
// prepare with TransactionBegin could be scavenged already
// so we could reach beyond the start of transaction. In that case we have to stop.
if (result.LogRecord.Position < commit.TransactionPosition)
break;
if (result.LogRecord.RecordType != LogRecordType.Prepare)
continue;
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.TransactionPosition != commit.TransactionPosition) // wrong prepare
continue;
// prepare with useful data or delete tombstone
if ((prepare.Flags & (PrepareFlags.Data | PrepareFlags.StreamDelete)) != 0)
{
if (new TFPos(commitPostPos, result.RecordPostPosition) <= pos)
{
var eventRecord = new EventRecord(commit.FirstEventNumber + prepare.TransactionOffset, prepare);
records.Add(new CommitEventRecord(eventRecord, commit.Position));
count++;
// for backward pass we allow read the same commit, but force to skip last read prepare
// so we put post-position of commit and pre-position of prepare
nextPos = new TFPos(commitPostPos, prepare.LogPosition);
}
}
if ((prepare.Flags & PrepareFlags.TransactionBegin) != 0)
break;
}
}
}
finally
{
ReturnSeqReader(seqReader);
}
return new IndexReadAllResult(records, maxCount, pos, nextPos, prevPos, lastCommitPosition);
}
private ClientMessage.ReadAllEventsBackwardCompleted NoData(ClientMessage.ReadAllEventsBackward msg, ReadAllResult result, TFPos pos, long lastCommitPosition, string error = null)
{
return(new ClientMessage.ReadAllEventsBackwardCompleted(
msg.CorrelationId, result, error, ResolvedEvent.EmptyArray, null, false,
msg.MaxCount, pos, TFPos.Invalid, TFPos.Invalid, lastCommitPosition));
}
internal CheckpointTag(TFPos position, Dictionary <string, int> streams)
{
Position = position;
Streams = streams;
Mode_ = CalculateMode();
}
public IndexReadAllResult ReadAllEventsForward(TFPos pos, int maxCount)
{
throw new System.NotImplementedException();
}
protected static ResolvedEvent CreateSampleEvent(
string streamId, int sequenceNumber, string eventType, string data, TFPos tfPos)
{
return new ResolvedEvent(
streamId, sequenceNumber, streamId, sequenceNumber, true, tfPos, Guid.NewGuid(), eventType, true, data,
"{}", "{\"position_meta\":1}");
}
IndexReadAllResult IReadIndex.ReadAllEventsBackwardFiltered(TFPos pos, int maxCount, int maxSearchWindow,
IEventFilter eventFilter)
{
return(_allReader.FilteredReadAllEventsBackward(pos, maxCount, maxSearchWindow, eventFilter));
}
public static CheckpointTag FromPosition(TFPos position)
{
return(new CheckpointTag(position));
}
IndexReadAllResult IReadIndex.ReadAllEventsBackward(TFPos pos, int maxCount)
{
return(_allReader.ReadAllEventsBackward(pos, maxCount));
}
private string TFPosToMetadata(TFPos tfPos)
{
return string.Format(@"{{""$c"":{0},""$p"":{1}}}", tfPos.CommitPosition, tfPos.PreparePosition);
}
public IndexReadAllResult ReadAllEventsForward(TFPos pos, int maxCount)
{
var records = new List <CommitEventRecord>();
var nextPos = pos;
// in case we are at position after which there is no commit at all, in that case we have to force
// PreparePosition to long.MaxValue, so if you decide to read backwards from PrevPos,
// you will receive all prepares.
var prevPos = new TFPos(pos.CommitPosition, long.MaxValue);
long count = 0;
bool firstCommit = true;
using (var reader = _backend.BorrowReader())
{
long nextCommitPos = pos.CommitPosition;
while (count < maxCount)
{
reader.Reposition(nextCommitPos);
SeqReadResult result;
while ((result = reader.TryReadNext()).Success && !IsCommitAlike(result.LogRecord))
{
// skip until commit
}
if (!result.Success) // no more records in TF
{
break;
}
nextCommitPos = result.RecordPostPosition;
switch (result.LogRecord.RecordType)
{
case LogRecordType.Prepare:
{
var prepare = (PrepareLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
prevPos = new TFPos(result.RecordPrePosition, result.RecordPrePosition);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete) &&
new TFPos(prepare.LogPosition, prepare.LogPosition) >= pos)
{
var eventRecord = new EventRecord(prepare.ExpectedVersion + 1 /* EventNumber */, prepare);
records.Add(new CommitEventRecord(eventRecord, prepare.LogPosition));
count++;
nextPos = new TFPos(result.RecordPostPosition, result.RecordPostPosition);
}
break;
}
case LogRecordType.Commit:
{
var commit = (CommitLogRecord)result.LogRecord;
if (firstCommit)
{
firstCommit = false;
// for backward pass we want to allow read the same commit and skip read prepares,
// so we put post-position of commit and post-position of prepare as TFPos for backward pass
prevPos = new TFPos(result.RecordPostPosition, pos.PreparePosition);
}
reader.Reposition(commit.TransactionPosition);
while (count < maxCount)
{
result = reader.TryReadNext();
if (!result.Success) // no more records in TF
{
break;
}
// prepare with TransactionEnd could be scavenged already
// so we could reach the same commit record. In that case have to stop
if (result.LogRecord.LogPosition >= commit.LogPosition)
{
break;
}
if (result.LogRecord.RecordType != LogRecordType.Prepare)
{
continue;
}
var prepare = (PrepareLogRecord)result.LogRecord;
if (prepare.TransactionPosition != commit.TransactionPosition) // wrong prepare
{
continue;
}
// prepare with useful data or delete tombstone
if (prepare.Flags.HasAnyOf(PrepareFlags.Data | PrepareFlags.StreamDelete) &&
new TFPos(commit.LogPosition, prepare.LogPosition) >= pos)
{
var eventRecord = new EventRecord(commit.FirstEventNumber + prepare.TransactionOffset, prepare);
records.Add(new CommitEventRecord(eventRecord, commit.LogPosition));
count++;
// for forward pass position is inclusive,
// so we put pre-position of commit and post-position of prepare
nextPos = new TFPos(commit.LogPosition, result.RecordPostPosition);
}
if (prepare.Flags.HasAnyOf(PrepareFlags.TransactionEnd))
{
break;
}
}
break;
}
default:
throw new Exception(string.Format("Unexpected log record type: {0}.", result.LogRecord.RecordType));
}
}
return(new IndexReadAllResult(records, pos, nextPos, prevPos));
}
}
public ReadAllEventsBackwardCompleted(Guid correlationId, ReadAllResult result, string error, ResolvedEvent[] events,
StreamMetadata streamMetadata, bool isCachePublic, int maxCount,
TFPos currentPos, TFPos nextPos, TFPos prevPos, long tfLastCommitPosition)
{
Ensure.NotNull(events, "events");
CorrelationId = correlationId;
Result = result;
Error = error;
Events = events;
StreamMetadata = streamMetadata;
IsCachePublic = isCachePublic;
MaxCount = maxCount;
CurrentPos = currentPos;
NextPos = nextPos;
PrevPos = prevPos;
TfLastCommitPosition = tfLastCommitPosition;
}
private string TFPosToMetadata(TFPos tfPos)
{
return(string.Format(@"{{""$c"":{0},""$p"":{1}}}", tfPos.CommitPosition, tfPos.PreparePosition));
}
public void read_all_events_backward_returns_correct_events_starting_in_the_middle_of_tf()
{
var pos = new TFPos(_pos6, _p4.LogPosition); // p3 post-pos
var res1 = ReadIndex.ReadAllEventsBackward(pos, 10, false);
Assert.AreEqual(4, res1.Records.Count);
Assert.AreEqual(_p3, res1.Records[0].Event);
Assert.AreEqual(_p1, res1.Records[1].Event);
Assert.AreEqual(_p4, res1.Records[2].Event);
Assert.AreEqual(_p2, res1.Records[3].Event);
var res2 = ReadIndex.ReadAllEventsForward(res1.PrevPos, 10, false);
Assert.AreEqual(1, res2.Records.Count);
Assert.AreEqual(_p5, res2.Records[0].Event);
}
public IndexReadAllResult ReadAllEventsBackward(TFPos pos, int maxCount)
{
throw new NotImplementedException();
}
public IndexReadAllResult ReadAllEventsBackwardFiltered(TFPos pos, int maxCount, int maxSearchWindow,
IEventFilter eventFilter)
{
throw new NotImplementedException();
}
public IndexReadAllResult ReadAllEventsBackward(TFPos pos, int maxCount)
{
return(ReadAllEventsBackwardInternal(pos, maxCount, maxCount, EventFilter.DefaultAllFilter));
}
private void GoLive(Position startPosition)
{
var liveEvents = Channel.CreateBounded <ResolvedEvent>(BoundedChannelOptions);
var caughtUpSource = new TaskCompletionSource <TFPos>();
var liveMessagesCancelled = 0;
Log.Information(
"Live subscription {subscriptionId} to $all running from {position}...",
_subscriptionId, startPosition);
_bus.Publish(new ClientMessage.SubscribeToStream(Guid.NewGuid(), _subscriptionId,
new ContinuationEnvelope(OnSubscriptionMessage, _semaphore, _cancellationToken), _subscriptionId,
string.Empty, _resolveLinks, _user));
Task.Factory.StartNew(PumpLiveMessages, _cancellationToken);
async Task PumpLiveMessages()
{
await caughtUpSource.Task.ConfigureAwait(false);
await foreach (var @event in liveEvents.Reader.ReadAllAsync(_cancellationToken)
.ConfigureAwait(false))
{
await _channel.Writer.WriteAsync(@event, _cancellationToken).ConfigureAwait(false);
}
}
async Task OnSubscriptionMessage(Message message, CancellationToken cancellationToken)
{
if (message is ClientMessage.NotHandled notHandled &&
RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
{
Fail(ex);
return;
}
switch (message)
{
case ClientMessage.SubscriptionConfirmation confirmed:
ConfirmSubscription();
var caughtUp = new TFPos(confirmed.LastIndexedPosition,
confirmed.LastIndexedPosition);
Log.Verbose(
"Live subscription {subscriptionId} to $all confirmed at {position}.",
_subscriptionId, caughtUp);
ReadHistoricalEvents(startPosition);
async Task OnHistoricalEventsMessage(Message message, CancellationToken ct)
{
if (message is ClientMessage.NotHandled notHandled &&
RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
{
Fail(ex);
return;
}
if (!(message is ClientMessage.ReadAllEventsForwardCompleted completed))
{
Fail(
RpcExceptions.UnknownMessage <ClientMessage.ReadAllEventsForwardCompleted>(
message));
return;
}
switch (completed.Result)
{
case ReadAllResult.Success:
if (completed.Events.Length == 0 && completed.IsEndOfStream)
{
NotifyCaughtUp(completed.CurrentPos);
return;
}
foreach (var @event in completed.Events)
{
var position = @event.OriginalPosition.Value;
if (position > caughtUp)
{
NotifyCaughtUp(position);
return;
}
await _channel.Writer.WriteAsync(@event, _cancellationToken)
.ConfigureAwait(false);
}
ReadHistoricalEvents(Position.FromInt64(
completed.NextPos.CommitPosition,
completed.NextPos.PreparePosition));
return;
void NotifyCaughtUp(TFPos position)
{
Log.Verbose(
"Live subscription {subscriptionId} to $all caught up at {position} because the end of stream was reached.",
_subscriptionId, position);
caughtUpSource.TrySetResult(caughtUp);
}
case ReadAllResult.AccessDenied:
Fail(RpcExceptions.AccessDenied());
return;
default:
Fail(RpcExceptions.UnknownError(completed.Result));
return;
}
}
void ReadHistoricalEvents(Position fromPosition)
{
if (fromPosition == Position.End)
{
throw new ArgumentOutOfRangeException(nameof(fromPosition));
}
Log.Verbose(
"Live subscription {subscriptionId} to $all loading any missed events starting from {position}.",
_subscriptionId, fromPosition);
ReadPage(fromPosition, OnHistoricalEventsMessage);
}
return;
case ClientMessage.SubscriptionDropped dropped:
switch (dropped.Reason)
{
case SubscriptionDropReason.AccessDenied:
Fail(RpcExceptions.AccessDenied());
return;
case SubscriptionDropReason.Unsubscribed:
return;
default:
Fail(RpcExceptions.UnknownError(dropped.Reason));
return;
}
case ClientMessage.StreamEventAppeared appeared: {
if (liveMessagesCancelled == 1)
{
return;
}
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1));
try {
Log.Verbose(
"Live subscription {subscriptionId} to $all enqueuing live message {position}.",
_subscriptionId, appeared.Event.OriginalPosition);
await liveEvents.Writer.WriteAsync(appeared.Event, cts.Token)
.ConfigureAwait(false);
} catch (Exception e) {
if (Interlocked.Exchange(ref liveMessagesCancelled, 1) != 0)
{
return;
}
Log.Verbose(
e,
"Live subscription {subscriptionId} to $all timed out at {position}; unsubscribing...",
_subscriptionId, appeared.Event.OriginalPosition.GetValueOrDefault());
Unsubscribe();
liveEvents.Writer.Complete();
var originalPosition =
_current.GetValueOrDefault().OriginalPosition.GetValueOrDefault();
CatchUp(Position.FromInt64(
originalPosition.CommitPosition,
originalPosition.PreparePosition));
}
return;
}
default:
Fail(
RpcExceptions.UnknownMessage <ClientMessage.SubscriptionConfirmation>(message));
return;
}
}
void Fail(Exception exception)
{
this.Fail(exception);
caughtUpSource.TrySetException(exception);
}
}
private static List <List <PTable> > LoadPTables(StreamReader reader, string indexmapFilename, TFPos checkpoints,
int cacheDepth, bool skipIndexVerify,
int threads,
int pTableMaxReaderCount)
{
var tables = new List <List <PTable> >();
try {
try {
Parallel.ForEach(
GetAllLines(reader)
.Reverse(), // Reverse so we load the highest levels (biggest files) first - ensures we use concurrency in the most efficient way.
new ParallelOptions {
MaxDegreeOfParallelism = threads
},
indexMapEntry => {
if (checkpoints.PreparePosition < 0 || checkpoints.CommitPosition < 0)
{
throw new CorruptIndexException(
string.Format("Negative prepare/commit checkpoint in non-empty IndexMap: {0}.",
checkpoints));
}
PTable ptable = null;
var pieces = indexMapEntry.Split(',');
try {
var level = int.Parse(pieces[0]);
var position = int.Parse(pieces[1]);
var file = pieces[2];
var path = Path.GetDirectoryName(indexmapFilename);
var ptablePath = Path.Combine(path, file);
ptable = PTable.FromFile(ptablePath, ESConsts.PTableInitialReaderCount, pTableMaxReaderCount, cacheDepth, skipIndexVerify);
lock (tables) {
InsertTableToTables(tables, level, position, ptable);
}
} catch (Exception) {
// if PTable file path was correct, but data is corrupted, we still need to dispose opened streams
if (ptable != null)
{
ptable.Dispose();
}
throw;
}
});
// Verify map is correct
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
if (tables[i][j] == null)
{
throw new CorruptIndexException(
$"indexmap is missing contiguous level,position {i},{j}");
}
}
}
} catch (AggregateException aggEx) {
// We only care that *something* has gone wrong, throw the first exception
throw aggEx.InnerException;
}
} catch (Exception exc) {
// also dispose all previously loaded correct PTables
for (int i = 0; i < tables.Count; ++i)
{
for (int j = 0; j < tables[i].Count; ++j)
{
if (tables[i][j] != null)
{
tables[i][j].Dispose();
}
}
}
throw new CorruptIndexException("Error while loading IndexMap.", exc);
}
return(tables);
}
