public static StreamOptions FromStreamNameAndRevision(
string streamName,
StreamPosition streamRevision)
{
if (streamName == null)
{
throw new ArgumentNullException(nameof(streamName));
}
if (streamRevision == StreamPosition.End)
{
return(new StreamOptions {
StreamName = streamName,
End = new Empty()
});
}
if (streamRevision == StreamPosition.Start)
{
return(new StreamOptions {
StreamName = streamName,
Start = new Empty()
});
}
return(new StreamOptions {
StreamName = streamName,
Revision = streamRevision
});
}
public void StreamReadAsync_1()
{
var streamPositions = new StreamPosition[] { };
wrapper.StreamReadAsync(streamPositions, null, CommandFlags.None);
mock.Verify(_ => _.StreamReadAsync(streamPositions, null, CommandFlags.None));
}
public async Task <IReadOnlyCollection <ResolvedEvent> > GetEventStream(string stream, string entityId, string model = null)
{
var response = new List <ResolvedEvent>();
var snapshots = await GetSnapshotStream(stream, entityId, model);
var position = StreamPosition.Start;
if (snapshots.Count > 0)
{
Console.WriteLine($"Using snapshot for model {model}");
var meta = JsonConvert.DeserializeObject <SnapshotMetaData>(Encoding.UTF8.GetString(snapshots.Last().Event.Metadata.Span));
position = new StreamPosition(meta.Position + 1);
response.Add(snapshots.Last());
}
var state = _client.ReadStreamAsync(
Direction.Forwards,
$"{stream}-{entityId}",
position);
var readState = await state.ReadState;
if (readState == ReadState.StreamNotFound)
{
return(new List <ResolvedEvent>());
}
response.AddRange(await state.ToListAsync());
return(response);
}
public void StreamStreamReadGroupAsync_2()
{
var streamPositions = new StreamPosition[] { };
wrapper.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, false, CommandFlags.None);
mock.Verify(_ => _.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, false, CommandFlags.None));
}
public async Task <StreamReadResults> ReadFrom(string streamId, Func <IAmAStoredEvent, bool> predicate = null, Direction direction = Direction.Backwards, CancellationToken cancellationToken = default)
{
var readResult = client.ReadStreamAsync(direction,
streamId,
direction == Direction.Backwards ? StreamPosition.End : StreamPosition.Start,
resolveLinkTos: true,
configureOperationOptions: options => options.TimeoutAfter = TimeSpan.FromSeconds(30),
cancellationToken: cancellationToken);
bool streamExists = false;
try
{
var readState = await readResult.ReadState;
streamExists = readState == ReadState.Ok;
}
#pragma warning disable 168
catch (StreamDeletedException ex)
// This happens when the stream is hard-deleted. We don't want to throw in that case
#pragma warning restore 168
{
streamExists = false;
}
if (!streamExists)
{
return(new StreamReadResults(emptyReadResult, false, StreamPosition.FromInt64(-1)));
}
predicate ??= _ => true;
var lastIndex = (await client.ReadStreamAsync(Direction.Backwards, streamId,
revision: StreamPosition.End,
maxCount: 1,
resolveLinkTos: false).FirstAsync(cancellationToken)).OriginalEventNumber;
IAsyncEnumerable <StoredEvent> storedEvents;
if (direction == Direction.Backwards)
{
storedEvents = readResult
// Trust me, resharper is wrong in this one. Event can be null
// ReSharper disable once ConditionIsAlwaysTrueOrFalse
.Where(e => e.Event != null)
.Select((e, _) => e.Event.ToStoredEvent(stateFactory))
.TakeWhile(e => e.DeserializedEvent is not EntitySoftDeleted)
.Where(e => predicate(e));
}
else
{
storedEvents = readResult
// Trust me, resharper is wrong in this one. Event can be null
// ReSharper disable once ConditionIsAlwaysTrueOrFalse
.Where(e => e.Event != null)
.Select((e, c) => e.Event.ToStoredEvent(stateFactory))
.Where(e => predicate(e));
}
return(new StreamReadResults(storedEvents, true, lastIndex));
}
// Placed object message payload into a segment.
private ArraySegment <byte> EncodeMessageIntoSegment(StreamPosition streamPosition, EventData queueMessage)
{
byte[] propertiesBytes = queueMessage.SerializeProperties();
byte[] payload = queueMessage.GetBytes();
// get size of namespace, offset, partitionkey, properties, and payload
int size = SegmentBuilder.CalculateAppendSize(streamPosition.StreamIdentity.Namespace) +
SegmentBuilder.CalculateAppendSize(queueMessage.Offset) +
SegmentBuilder.CalculateAppendSize(queueMessage.PartitionKey) +
SegmentBuilder.CalculateAppendSize(propertiesBytes) +
SegmentBuilder.CalculateAppendSize(payload);
// get segment
ArraySegment <byte> segment = GetSegment(size);
// encode namespace, offset, partitionkey, properties and payload into segment
int writeOffset = 0;
SegmentBuilder.Append(segment, ref writeOffset, streamPosition.StreamIdentity.Namespace);
SegmentBuilder.Append(segment, ref writeOffset, queueMessage.Offset);
SegmentBuilder.Append(segment, ref writeOffset, queueMessage.PartitionKey);
SegmentBuilder.Append(segment, ref writeOffset, propertiesBytes);
SegmentBuilder.Append(segment, ref writeOffset, payload);
return(segment);
}
public async Task ProcessStreamItem(StreamPosition position)
{
StreamPosition currentPosition = null;
bool found = _positions.TryGetValue(position.Id, out currentPosition);
switch (position.Operation)
{
case StreamOperation.Insert:
case StreamOperation.Update:
if (!found || currentPosition.Compare(position) != 0)
{
if (_persistenceLayer != null)
{
await _persistenceLayer.ProcessStreamItem(position);
}
}
_positions[position.Id] = position;
break;
case StreamOperation.Delete:
if (found)
{
if (_persistenceLayer != null)
{
await _persistenceLayer.ProcessStreamItem(position);
}
_positions.TryRemove(position.Id, out currentPosition);
}
break;
default:
break;
}
}
Task <(StreamProcessorState, FailingPartitionState)> ChangePositionInFailingPartition(
IStreamProcessorId streamProcessorId,
StreamProcessorState oldState,
PartitionId partitionId,
StreamPosition newPosition,
DateTimeOffset lastFailed,
CancellationToken cancellationToken) =>
public IEnumerable<string> ReadAllLines(Stream stream, Encoding encoding)
{
using (var sr = new StreamReader(stream, encoding))
{
if (stream.SetStreamPosition(_position))
{
if (!IsEndOfLine(_position.CheckBuffer))
{
ResetStream(_items, stream);
}
}
else
{
_items.Clear();
}
while (!sr.EndOfStream)
{
_items.Add(sr.ReadLine());
}
_position = stream.GetStreamPosition(128);
}
return _items;
}
StreamPosition iTrackReader.findKeyFrame(StreamPosition seekFrame)
{
var sp = findKeyFrame((MkvSeekPosition)seekFrame);
Logger.logDebug("findKeyFrame: {0} -> {1}", seekFrame, sp);
return(sp);
}
public async IAsyncEnumerable <ResolvedEvent> ReadStreamAsync(
IStream stream,
Direction direction,
StreamPosition revision,
long maxCount = long.MaxValue,
Action <EventStoreClientOperationOptions>?configureOperationOptions = null,
bool resolveLinkTos = false,
UserCredentials?userCredentials = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var results = this._eventStoreClient.ReadStreamAsync(
direction,
stream.Name,
revision,
maxCount,
configureOperationOptions,
resolveLinkTos,
userCredentials,
cancellationToken);
var state = await results.ReadState;
if (state != ReadState.StreamNotFound)
{
// todo, i've killed a kitten here, i'm sorry, but i wanted to avoid reflexion, seriously though, why the internal constructor on ReadStreamResults..
await foreach (var @event in results)
{
yield return(@event);
}
}
}
/// <inheritdoc/>
public async Task Write <TEvent>(
IMongoCollection <TEvent> stream,
FilterDefinitionBuilder <TEvent> filter,
Func <StreamPosition, TEvent> createStoreEvent,
CancellationToken cancellationToken)
where TEvent : class
{
StreamPosition streamPosition = null;
try
{
using var session = await _streams.StartSessionAsync().ConfigureAwait(false);
await session.WithTransactionAsync(
async (transaction, cancellationToken) =>
{
streamPosition = (ulong)await stream.CountDocumentsAsync(
transaction,
filter.Empty).ConfigureAwait(false);
await stream.InsertOneAsync(
transaction,
createStoreEvent(streamPosition),
cancellationToken: cancellationToken).ConfigureAwait(false);
return(Task.CompletedTask);
},
cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (MongoWaitQueueFullException ex)
{
throw new EventStoreUnavailable("Mongo wait queue is full", ex);
}
}
public async Task <IList <IBatchContainer> > GetQueueMessagesAsync(int maxCount)
{
if (receiver == null || maxCount <= 0)
{
return(new List <IBatchContainer>());
}
List <EventData> messages = (await receiver.ReceiveAsync(maxCount, ReceiveTimeout)).ToList();
var batches = new List <IBatchContainer>();
if (messages.Count == 0)
{
return(batches);
}
DateTime dequeueTimeUtc = DateTime.UtcNow;
foreach (EventData message in messages)
{
StreamPosition streamPosition = cache.Add(message, dequeueTimeUtc);
batches.Add(new StreamActivityNotificationBatch(streamPosition.StreamIdentity.Guid,
streamPosition.StreamIdentity.Namespace, streamPosition.SequenceToken));
}
if (!checkpointer.CheckpointExists)
{
checkpointer.Update(messages[0].Offset, DateTime.UtcNow);
}
return(batches);
}
public async Task <IList <IBatchContainer> > GetQueueMessagesAsync(int maxCount)
{
if (receiver == null || maxCount <= 0)
{
return(new List <IBatchContainer>());
}
List <EventData> messages;
try
{
var watch = Stopwatch.StartNew();
messages = (await receiver.ReceiveAsync(maxCount, ReceiveTimeout)).ToList();
watch.Stop();
logger.TrackMetric(hubReceiveTimeMetric, watch.Elapsed);
logger.TrackMetric(partitionReceiveTimeMetric, watch.Elapsed);
logger.TrackMetric(hubReadFailure, 0);
logger.TrackMetric(partitionReadFailure, 0);
}
catch (Exception ex)
{
logger.TrackMetric(hubReadFailure, 1);
logger.TrackMetric(partitionReadFailure, 1);
logger.Warn(OrleansServiceBusErrorCode.FailedPartitionRead, "Failed to read from EventHub partition {0}-{1}. : Exception: {2}.", config.Hub.Path,
config.Partition, ex);
throw;
}
var batches = new List <IBatchContainer>();
if (messages.Count == 0)
{
return(batches);
}
logger.TrackMetric(hubMessagesRecieved, messages.Count);
logger.TrackMetric(partitionMessagesReceived, messages.Count);
// monitor message age
var dequeueTimeUtc = DateTime.UtcNow;
TimeSpan difference = dequeueTimeUtc - messages[messages.Count - 1].EnqueuedTimeUtc;
logger.TrackMetric(hubAgeOfMessagesBeingProcessed, difference);
logger.TrackMetric(partitionAgeOfMessagesBeingProcessed, difference);
foreach (EventData message in messages)
{
StreamPosition streamPosition = cache.Add(message, dequeueTimeUtc);
batches.Add(new StreamActivityNotificationBatch(streamPosition.StreamIdentity.Guid,
streamPosition.StreamIdentity.Namespace, streamPosition.SequenceToken));
}
if (!checkpointer.CheckpointExists)
{
checkpointer.Update(messages[0].Offset, DateTime.UtcNow);
}
return(batches);
}
async Task WriteEventsToEventHorizon(
IReverseCallDispatcher <EventHorizonConsumerToProducerMessage, EventHorizonProducerToConsumerMessage, ConsumerSubscriptionRequest, Contracts.SubscriptionResponse, ConsumerRequest, ConsumerResponse> dispatcher,
TenantId producerTenant,
StreamId publicStream,
PartitionId partition,
StreamPosition streamPosition,
CancellationToken cancellationToken)
{
try
{
_executionContextManager.CurrentFor(
_thisMicroservice,
producerTenant,
_executionContextManager.Current.CorrelationId);
var publicEvents = await _getEventFetchers().GetPartitionedFetcherFor(
ScopeId.Default,
new StreamDefinition(new PublicFilterDefinition(StreamId.EventLog, publicStream)),
cancellationToken).ConfigureAwait(false);
while (!cancellationToken.IsCancellationRequested && !_disposed)
{
try
{
var tryGetStreamEvent = await publicEvents.FetchInPartition(partition, streamPosition, cancellationToken).ConfigureAwait(false);
if (!tryGetStreamEvent.Success)
{
await Task.Delay(250).ConfigureAwait(false);
continue;
}
var streamEvent = tryGetStreamEvent.Result;
var response = await dispatcher.Call(
new ConsumerRequest { Event = streamEvent.ToEventHorizonEvent() },
cancellationToken).ConfigureAwait(false);
if (response.Failure != null)
{
_logger.Warning(
"An error occurred while handling request. FailureId: {FailureId} Reason: {Reason}",
response.Failure.Id,
response.Failure.Reason);
return;
}
streamPosition = streamEvent.Position + 1;
}
catch (EventStoreUnavailable)
{
await Task.Delay(1000).ConfigureAwait(false);
}
}
}
catch (Exception ex)
{
_logger.Warning(ex, "An error ocurred while writing events to event horizon");
}
}
internal static partial void SuccessfullySubscribed(
ILogger logger,
MicroserviceId consumerMicroservice,
TenantId consumerTenant,
TenantId producerTenant,
StreamPosition streamPosition,
PartitionId partition,
StreamId publicStream);
internal static partial void IncomingEventHorizonSubscriptionWithArguments(
ILogger logger,
MicroserviceId consumerMicroservice,
TenantId consumerTenant,
TenantId producerTenant,
StreamPosition streamPosition,
PartitionId partition,
StreamId publicStream);
/// <summary>
/// Initializes a new instance of the <see cref="FailingPartitionState"/> class.
/// </summary>
/// <param name="position">The <see cref="StreamPosition" />.</param>
/// <param name="retryTime">The <see cref="DateTimeOffset" /> to retry processing.</param>
/// <param name="reason">The reason for failing.</param>
/// <param name="processingAttempt">The number of times the Event has been processed.</param>
/// <param name="lastFailed">The <see cref="DateTimeOffset" /> for when this partition last failed.</param>
public FailingPartitionState(StreamPosition position, DateTimeOffset retryTime, string reason, uint processingAttempt, DateTimeOffset lastFailed)
{
Position = position;
RetryTime = retryTime;
Reason = reason;
ProcessingAttempts = processingAttempt;
LastFailed = lastFailed;
}
public void StreamStreamReadGroupAsync_2()
{
var streamPositions = new StreamPosition[0] {
};
wrapper.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, CommandFlags.HighPriority);
mock.Verify(_ => _.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, CommandFlags.HighPriority));
}
public void StreamReadAsync_1()
{
var streamPositions = new StreamPosition[0] {
};
wrapper.StreamReadAsync(streamPositions, null, CommandFlags.HighPriority);
mock.Verify(_ => _.StreamReadAsync(streamPositions, null, CommandFlags.HighPriority));
}
public StreamPosition QueueMessageToCachedMessage(ref MemoryMessageData cachedMessage,
MemoryMessageData queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition setreamPosition = GetStreamPosition(queueMessage);
cachedMessage = queueMessage;
cachedMessage.Payload = SerializeMessageIntoPooledSegment(queueMessage);
return(setreamPosition);
}
/// <inheritdoc/>
public mongoDB.StreamEvent ToStoreStreamEvent(CommittedEvent committedEvent, StreamPosition streamPosition, PartitionId partition) =>
new mongoDB.StreamEvent(
streamPosition,
partition,
committedEvent.ExecutionContext.ToStoreRepresentation(),
committedEvent.GetStreamEventMetadata(),
committedEvent.GetAggregateMetadata(),
committedEvent.GetEventHorizonMetadata(),
BsonDocument.Parse(committedEvent.Content));
public CompressorPartial(MemoryMappedFile mmf, string targetFilePath, StreamPosition position, int bufferSize)
{
_mmf = mmf;
_targetFilePath = targetFilePath;
_position = position;
_bufferSize = bufferSize;
_asyncResult = new CompressorAsyncResult(_resetEvent);
}
/// <summary>
/// Initializes a new instance of the <see cref="StreamProcessorState"/> class.
/// </summary>
/// <param name="streamPosition">The <see cref="StreamPosition"/>position of the stream.</param>
/// <param name="failureReason">The reason for failing.</param>
/// <param name="retryTime">The <see cref="DateTimeOffset" /> for when to retry processing.</param>
/// <param name="processingAttempts">The number of times it has processed the Event at <see cref="Position" />.</param>
/// <param name="lastSuccessfullyProcessed">Timestamp of last successfull Stream process.</param>
/// <param name="isFailing">Whether the stream processor is failing.</param>
public StreamProcessorState(StreamPosition streamPosition, string failureReason, DateTimeOffset retryTime, uint processingAttempts, DateTimeOffset lastSuccessfullyProcessed, bool isFailing)
{
IsFailing = isFailing;
Position = streamPosition;
RetryTime = retryTime;
FailureReason = failureReason;
ProcessingAttempts = processingAttempts;
LastSuccessfullyProcessed = lastSuccessfullyProcessed;
}
public StreamPosition QueueMessageToCachedMessage(ref TestCachedMessage cachedMessage, TestQueueMessage queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition streamPosition = GetStreamPosition(queueMessage);
cachedMessage.StreamGuid = streamPosition.StreamIdentity.Guid;
cachedMessage.SequenceNumber = queueMessage.SequenceToken.SequenceNumber;
cachedMessage.EventIndex = queueMessage.SequenceToken.EventIndex;
return(streamPosition);
}
/// <summary>
/// Initializes a new instance of the <see cref="StreamProcessorState"/> class.
/// </summary>
/// <param name="streamPosition">The <see cref="StreamPosition"/>position of the stream.</param>
StreamProcessorState(StreamPosition streamPosition)
{
IsFailing = false;
Position = streamPosition;
RetryTime = DateTimeOffset.UtcNow;
FailureReason = string.Empty;
ProcessingAttempts = 0;
LastSuccessfullyProcessed = DateTimeOffset.MinValue;
}
private static async Task SubscribeToStream(EventStoreClient client)
{
#region subscribe-to-stream
await client.SubscribeToStreamAsync("some-stream",
async (subscription, evnt, cancellationToken) => {
Console.WriteLine($"Received event {evnt.OriginalEventNumber}@{evnt.OriginalStreamId}");
await HandleEvent(evnt);
});
#endregion subscribe-to-stream
#region subscribe-to-stream-from-position
await client.SubscribeToStreamAsync(
"some-stream",
StreamPosition.FromInt64(20),
EventAppeared);
#endregion subscribe-to-stream-from-position
#region subscribe-to-stream-live
await client.SubscribeToStreamAsync(
"some-stream",
StreamPosition.End,
EventAppeared);
#endregion subscribe-to-stream-live
#region subscribe-to-stream-resolving-linktos
await client.SubscribeToStreamAsync(
"$et-myEventType",
StreamPosition.Start,
EventAppeared,
resolveLinkTos : true);
#endregion subscribe-to-stream-resolving-linktos
#region subscribe-to-stream-subscription-dropped
var checkpoint = StreamPosition.Start;
await client.SubscribeToStreamAsync(
"some-stream",
checkpoint,
eventAppeared : async(subscription, evnt, cancellationToken) => {
await HandleEvent(evnt);
checkpoint = evnt.OriginalEventNumber;
},
subscriptionDropped : ((subscription, reason, exception) => {
Console.WriteLine($"Subscription was dropped due to {reason}. {exception}");
if (reason != SubscriptionDroppedReason.Disposed)
{
// Resubscribe if the client didn't stop the subscription
Resubscribe(checkpoint);
}
}));
#endregion subscribe-to-stream-subscription-dropped
}
public StreamPosition QueueMessageToCachedMessage(ref CachedMessage cachedMessage, GeneratedBatchContainer queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition setreamPosition = GetStreamPosition(queueMessage);
cachedMessage.StreamGuid = setreamPosition.StreamIdentity.Guid;
cachedMessage.StreamNamespace = setreamPosition.StreamIdentity.Namespace;
cachedMessage.SequenceNumber = queueMessage.RealToken.SequenceNumber;
cachedMessage.Payload = SerializeMessageIntoPooledSegment(queueMessage);
return(setreamPosition);
}
public StreamPosition QueueMessageToCachedMessage(ref TestCachedMessage cachedMessage, TestQueueMessage queueMessage, DateTime dequeueTimeUtc)
{
StreamPosition streamPosition = GetStreamPosition(queueMessage);
cachedMessage.StreamGuid = streamPosition.StreamIdentity.Guid;
cachedMessage.StreamNamespace = streamPosition.StreamIdentity.Namespace;
cachedMessage.SequenceNumber = queueMessage.SequenceNumber;
cachedMessage.Payload = SerializeMessageIntoPooledSegment(queueMessage);
return(streamPosition);
}
public async Task Connect(TransientStreamSubscriptionConnectionSettings connectionSettings)
{
_streamId = connectionSettings.StreamId;
_aggregateType = connectionSettings.AggregateType;
_startPosition = connectionSettings.StreamPosition;
_subscriptionName = connectionSettings.SubscriptionName;
await ConnectAsync();
_startDate = DateTime.UtcNow;
}
Task <(StreamProcessorState, FailingPartitionState)> SetFailingPartitionState(
IStreamProcessorId streamProcessorId,
StreamProcessorState oldState,
PartitionId partitionId,
uint processingAttempts,
TimeSpan retryTimeout,
string reason,
StreamPosition position,
DateTimeOffset lastFailed,
CancellationToken cancellationToken) =>
public void Stream_SetStreamPosition_Test1()
{
using (var ms = new MemoryStream(GetTestBuffer()))
{
var position = new StreamPosition(128, null);
Assert.AreEqual(false, ms.SetStreamPosition(position));
Assert.AreEqual(0, ms.Position);
}
}
public void Stream_SetStreamPosition_Test2()
{
using (var ms = new MemoryStream(GetTestBuffer()))
{
var position = new StreamPosition(32, null);
Assert.AreEqual(true, ms.SetStreamPosition(position));
Assert.AreEqual(32, ms.Position);
}
}
public void Stream_SetStreamPosition_Test3()
{
using (var fs = new FileStream(Path.Combine(Environment.CurrentDirectory, @"TestFiles\gpl-3.0-3.txt"), FileMode.Open, FileAccess.Read))
{
fs.Seek(512, SeekOrigin.Current);
var checkBuffer = new byte[128];
fs.Read(checkBuffer, 0, checkBuffer.Length);
var position = new StreamPosition(640, checkBuffer);
fs.Seek(0, SeekOrigin.Begin);
Assert.AreEqual(true, fs.SetStreamPosition(position));
Assert.AreEqual(640, fs.Position);
}
}
public void Stream_SetStreamPosition_Test4()
{
using (var fs = new FileStream(Path.Combine(Environment.CurrentDirectory, @"TestFiles\gpl-3.0-3.txt"), FileMode.Open, FileAccess.Read))
{
var random = new Random();
var checkBuffer = new byte[128];
random.NextBytes(checkBuffer);
var position = new StreamPosition(640, checkBuffer);
Assert.AreEqual(false, fs.SetStreamPosition(position));
Assert.AreEqual(0, fs.Position);
}
}
public BufferedTextFileReader()
{
_position = StreamPosition.Empty;
_items = new List<string>();
}
