public static StreamOptions FromStreamNameAndRevision( string streamName, StreamPosition streamRevision) { if (streamName == null) { throw new ArgumentNullException(nameof(streamName)); } if (streamRevision == StreamPosition.End) { return(new StreamOptions { StreamName = streamName, End = new Empty() }); } if (streamRevision == StreamPosition.Start) { return(new StreamOptions { StreamName = streamName, Start = new Empty() }); } return(new StreamOptions { StreamName = streamName, Revision = streamRevision }); }
public void StreamReadAsync_1() { var streamPositions = new StreamPosition[] { }; wrapper.StreamReadAsync(streamPositions, null, CommandFlags.None); mock.Verify(_ => _.StreamReadAsync(streamPositions, null, CommandFlags.None)); }
public async Task <IReadOnlyCollection <ResolvedEvent> > GetEventStream(string stream, string entityId, string model = null) { var response = new List <ResolvedEvent>(); var snapshots = await GetSnapshotStream(stream, entityId, model); var position = StreamPosition.Start; if (snapshots.Count > 0) { Console.WriteLine($"Using snapshot for model {model}"); var meta = JsonConvert.DeserializeObject <SnapshotMetaData>(Encoding.UTF8.GetString(snapshots.Last().Event.Metadata.Span)); position = new StreamPosition(meta.Position + 1); response.Add(snapshots.Last()); } var state = _client.ReadStreamAsync( Direction.Forwards, $"{stream}-{entityId}", position); var readState = await state.ReadState; if (readState == ReadState.StreamNotFound) { return(new List <ResolvedEvent>()); } response.AddRange(await state.ToListAsync()); return(response); }
public void StreamStreamReadGroupAsync_2() { var streamPositions = new StreamPosition[] { }; wrapper.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, false, CommandFlags.None); mock.Verify(_ => _.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, false, CommandFlags.None)); }
public async Task <StreamReadResults> ReadFrom(string streamId, Func <IAmAStoredEvent, bool> predicate = null, Direction direction = Direction.Backwards, CancellationToken cancellationToken = default) { var readResult = client.ReadStreamAsync(direction, streamId, direction == Direction.Backwards ? StreamPosition.End : StreamPosition.Start, resolveLinkTos: true, configureOperationOptions: options => options.TimeoutAfter = TimeSpan.FromSeconds(30), cancellationToken: cancellationToken); bool streamExists = false; try { var readState = await readResult.ReadState; streamExists = readState == ReadState.Ok; } #pragma warning disable 168 catch (StreamDeletedException ex) // This happens when the stream is hard-deleted. We don't want to throw in that case #pragma warning restore 168 { streamExists = false; } if (!streamExists) { return(new StreamReadResults(emptyReadResult, false, StreamPosition.FromInt64(-1))); } predicate ??= _ => true; var lastIndex = (await client.ReadStreamAsync(Direction.Backwards, streamId, revision: StreamPosition.End, maxCount: 1, resolveLinkTos: false).FirstAsync(cancellationToken)).OriginalEventNumber; IAsyncEnumerable <StoredEvent> storedEvents; if (direction == Direction.Backwards) { storedEvents = readResult // Trust me, resharper is wrong in this one. Event can be null // ReSharper disable once ConditionIsAlwaysTrueOrFalse .Where(e => e.Event != null) .Select((e, _) => e.Event.ToStoredEvent(stateFactory)) .TakeWhile(e => e.DeserializedEvent is not EntitySoftDeleted) .Where(e => predicate(e)); } else { storedEvents = readResult // Trust me, resharper is wrong in this one. Event can be null // ReSharper disable once ConditionIsAlwaysTrueOrFalse .Where(e => e.Event != null) .Select((e, c) => e.Event.ToStoredEvent(stateFactory)) .Where(e => predicate(e)); } return(new StreamReadResults(storedEvents, true, lastIndex)); }
// Placed object message payload into a segment. private ArraySegment <byte> EncodeMessageIntoSegment(StreamPosition streamPosition, EventData queueMessage) { byte[] propertiesBytes = queueMessage.SerializeProperties(); byte[] payload = queueMessage.GetBytes(); // get size of namespace, offset, partitionkey, properties, and payload int size = SegmentBuilder.CalculateAppendSize(streamPosition.StreamIdentity.Namespace) + SegmentBuilder.CalculateAppendSize(queueMessage.Offset) + SegmentBuilder.CalculateAppendSize(queueMessage.PartitionKey) + SegmentBuilder.CalculateAppendSize(propertiesBytes) + SegmentBuilder.CalculateAppendSize(payload); // get segment ArraySegment <byte> segment = GetSegment(size); // encode namespace, offset, partitionkey, properties and payload into segment int writeOffset = 0; SegmentBuilder.Append(segment, ref writeOffset, streamPosition.StreamIdentity.Namespace); SegmentBuilder.Append(segment, ref writeOffset, queueMessage.Offset); SegmentBuilder.Append(segment, ref writeOffset, queueMessage.PartitionKey); SegmentBuilder.Append(segment, ref writeOffset, propertiesBytes); SegmentBuilder.Append(segment, ref writeOffset, payload); return(segment); }
public async Task ProcessStreamItem(StreamPosition position) { StreamPosition currentPosition = null; bool found = _positions.TryGetValue(position.Id, out currentPosition); switch (position.Operation) { case StreamOperation.Insert: case StreamOperation.Update: if (!found || currentPosition.Compare(position) != 0) { if (_persistenceLayer != null) { await _persistenceLayer.ProcessStreamItem(position); } } _positions[position.Id] = position; break; case StreamOperation.Delete: if (found) { if (_persistenceLayer != null) { await _persistenceLayer.ProcessStreamItem(position); } _positions.TryRemove(position.Id, out currentPosition); } break; default: break; } }
Task <(StreamProcessorState, FailingPartitionState)> ChangePositionInFailingPartition( IStreamProcessorId streamProcessorId, StreamProcessorState oldState, PartitionId partitionId, StreamPosition newPosition, DateTimeOffset lastFailed, CancellationToken cancellationToken) =>
public IEnumerable<string> ReadAllLines(Stream stream, Encoding encoding) { using (var sr = new StreamReader(stream, encoding)) { if (stream.SetStreamPosition(_position)) { if (!IsEndOfLine(_position.CheckBuffer)) { ResetStream(_items, stream); } } else { _items.Clear(); } while (!sr.EndOfStream) { _items.Add(sr.ReadLine()); } _position = stream.GetStreamPosition(128); } return _items; }
StreamPosition iTrackReader.findKeyFrame(StreamPosition seekFrame) { var sp = findKeyFrame((MkvSeekPosition)seekFrame); Logger.logDebug("findKeyFrame: {0} -> {1}", seekFrame, sp); return(sp); }
public async IAsyncEnumerable <ResolvedEvent> ReadStreamAsync( IStream stream, Direction direction, StreamPosition revision, long maxCount = long.MaxValue, Action <EventStoreClientOperationOptions>?configureOperationOptions = null, bool resolveLinkTos = false, UserCredentials?userCredentials = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var results = this._eventStoreClient.ReadStreamAsync( direction, stream.Name, revision, maxCount, configureOperationOptions, resolveLinkTos, userCredentials, cancellationToken); var state = await results.ReadState; if (state != ReadState.StreamNotFound) { // todo, i've killed a kitten here, i'm sorry, but i wanted to avoid reflexion, seriously though, why the internal constructor on ReadStreamResults.. await foreach (var @event in results) { yield return(@event); } } }
/// <inheritdoc/> public async Task Write <TEvent>( IMongoCollection <TEvent> stream, FilterDefinitionBuilder <TEvent> filter, Func <StreamPosition, TEvent> createStoreEvent, CancellationToken cancellationToken) where TEvent : class { StreamPosition streamPosition = null; try { using var session = await _streams.StartSessionAsync().ConfigureAwait(false); await session.WithTransactionAsync( async (transaction, cancellationToken) => { streamPosition = (ulong)await stream.CountDocumentsAsync( transaction, filter.Empty).ConfigureAwait(false); await stream.InsertOneAsync( transaction, createStoreEvent(streamPosition), cancellationToken: cancellationToken).ConfigureAwait(false); return(Task.CompletedTask); }, cancellationToken : cancellationToken).ConfigureAwait(false); } catch (MongoWaitQueueFullException ex) { throw new EventStoreUnavailable("Mongo wait queue is full", ex); } }
public async Task <IList <IBatchContainer> > GetQueueMessagesAsync(int maxCount) { if (receiver == null || maxCount <= 0) { return(new List <IBatchContainer>()); } List <EventData> messages = (await receiver.ReceiveAsync(maxCount, ReceiveTimeout)).ToList(); var batches = new List <IBatchContainer>(); if (messages.Count == 0) { return(batches); } DateTime dequeueTimeUtc = DateTime.UtcNow; foreach (EventData message in messages) { StreamPosition streamPosition = cache.Add(message, dequeueTimeUtc); batches.Add(new StreamActivityNotificationBatch(streamPosition.StreamIdentity.Guid, streamPosition.StreamIdentity.Namespace, streamPosition.SequenceToken)); } if (!checkpointer.CheckpointExists) { checkpointer.Update(messages[0].Offset, DateTime.UtcNow); } return(batches); }
public async Task <IList <IBatchContainer> > GetQueueMessagesAsync(int maxCount) { if (receiver == null || maxCount <= 0) { return(new List <IBatchContainer>()); } List <EventData> messages; try { var watch = Stopwatch.StartNew(); messages = (await receiver.ReceiveAsync(maxCount, ReceiveTimeout)).ToList(); watch.Stop(); logger.TrackMetric(hubReceiveTimeMetric, watch.Elapsed); logger.TrackMetric(partitionReceiveTimeMetric, watch.Elapsed); logger.TrackMetric(hubReadFailure, 0); logger.TrackMetric(partitionReadFailure, 0); } catch (Exception ex) { logger.TrackMetric(hubReadFailure, 1); logger.TrackMetric(partitionReadFailure, 1); logger.Warn(OrleansServiceBusErrorCode.FailedPartitionRead, "Failed to read from EventHub partition {0}-{1}. : Exception: {2}.", config.Hub.Path, config.Partition, ex); throw; } var batches = new List <IBatchContainer>(); if (messages.Count == 0) { return(batches); } logger.TrackMetric(hubMessagesRecieved, messages.Count); logger.TrackMetric(partitionMessagesReceived, messages.Count); // monitor message age var dequeueTimeUtc = DateTime.UtcNow; TimeSpan difference = dequeueTimeUtc - messages[messages.Count - 1].EnqueuedTimeUtc; logger.TrackMetric(hubAgeOfMessagesBeingProcessed, difference); logger.TrackMetric(partitionAgeOfMessagesBeingProcessed, difference); foreach (EventData message in messages) { StreamPosition streamPosition = cache.Add(message, dequeueTimeUtc); batches.Add(new StreamActivityNotificationBatch(streamPosition.StreamIdentity.Guid, streamPosition.StreamIdentity.Namespace, streamPosition.SequenceToken)); } if (!checkpointer.CheckpointExists) { checkpointer.Update(messages[0].Offset, DateTime.UtcNow); } return(batches); }
async Task WriteEventsToEventHorizon( IReverseCallDispatcher <EventHorizonConsumerToProducerMessage, EventHorizonProducerToConsumerMessage, ConsumerSubscriptionRequest, Contracts.SubscriptionResponse, ConsumerRequest, ConsumerResponse> dispatcher, TenantId producerTenant, StreamId publicStream, PartitionId partition, StreamPosition streamPosition, CancellationToken cancellationToken) { try { _executionContextManager.CurrentFor( _thisMicroservice, producerTenant, _executionContextManager.Current.CorrelationId); var publicEvents = await _getEventFetchers().GetPartitionedFetcherFor( ScopeId.Default, new StreamDefinition(new PublicFilterDefinition(StreamId.EventLog, publicStream)), cancellationToken).ConfigureAwait(false); while (!cancellationToken.IsCancellationRequested && !_disposed) { try { var tryGetStreamEvent = await publicEvents.FetchInPartition(partition, streamPosition, cancellationToken).ConfigureAwait(false); if (!tryGetStreamEvent.Success) { await Task.Delay(250).ConfigureAwait(false); continue; } var streamEvent = tryGetStreamEvent.Result; var response = await dispatcher.Call( new ConsumerRequest { Event = streamEvent.ToEventHorizonEvent() }, cancellationToken).ConfigureAwait(false); if (response.Failure != null) { _logger.Warning( "An error occurred while handling request. FailureId: {FailureId} Reason: {Reason}", response.Failure.Id, response.Failure.Reason); return; } streamPosition = streamEvent.Position + 1; } catch (EventStoreUnavailable) { await Task.Delay(1000).ConfigureAwait(false); } } } catch (Exception ex) { _logger.Warning(ex, "An error ocurred while writing events to event horizon"); } }
internal static partial void SuccessfullySubscribed( ILogger logger, MicroserviceId consumerMicroservice, TenantId consumerTenant, TenantId producerTenant, StreamPosition streamPosition, PartitionId partition, StreamId publicStream);
internal static partial void IncomingEventHorizonSubscriptionWithArguments( ILogger logger, MicroserviceId consumerMicroservice, TenantId consumerTenant, TenantId producerTenant, StreamPosition streamPosition, PartitionId partition, StreamId publicStream);
/// <summary> /// Initializes a new instance of the <see cref="FailingPartitionState"/> class. /// </summary> /// <param name="position">The <see cref="StreamPosition" />.</param> /// <param name="retryTime">The <see cref="DateTimeOffset" /> to retry processing.</param> /// <param name="reason">The reason for failing.</param> /// <param name="processingAttempt">The number of times the Event has been processed.</param> /// <param name="lastFailed">The <see cref="DateTimeOffset" /> for when this partition last failed.</param> public FailingPartitionState(StreamPosition position, DateTimeOffset retryTime, string reason, uint processingAttempt, DateTimeOffset lastFailed) { Position = position; RetryTime = retryTime; Reason = reason; ProcessingAttempts = processingAttempt; LastFailed = lastFailed; }
public void StreamStreamReadGroupAsync_2() { var streamPositions = new StreamPosition[0] { }; wrapper.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, CommandFlags.HighPriority); mock.Verify(_ => _.StreamReadGroupAsync(streamPositions, "group", "consumer", 10, CommandFlags.HighPriority)); }
public void StreamReadAsync_1() { var streamPositions = new StreamPosition[0] { }; wrapper.StreamReadAsync(streamPositions, null, CommandFlags.HighPriority); mock.Verify(_ => _.StreamReadAsync(streamPositions, null, CommandFlags.HighPriority)); }
public StreamPosition QueueMessageToCachedMessage(ref MemoryMessageData cachedMessage, MemoryMessageData queueMessage, DateTime dequeueTimeUtc) { StreamPosition setreamPosition = GetStreamPosition(queueMessage); cachedMessage = queueMessage; cachedMessage.Payload = SerializeMessageIntoPooledSegment(queueMessage); return(setreamPosition); }
/// <inheritdoc/> public mongoDB.StreamEvent ToStoreStreamEvent(CommittedEvent committedEvent, StreamPosition streamPosition, PartitionId partition) => new mongoDB.StreamEvent( streamPosition, partition, committedEvent.ExecutionContext.ToStoreRepresentation(), committedEvent.GetStreamEventMetadata(), committedEvent.GetAggregateMetadata(), committedEvent.GetEventHorizonMetadata(), BsonDocument.Parse(committedEvent.Content));
public CompressorPartial(MemoryMappedFile mmf, string targetFilePath, StreamPosition position, int bufferSize) { _mmf = mmf; _targetFilePath = targetFilePath; _position = position; _bufferSize = bufferSize; _asyncResult = new CompressorAsyncResult(_resetEvent); }
/// <summary> /// Initializes a new instance of the <see cref="StreamProcessorState"/> class. /// </summary> /// <param name="streamPosition">The <see cref="StreamPosition"/>position of the stream.</param> /// <param name="failureReason">The reason for failing.</param> /// <param name="retryTime">The <see cref="DateTimeOffset" /> for when to retry processing.</param> /// <param name="processingAttempts">The number of times it has processed the Event at <see cref="Position" />.</param> /// <param name="lastSuccessfullyProcessed">Timestamp of last successfull Stream process.</param> /// <param name="isFailing">Whether the stream processor is failing.</param> public StreamProcessorState(StreamPosition streamPosition, string failureReason, DateTimeOffset retryTime, uint processingAttempts, DateTimeOffset lastSuccessfullyProcessed, bool isFailing) { IsFailing = isFailing; Position = streamPosition; RetryTime = retryTime; FailureReason = failureReason; ProcessingAttempts = processingAttempts; LastSuccessfullyProcessed = lastSuccessfullyProcessed; }
public StreamPosition QueueMessageToCachedMessage(ref TestCachedMessage cachedMessage, TestQueueMessage queueMessage, DateTime dequeueTimeUtc) { StreamPosition streamPosition = GetStreamPosition(queueMessage); cachedMessage.StreamGuid = streamPosition.StreamIdentity.Guid; cachedMessage.SequenceNumber = queueMessage.SequenceToken.SequenceNumber; cachedMessage.EventIndex = queueMessage.SequenceToken.EventIndex; return(streamPosition); }
/// <summary> /// Initializes a new instance of the <see cref="StreamProcessorState"/> class. /// </summary> /// <param name="streamPosition">The <see cref="StreamPosition"/>position of the stream.</param> StreamProcessorState(StreamPosition streamPosition) { IsFailing = false; Position = streamPosition; RetryTime = DateTimeOffset.UtcNow; FailureReason = string.Empty; ProcessingAttempts = 0; LastSuccessfullyProcessed = DateTimeOffset.MinValue; }
private static async Task SubscribeToStream(EventStoreClient client) { #region subscribe-to-stream await client.SubscribeToStreamAsync("some-stream", async (subscription, evnt, cancellationToken) => { Console.WriteLine($"Received event {evnt.OriginalEventNumber}@{evnt.OriginalStreamId}"); await HandleEvent(evnt); }); #endregion subscribe-to-stream #region subscribe-to-stream-from-position await client.SubscribeToStreamAsync( "some-stream", StreamPosition.FromInt64(20), EventAppeared); #endregion subscribe-to-stream-from-position #region subscribe-to-stream-live await client.SubscribeToStreamAsync( "some-stream", StreamPosition.End, EventAppeared); #endregion subscribe-to-stream-live #region subscribe-to-stream-resolving-linktos await client.SubscribeToStreamAsync( "$et-myEventType", StreamPosition.Start, EventAppeared, resolveLinkTos : true); #endregion subscribe-to-stream-resolving-linktos #region subscribe-to-stream-subscription-dropped var checkpoint = StreamPosition.Start; await client.SubscribeToStreamAsync( "some-stream", checkpoint, eventAppeared : async(subscription, evnt, cancellationToken) => { await HandleEvent(evnt); checkpoint = evnt.OriginalEventNumber; }, subscriptionDropped : ((subscription, reason, exception) => { Console.WriteLine($"Subscription was dropped due to {reason}. {exception}"); if (reason != SubscriptionDroppedReason.Disposed) { // Resubscribe if the client didn't stop the subscription Resubscribe(checkpoint); } })); #endregion subscribe-to-stream-subscription-dropped }
public StreamPosition QueueMessageToCachedMessage(ref CachedMessage cachedMessage, GeneratedBatchContainer queueMessage, DateTime dequeueTimeUtc) { StreamPosition setreamPosition = GetStreamPosition(queueMessage); cachedMessage.StreamGuid = setreamPosition.StreamIdentity.Guid; cachedMessage.StreamNamespace = setreamPosition.StreamIdentity.Namespace; cachedMessage.SequenceNumber = queueMessage.RealToken.SequenceNumber; cachedMessage.Payload = SerializeMessageIntoPooledSegment(queueMessage); return(setreamPosition); }
public StreamPosition QueueMessageToCachedMessage(ref TestCachedMessage cachedMessage, TestQueueMessage queueMessage, DateTime dequeueTimeUtc) { StreamPosition streamPosition = GetStreamPosition(queueMessage); cachedMessage.StreamGuid = streamPosition.StreamIdentity.Guid; cachedMessage.StreamNamespace = streamPosition.StreamIdentity.Namespace; cachedMessage.SequenceNumber = queueMessage.SequenceNumber; cachedMessage.Payload = SerializeMessageIntoPooledSegment(queueMessage); return(streamPosition); }
public async Task Connect(TransientStreamSubscriptionConnectionSettings connectionSettings) { _streamId = connectionSettings.StreamId; _aggregateType = connectionSettings.AggregateType; _startPosition = connectionSettings.StreamPosition; _subscriptionName = connectionSettings.SubscriptionName; await ConnectAsync(); _startDate = DateTime.UtcNow; }
Task <(StreamProcessorState, FailingPartitionState)> SetFailingPartitionState( IStreamProcessorId streamProcessorId, StreamProcessorState oldState, PartitionId partitionId, uint processingAttempts, TimeSpan retryTimeout, string reason, StreamPosition position, DateTimeOffset lastFailed, CancellationToken cancellationToken) =>
public void Stream_SetStreamPosition_Test1() { using (var ms = new MemoryStream(GetTestBuffer())) { var position = new StreamPosition(128, null); Assert.AreEqual(false, ms.SetStreamPosition(position)); Assert.AreEqual(0, ms.Position); } }
public void Stream_SetStreamPosition_Test2() { using (var ms = new MemoryStream(GetTestBuffer())) { var position = new StreamPosition(32, null); Assert.AreEqual(true, ms.SetStreamPosition(position)); Assert.AreEqual(32, ms.Position); } }
public void Stream_SetStreamPosition_Test3() { using (var fs = new FileStream(Path.Combine(Environment.CurrentDirectory, @"TestFiles\gpl-3.0-3.txt"), FileMode.Open, FileAccess.Read)) { fs.Seek(512, SeekOrigin.Current); var checkBuffer = new byte[128]; fs.Read(checkBuffer, 0, checkBuffer.Length); var position = new StreamPosition(640, checkBuffer); fs.Seek(0, SeekOrigin.Begin); Assert.AreEqual(true, fs.SetStreamPosition(position)); Assert.AreEqual(640, fs.Position); } }
public void Stream_SetStreamPosition_Test4() { using (var fs = new FileStream(Path.Combine(Environment.CurrentDirectory, @"TestFiles\gpl-3.0-3.txt"), FileMode.Open, FileAccess.Read)) { var random = new Random(); var checkBuffer = new byte[128]; random.NextBytes(checkBuffer); var position = new StreamPosition(640, checkBuffer); Assert.AreEqual(false, fs.SetStreamPosition(position)); Assert.AreEqual(0, fs.Position); } }
public BufferedTextFileReader() { _position = StreamPosition.Empty; _items = new List<string>(); }