Beispiel #1
0
            public StreamSubscription(IPublisher bus,
                                      string streamName,
                                      StreamRevision startRevision,
                                      bool resolveLinks,
                                      IPrincipal user,
                                      IReadIndex readIndex,
                                      CancellationToken cancellationToken)
            {
                if (bus == null)
                {
                    throw new ArgumentNullException(nameof(bus));
                }

                if (streamName == null)
                {
                    throw new ArgumentNullException(nameof(streamName));
                }

                if (readIndex == null)
                {
                    throw new ArgumentNullException(nameof(readIndex));
                }

                _bus                 = bus;
                _streamName          = streamName;
                _nextRevision        = startRevision;
                _resolveLinks        = resolveLinks;
                _user                = user;
                _readIndex           = readIndex;
                _disposedTokenSource = new CancellationTokenSource();
                _buffer              = new ConcurrentQueue <ResolvedEvent>();
                _tokenRegistration   = cancellationToken.Register(_disposedTokenSource.Dispose);
            }
            public ReadStreamForwards(IPublisher bus,
                                      string streamName,
                                      StreamRevision startRevision,
                                      ulong maxCount,
                                      bool resolveLinks,
                                      ClaimsPrincipal user,
                                      bool requiresLeader,
                                      DateTime deadline,
                                      Func <RpcException, Task> handleFailure,
                                      CancellationToken cancellationToken)
            {
                if (bus == null)
                {
                    throw new ArgumentNullException(nameof(bus));
                }

                if (streamName == null)
                {
                    throw new ArgumentNullException(nameof(streamName));
                }

                _bus                 = bus;
                _streamName          = streamName;
                _nextRevision        = startRevision;
                _maxCount            = maxCount;
                _resolveLinks        = resolveLinks;
                _user                = user;
                _requiresLeader      = requiresLeader;
                _deadline            = deadline;
                _disposedTokenSource = new CancellationTokenSource();
                _buffer              = new ConcurrentQueue <ResolvedEvent>();
                _handleFailure       = handleFailure;
                _tokenRegistration   = cancellationToken.Register(_disposedTokenSource.Dispose);
            }
Beispiel #3
0
            public ReadStreamBackwards(
                IPublisher bus,
                string streamName,
                StreamRevision startRevision,
                ulong maxCount,
                bool resolveLinks,
                ClaimsPrincipal user,
                CancellationToken cancellationToken)
            {
                if (bus == null)
                {
                    throw new ArgumentNullException(nameof(bus));
                }

                if (streamName == null)
                {
                    throw new ArgumentNullException(nameof(streamName));
                }

                _bus                 = bus;
                _streamName          = streamName;
                _nextRevision        = startRevision;
                _maxCount            = maxCount;
                _resolveLinks        = resolveLinks;
                _user                = user;
                _disposedTokenSource = new CancellationTokenSource();
                _buffer              = new ConcurrentQueue <ResolvedEvent>();
                _tokenRegistration   = cancellationToken.Register(_disposedTokenSource.Dispose);
            }
            public ReadStreamForwards(IPublisher bus,
                                      string streamName,
                                      StreamRevision startRevision,
                                      ulong maxCount,
                                      bool resolveLinks,
                                      ClaimsPrincipal user,
                                      bool requiresLeader,
                                      DateTime deadline,
                                      ReadReq.Types.Options.Types.UUIDOption uuidOption,
                                      uint compatibility,
                                      CancellationToken cancellationToken)
            {
                _bus               = bus ?? throw new ArgumentNullException(nameof(bus));
                _streamName        = streamName ?? throw new ArgumentNullException(nameof(streamName));
                _maxCount          = maxCount;
                _resolveLinks      = resolveLinks;
                _user              = user;
                _requiresLeader    = requiresLeader;
                _deadline          = deadline;
                _uuidOption        = uuidOption;
                _compatibility     = compatibility;
                _cancellationToken = cancellationToken;
                _semaphore         = new SemaphoreSlim(1, 1);
                _channel           = Channel.CreateBounded <ReadResp>(BoundedChannelOptions);

                ReadPage(startRevision);
            }
                public async ValueTask <bool> MoveNextAsync()
                {
ReadLoop:
                    if (_disposedTokenSource.IsCancellationRequested)
                    {
                        return(false);
                    }

                    if (_buffer.TryDequeue(out var current))
                    {
                        _current = current;
                        _currentStreamRevision = StreamRevision.FromInt64(current.OriginalEventNumber);
                        return(true);
                    }

                    var readNextSource = new TaskCompletionSource <bool>();

                    Guid correlationId = Guid.NewGuid();

                    Log.Verbose(
                        "Catch-up subscription {subscriptionId} to {streamName} reading next page starting from {nextRevision}.",
                        _subscriptionId, _streamName, _nextRevision);

                    _bus.Publish(new ClientMessage.ReadStreamEventsForward(
                                     correlationId, correlationId, new CallbackEnvelope(OnMessage), _streamName,
                                     _nextRevision.ToInt64(), ReadBatchSize, _resolveLinks, _requiresLeader, default, _user));
            public ReadStreamForwards(IPublisher bus,
                                      string streamName,
                                      StreamRevision startRevision,
                                      ulong maxCount,
                                      bool resolveLinks,
                                      ClaimsPrincipal user,
                                      bool requiresLeader,
                                      DateTime deadline,
                                      Func <RpcException, Task> onStreamNotFound,
                                      CancellationToken cancellationToken)
            {
                if (bus == null)
                {
                    throw new ArgumentNullException(nameof(bus));
                }

                if (streamName == null)
                {
                    throw new ArgumentNullException(nameof(streamName));
                }

                _bus               = bus;
                _streamName        = streamName;
                _maxCount          = maxCount;
                _resolveLinks      = resolveLinks;
                _user              = user;
                _requiresLeader    = requiresLeader;
                _deadline          = deadline;
                _onStreamNotFound  = onStreamNotFound;
                _cancellationToken = cancellationToken;
                _semaphore         = new SemaphoreSlim(1, 1);
                _channel           = Channel.CreateBounded <ResolvedEvent>(BoundedChannelOptions);

                ReadPage(startRevision);
            }
            private void ReadPage(StreamRevision startRevision)
            {
                Guid correlationId = Guid.NewGuid();

                _bus.Publish(new ClientMessage.ReadStreamEventsForward(
                                 correlationId, correlationId, new ContinuationEnvelope(OnMessage, _semaphore, _cancellationToken),
                                 _streamName, startRevision.ToInt64(), (int)Math.Min(ReadBatchSize, _maxCount), _resolveLinks,
                                 _requiresLeader, default, _user, expires: _deadline));
                public CatchupStreamSubscription(Guid subscriptionId,
                                                 IPublisher bus,
                                                 string streamName,
                                                 StreamRevision startRevision,
                                                 bool resolveLinks,
                                                 ClaimsPrincipal user,
                                                 bool requiresLeader,
                                                 IReadIndex readIndex,
                                                 TaskCompletionSource <bool> subscriptionStarted,
                                                 CancellationToken cancellationToken)
                {
                    if (bus == null)
                    {
                        throw new ArgumentNullException(nameof(bus));
                    }

                    if (streamName == null)
                    {
                        throw new ArgumentNullException(nameof(streamName));
                    }

                    if (readIndex == null)
                    {
                        throw new ArgumentNullException(nameof(readIndex));
                    }

                    if (subscriptionStarted == null)
                    {
                        throw new ArgumentNullException(nameof(subscriptionStarted));
                    }

                    _subscriptionId = subscriptionId;
                    _bus            = bus;
                    _streamName     = streamName;
                    _nextRevision   = startRevision == StreamRevision.End
                                                ? StreamRevision.FromInt64(readIndex.GetStreamLastEventNumber(_streamName) + 1)
                                                : startRevision;
                    _startRevision  = startRevision == StreamRevision.End ? StreamRevision.Start : startRevision;
                    _resolveLinks   = resolveLinks;
                    _user           = user;
                    _requiresLeader = requiresLeader;
                    var subscriptionStarted1 = subscriptionStarted;

                    _disposedTokenSource = new CancellationTokenSource();
                    _buffer            = new ConcurrentQueue <ResolvedEvent>();
                    _tokenRegistration = cancellationToken.Register(_disposedTokenSource.Dispose);

                    if (!subscriptionStarted1.Task.IsCompleted)
                    {
                        subscriptionStarted1.SetResult(true);
                    }

                    Log.Information(
                        "Catch-up subscription {subscriptionId} to {streamName}@{streamRevision} running...",
                        _subscriptionId, streamName, _nextRevision);
                }
Beispiel #9
0
            private void ReadPage(StreamRevision startRevision, Func <Message, CancellationToken, Task> onMessage)
            {
                Guid correlationId = Guid.NewGuid();

                Log.Verbose(
                    "Subscription {subscriptionId} to {streamName} reading next page starting from {nextRevision}.",
                    _subscriptionId, _streamName, startRevision);

                _bus.Publish(new ClientMessage.ReadStreamEventsForward(
                                 correlationId, correlationId, new ContinuationEnvelope(onMessage, _semaphore, _cancellationToken),
                                 _streamName, startRevision.ToInt64(), ReadBatchSize, _resolveLinks, _requiresLeader, default,
Beispiel #10
0
 private void Subscribe(StreamRevision startRevision, bool catchUp)
 {
     if (catchUp)
     {
         CatchUp(startRevision);
     }
     else
     {
         GoLive(startRevision);
     }
 }
Beispiel #11
0
            private void ReadPage(StreamRevision startRevision, ulong readCount = 0)
            {
                Guid correlationId = Guid.NewGuid();

                _bus.Publish(new ClientMessage.ReadStreamEventsForward(
                                 correlationId, correlationId, new ContinuationEnvelope(OnMessage, _semaphore, _cancellationToken),
                                 _streamName, startRevision.ToInt64(), (int)Math.Min(ReadBatchSize, _maxCount), _resolveLinks,
                                 _requiresLeader, null, _user, expires: _deadline));

                async Task OnMessage(Message message, CancellationToken ct)
                {
                    if (message is ClientMessage.NotHandled notHandled &&
                        RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                    {
                        _channel.Writer.TryComplete(ex);
                        return;
                    }

                    if (!(message is ClientMessage.ReadStreamEventsForwardCompleted completed))
                    {
                        _channel.Writer.TryComplete(
                            RpcExceptions.UnknownMessage <ClientMessage.ReadStreamEventsForwardCompleted>(message));
                        return;
                    }

                    switch (completed.Result)
                    {
                    case ReadStreamResult.Success:
                        var nextStreamPosition = (ulong)completed.NextEventNumber;

                        foreach (var @event in completed.Events)
                        {
                            if (readCount >= _maxCount)
                            {
                                await _channel.Writer.WriteAsync(new ReadResp {
                                    StreamPosition = new() {
                                        LastStreamPosition = (ulong)completed.LastEventNumber,
                                        NextStreamPosition = nextStreamPosition
                                    }
                                }, ct).ConfigureAwait(false);

                                _channel.Writer.TryComplete();
                                return;
                            }
                            await _channel.Writer.WriteAsync(new ReadResp {
                                Event = ConvertToReadEvent(_uuidOption, @event),
                            }, ct).ConfigureAwait(false);

                            nextStreamPosition = (ulong)@event.OriginalEvent.EventNumber;
                            readCount++;
                        }
            public StreamSubscription(
                IPublisher bus,
                string streamName,
                StreamRevision?startRevision,
                bool resolveLinks,
                ClaimsPrincipal user,
                bool requiresLeader,
                IReadIndex readIndex,
                CancellationToken cancellationToken)
            {
                if (bus == null)
                {
                    throw new ArgumentNullException(nameof(bus));
                }

                if (streamName == null)
                {
                    throw new ArgumentNullException(nameof(streamName));
                }

                if (readIndex == null)
                {
                    throw new ArgumentNullException(nameof(readIndex));
                }

                _subscriptionId      = Guid.NewGuid();
                _bus                 = bus;
                _streamName          = streamName;
                _resolveLinks        = resolveLinks;
                _user                = user;
                _requiresLeader      = requiresLeader;
                _readIndex           = readIndex;
                _cancellationToken   = cancellationToken;
                _subscriptionStarted = new TaskCompletionSource <bool>();

                _startRevision = startRevision == StreamRevision.End
                                        ? StreamRevision.FromInt64(readIndex.GetStreamLastEventNumber(_streamName) + 1)
                                        : startRevision + 1 ?? StreamRevision.Start;

                _inner = startRevision == StreamRevision.End
                                        ? (IStreamEnumerator) new LiveStreamSubscription(_subscriptionId, _bus, _streamName,
                                                                                         StreamRevision.FromInt64(readIndex.GetStreamLastEventNumber(_streamName) + 1), _resolveLinks,
                                                                                         _user, _requiresLeader, _subscriptionStarted, _cancellationToken)
                                        : new CatchupStreamSubscription(_subscriptionId, bus, streamName,
                                                                        startRevision + 1 ?? StreamRevision.Start, resolveLinks, user, _requiresLeader, readIndex,
                                                                        _subscriptionStarted, cancellationToken);
            }
Beispiel #13
0
            public StreamSubscription(
                IPublisher bus,
                string streamName,
                StreamRevision?startRevision,
                bool resolveLinks,
                ClaimsPrincipal user,
                bool requiresLeader,
                IReadIndex readIndex,
                CancellationToken cancellationToken)
            {
                if (bus == null)
                {
                    throw new ArgumentNullException(nameof(bus));
                }

                if (streamName == null)
                {
                    throw new ArgumentNullException(nameof(streamName));
                }

                if (readIndex == null)
                {
                    throw new ArgumentNullException(nameof(readIndex));
                }

                _subscriptionId      = Guid.NewGuid();
                _bus                 = bus;
                _streamName          = streamName;
                _resolveLinks        = resolveLinks;
                _user                = user;
                _requiresLeader      = requiresLeader;
                _cancellationToken   = cancellationToken;
                _subscriptionStarted = new TaskCompletionSource <bool>();
                _channel             = Channel.CreateBounded <ResolvedEvent>(BoundedChannelOptions);
                _semaphore           = new SemaphoreSlim(1, 1);

                SubscriptionId = _subscriptionId.ToString();

                Subscribe(startRevision == StreamRevision.End
                                        ? StreamRevision.FromInt64(readIndex.GetStreamLastEventNumber(_streamName) + 1)
                                        : startRevision + 1 ?? StreamRevision.Start, startRevision != StreamRevision.End);
            }
Beispiel #14
0
            private void GoLive(StreamRevision startRevision)
            {
                var liveEvents            = Channel.CreateBounded <ResolvedEvent>(BoundedChannelOptions);
                var caughtUpSource        = new TaskCompletionSource <StreamRevision>();
                var liveMessagesCancelled = 0;

                Log.Information(
                    "Live subscription {subscriptionId} to {streamName} running from {streamRevision}...",
                    _subscriptionId, _streamName, startRevision);

                _bus.Publish(new ClientMessage.SubscribeToStream(Guid.NewGuid(), _subscriptionId,
                                                                 new ContinuationEnvelope(OnSubscriptionMessage, _semaphore, _cancellationToken), _subscriptionId,
                                                                 _streamName, _resolveLinks, _user));

                Task.Factory.StartNew(PumpLiveMessages, _cancellationToken);

                async Task PumpLiveMessages()
                {
                    await caughtUpSource.Task.ConfigureAwait(false);

                    await foreach (var @event in liveEvents.Reader.ReadAllAsync(_cancellationToken)
                                   .ConfigureAwait(false))
                    {
                        await _channel.Writer.WriteAsync(@event, _cancellationToken).ConfigureAwait(false);
                    }
                }

                async Task OnSubscriptionMessage(Message message, CancellationToken ct)
                {
                    if (message is ClientMessage.NotHandled notHandled &&
                        RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                    {
                        Fail(ex);
                        return;
                    }

                    switch (message)
                    {
                    case ClientMessage.SubscriptionConfirmation confirmed:
                        ConfirmSubscription();
                        var caughtUp = StreamRevision.FromInt64(confirmed.LastEventNumber.Value);
                        Log.Verbose(
                            "Live subscription {subscriptionId} to {streamName} confirmed at {streamRevision}.",
                            _subscriptionId, _streamName, caughtUp);
                        ReadHistoricalEvents(startRevision);

                        async Task OnHistoricalEventsMessage(Message message, CancellationToken ct)
                        {
                            if (message is ClientMessage.NotHandled notHandled &&
                                RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                            {
                                Fail(ex);
                                return;
                            }

                            if (!(message is ClientMessage.ReadStreamEventsForwardCompleted completed))
                            {
                                Fail(
                                    RpcExceptions.UnknownMessage <ClientMessage.ReadStreamEventsForwardCompleted>(
                                        message));
                                return;
                            }

                            switch (completed.Result)
                            {
                            case ReadStreamResult.Success:
                                if (completed.Events.Length == 0 && completed.IsEndOfStream)
                                {
                                    NotifyCaughtUp(StreamRevision.FromInt64(completed.FromEventNumber));
                                    return;
                                }

                                foreach (var @event in completed.Events)
                                {
                                    var streamRevision = StreamRevision.FromInt64(@event.OriginalEventNumber);
                                    if (streamRevision > caughtUp)
                                    {
                                        NotifyCaughtUp(streamRevision);
                                        return;
                                    }

                                    await _channel.Writer.WriteAsync(@event, _cancellationToken)
                                    .ConfigureAwait(false);
                                }

                                ReadHistoricalEvents(StreamRevision.FromInt64(completed.NextEventNumber));

                                void NotifyCaughtUp(StreamRevision streamRevision)
                                {
                                    Log.Verbose(
                                        "Live subscription {subscriptionId} to {streamName} caught up at {streamRevision} because the end of stream was reached.",
                                        _subscriptionId, _streamName, streamRevision);
                                    caughtUpSource.TrySetResult(caughtUp);
                                }

                                return;

                            case ReadStreamResult.NoStream:
                                Log.Verbose(
                                    "Live subscription {subscriptionId} to {streamName} stream not found.",
                                    _subscriptionId, _streamName);
                                await Task.Delay(TimeSpan.FromMilliseconds(50), ct).ConfigureAwait(false);

                                ReadHistoricalEvents(startRevision);
                                return;

                            case ReadStreamResult.StreamDeleted:
                                Log.Verbose(
                                    "Live subscription {subscriptionId} to {streamName} stream deleted.",
                                    _subscriptionId, _streamName);
                                Fail(RpcExceptions.StreamDeleted(_streamName));
                                return;

                            case ReadStreamResult.AccessDenied:
                                Fail(RpcExceptions.AccessDenied());
                                return;

                            default:
                                Fail(RpcExceptions.UnknownError(completed.Result));
                                return;
                            }
                        }

                        void ReadHistoricalEvents(StreamRevision fromStreamRevision)
                        {
                            if (fromStreamRevision == StreamRevision.End)
                            {
                                throw new ArgumentOutOfRangeException(nameof(fromStreamRevision));
                            }

                            Log.Verbose(
                                "Live subscription {subscriptionId} to {streamName} loading any missed events starting from {streamRevision}.",
                                _subscriptionId, _streamName, fromStreamRevision);

                            ReadPage(fromStreamRevision, OnHistoricalEventsMessage);
                        }

                        return;

                    case ClientMessage.SubscriptionDropped dropped:
                        Log.Debug(
                            "Live subscription {subscriptionId} to {streamName} dropped: {droppedReason}",
                            _subscriptionId, _streamName, dropped.Reason);
                        switch (dropped.Reason)
                        {
                        case SubscriptionDropReason.AccessDenied:
                            Fail(RpcExceptions.AccessDenied());
                            return;

                        case SubscriptionDropReason.NotFound:
                            Fail(RpcExceptions.StreamNotFound(_streamName));
                            return;

                        case SubscriptionDropReason.Unsubscribed:
                            return;

                        default:
                            Fail(RpcExceptions.UnknownError(dropped.Reason));
                            return;
                        }

                    case ClientMessage.StreamEventAppeared appeared: {
                        if (liveMessagesCancelled == 1)
                        {
                            return;
                        }

                        using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(1));
                        try {
                            Log.Verbose(
                                "Live subscription {subscriptionId} to {streamName} received event {streamRevision}.",
                                _subscriptionId, _streamName, appeared.Event.OriginalEventNumber);

                            await liveEvents.Writer.WriteAsync(appeared.Event, cts.Token)
                            .ConfigureAwait(false);
                        } catch (Exception e) {
                            if (Interlocked.Exchange(ref liveMessagesCancelled, 1) != 0)
                            {
                                return;
                            }

                            Log.Verbose(
                                e,
                                "Live subscription {subscriptionId} to {streamName} timed out at {streamRevision}; unsubscribing...",
                                _subscriptionId, _streamName,
                                StreamRevision.FromInt64(appeared.Event.OriginalEventNumber));

                            Unsubscribe();

                            liveEvents.Writer.Complete();

                            CatchUp(StreamRevision.FromInt64(
                                        _current.GetValueOrDefault().OriginalEvent.EventNumber));
                        }

                        return;
                    }

                    default:
                        Fail(
                            RpcExceptions.UnknownMessage <ClientMessage.SubscriptionConfirmation>(message));
                        return;
                    }
                }

                void Fail(Exception exception)
                {
                    this.Fail(exception);
                    caughtUpSource.TrySetException(exception);
                }
            }
Beispiel #15
0
        public override async Task <AppendResp> Append(
            IAsyncStreamReader <AppendReq> requestStream,
            ServerCallContext context)
        {
            if (!await requestStream.MoveNext().ConfigureAwait(false))
            {
                throw new InvalidOperationException();
            }

            if (requestStream.Current.ContentCase != AppendReq.ContentOneofCase.Options)
            {
                throw new InvalidOperationException();
            }

            var options         = requestStream.Current.Options;
            var streamName      = options.StreamName;
            var expectedVersion = options.ExpectedStreamRevisionCase switch {
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.Revision => new StreamRevision(
                    options.Revision).ToInt64(),
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.Any => AnyStreamRevision.Any.ToInt64(),
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.StreamExists => AnyStreamRevision.StreamExists.ToInt64(),
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.NoStream => AnyStreamRevision.NoStream.ToInt64(),
                _ => throw new InvalidOperationException()
            };

            var user = await GetUser(_authenticationProvider, context.RequestHeaders).ConfigureAwait(false);

            var correlationId = Guid.NewGuid();             // TODO: JPB use request id?

            var events = new List <Event>();

            var size = 0;

            while (await requestStream.MoveNext().ConfigureAwait(false))
            {
                if (requestStream.Current.ContentCase != AppendReq.ContentOneofCase.ProposedMessage)
                {
                    throw new InvalidOperationException();
                }

                var proposedMessage = requestStream.Current.ProposedMessage;
                var data            = proposedMessage.Data.ToByteArray();
                size += data.Length;

                if (size > _maxAppendSize)
                {
                    throw RpcExceptions.MaxAppendSizeExceeded(_maxAppendSize);
                }

                if (!proposedMessage.Metadata.TryGetValue(Constants.Metadata.Type, out var eventType))
                {
                    throw RpcExceptions.RequiredMetadataPropertyMissing(Constants.Metadata.Type);
                }

                if (!proposedMessage.Metadata.TryGetValue(Constants.Metadata.IsJson, out var isJson))
                {
                    throw RpcExceptions.RequiredMetadataPropertyMissing(Constants.Metadata.IsJson);
                }

                events.Add(new Event(
                               Uuid.FromDto(proposedMessage.Id).ToGuid(),
                               eventType,
                               bool.Parse(isJson),
                               data,
                               proposedMessage.CustomMetadata.ToByteArray()));
            }

            var appendResponseSource = new TaskCompletionSource <AppendResp>();

            var envelope = new CallbackEnvelope(HandleWriteEventsCompleted);

            _queue.Publish(new ClientMessage.WriteEvents(
                               correlationId,
                               correlationId,
                               envelope,
                               true,
                               streamName,
                               expectedVersion,
                               events.ToArray(),
                               user));

            return(await appendResponseSource.Task.ConfigureAwait(false));

            void HandleWriteEventsCompleted(Message message)
            {
                if (message is ClientMessage.NotHandled notHandled && RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                {
                    appendResponseSource.TrySetException(ex);
                    return;
                }

                if (!(message is ClientMessage.WriteEventsCompleted completed))
                {
                    appendResponseSource.TrySetException(
                        RpcExceptions.UnknownMessage <ClientMessage.WriteEventsCompleted>(message));
                    return;
                }

                switch (completed.Result)
                {
                case OperationResult.Success:
                    var response = new AppendResp();

                    if (completed.LastEventNumber == -1)
                    {
                        response.NoStream = new AppendResp.Types.Empty();
                    }
                    else
                    {
                        response.CurrentRevision = StreamRevision.FromInt64(completed.LastEventNumber);
                    }

                    if (completed.CommitPosition == -1)
                    {
                        response.Empty = new AppendResp.Types.Empty();
                    }
                    else
                    {
                        var position = Position.FromInt64(completed.CommitPosition, completed.PreparePosition);
                        response.Position = new AppendResp.Types.Position {
                            CommitPosition  = position.CommitPosition,
                            PreparePosition = position.PreparePosition
                        };
                    }

                    appendResponseSource.TrySetResult(response);
                    return;

                case OperationResult.PrepareTimeout:
                case OperationResult.CommitTimeout:
                case OperationResult.ForwardTimeout:
                    appendResponseSource.TrySetException(RpcExceptions.Timeout());
                    return;

                case OperationResult.WrongExpectedVersion:
                    appendResponseSource.TrySetException(RpcExceptions.WrongExpectedVersion(
                                                             streamName,
                                                             expectedVersion,
                                                             completed.CurrentVersion));
                    return;

                case OperationResult.StreamDeleted:
                    appendResponseSource.TrySetException(RpcExceptions.StreamDeleted(streamName));
                    return;

                case OperationResult.InvalidTransaction:
                    appendResponseSource.TrySetException(RpcExceptions.InvalidTransaction());
                    return;

                case OperationResult.AccessDenied:
                    appendResponseSource.TrySetException(RpcExceptions.AccessDenied());
                    return;

                default:
                    appendResponseSource.TrySetException(RpcExceptions.UnknownError(completed.Result));
                    return;
                }
            }
        }
    }
Beispiel #16
0
        public override async Task <AppendResp> Append(
            IAsyncStreamReader <AppendReq> requestStream,
            ServerCallContext context)
        {
            if (!await requestStream.MoveNext().ConfigureAwait(false))
            {
                throw new InvalidOperationException();
            }

            if (requestStream.Current.ContentCase != AppendReq.ContentOneofCase.Options)
            {
                throw new InvalidOperationException();
            }

            var options    = requestStream.Current.Options;
            var streamName = options.StreamIdentifier;

            var expectedVersion = options.ExpectedStreamRevisionCase switch {
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.Revision => new StreamRevision(
                    options.Revision).ToInt64(),
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.Any => AnyStreamRevision.Any.ToInt64(),
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.StreamExists => AnyStreamRevision.StreamExists.ToInt64(),
                AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.NoStream => AnyStreamRevision.NoStream.ToInt64(),
                _ => throw new InvalidOperationException()
            };

            var requiresLeader = GetRequiresLeader(context.RequestHeaders);

            var user = context.GetHttpContext().User;
            var op   = WriteOperation.WithParameter(Plugins.Authorization.Operations.Streams.Parameters.StreamId(streamName));

            if (!await _provider.CheckAccessAsync(user, op, context.CancellationToken).ConfigureAwait(false))
            {
                throw AccessDenied();
            }

            var correlationId = Guid.NewGuid();             // TODO: JPB use request id?

            var events = new List <Event>();

            var size = 0;

            while (await requestStream.MoveNext().ConfigureAwait(false))
            {
                if (requestStream.Current.ContentCase != AppendReq.ContentOneofCase.ProposedMessage)
                {
                    throw new InvalidOperationException();
                }

                var proposedMessage = requestStream.Current.ProposedMessage;
                var data            = proposedMessage.Data.ToByteArray();
                var metadata        = proposedMessage.CustomMetadata.ToByteArray();

                if (!proposedMessage.Metadata.TryGetValue(Constants.Metadata.Type, out var eventType))
                {
                    throw RpcExceptions.RequiredMetadataPropertyMissing(Constants.Metadata.Type);
                }

                if (!proposedMessage.Metadata.TryGetValue(Constants.Metadata.ContentType, out var contentType))
                {
                    throw RpcExceptions.RequiredMetadataPropertyMissing(Constants.Metadata.ContentType);
                }

                size += Event.SizeOnDisk(eventType, data, metadata);

                if (size > _maxAppendSize)
                {
                    throw RpcExceptions.MaxAppendSizeExceeded(_maxAppendSize);
                }

                events.Add(new Event(
                               Uuid.FromDto(proposedMessage.Id).ToGuid(),
                               eventType,
                               contentType == Constants.Metadata.ContentTypes.ApplicationJson,
                               data,
                               metadata));
            }

            var appendResponseSource = new TaskCompletionSource <AppendResp>();

            var envelope = new CallbackEnvelope(HandleWriteEventsCompleted);

            _publisher.Publish(new ClientMessage.WriteEvents(
                                   correlationId,
                                   correlationId,
                                   envelope,
                                   requiresLeader,
                                   streamName,
                                   expectedVersion,
                                   events.ToArray(),
                                   user,
                                   cancellationToken: context.CancellationToken));

            return(await appendResponseSource.Task.ConfigureAwait(false));

            void HandleWriteEventsCompleted(Message message)
            {
                if (message is ClientMessage.NotHandled notHandled && RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                {
                    appendResponseSource.TrySetException(ex);
                    return;
                }

                if (!(message is ClientMessage.WriteEventsCompleted completed))
                {
                    appendResponseSource.TrySetException(
                        RpcExceptions.UnknownMessage <ClientMessage.WriteEventsCompleted>(message));
                    return;
                }

                var response = new AppendResp();

                switch (completed.Result)
                {
                case OperationResult.Success:
                    response.Success = new AppendResp.Types.Success();
                    if (completed.LastEventNumber == -1)
                    {
                        response.Success.NoStream = new Empty();
                    }
                    else
                    {
                        response.Success.CurrentRevision = StreamRevision.FromInt64(completed.LastEventNumber);
                    }

                    if (completed.CommitPosition == -1)
                    {
                        response.Success.NoPosition = new Empty();
                    }
                    else
                    {
                        var position = Position.FromInt64(completed.CommitPosition, completed.PreparePosition);
                        response.Success.Position = new AppendResp.Types.Position {
                            CommitPosition  = position.CommitPosition,
                            PreparePosition = position.PreparePosition
                        };
                    }

                    appendResponseSource.TrySetResult(response);
                    return;

                case OperationResult.PrepareTimeout:
                case OperationResult.CommitTimeout:
                case OperationResult.ForwardTimeout:
                    appendResponseSource.TrySetException(RpcExceptions.Timeout());
                    return;

                case OperationResult.WrongExpectedVersion:
                    response.WrongExpectedVersion = new AppendResp.Types.WrongExpectedVersion();

                    switch (options.ExpectedStreamRevisionCase)
                    {
                    case AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.Any:
                        response.WrongExpectedVersion.ExpectedAny = new Empty();
                        response.WrongExpectedVersion.Any2060     = new Empty();
                        break;

                    case AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.StreamExists:
                        response.WrongExpectedVersion.ExpectedStreamExists = new Empty();
                        response.WrongExpectedVersion.StreamExists2060     = new Empty();
                        break;

                    case AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.NoStream:
                        response.WrongExpectedVersion.ExpectedNoStream     = new Empty();
                        response.WrongExpectedVersion.ExpectedRevision2060 = ulong.MaxValue;
                        break;

                    case AppendReq.Types.Options.ExpectedStreamRevisionOneofCase.Revision:
                        response.WrongExpectedVersion.ExpectedRevision =
                            StreamRevision.FromInt64(expectedVersion);
                        response.WrongExpectedVersion.ExpectedRevision2060 =
                            StreamRevision.FromInt64(expectedVersion);
                        break;
                    }

                    if (completed.CurrentVersion == -1)
                    {
                        response.WrongExpectedVersion.CurrentNoStream = new Empty();
                        response.WrongExpectedVersion.NoStream2060    = new Empty();
                    }
                    else
                    {
                        response.WrongExpectedVersion.CurrentRevision =
                            StreamRevision.FromInt64(completed.CurrentVersion);
                        response.WrongExpectedVersion.CurrentRevision2060 =
                            StreamRevision.FromInt64(completed.CurrentVersion);
                    }

                    appendResponseSource.TrySetResult(response);
                    return;

                case OperationResult.StreamDeleted:
                    appendResponseSource.TrySetException(RpcExceptions.StreamDeleted(streamName));
                    return;

                case OperationResult.InvalidTransaction:
                    appendResponseSource.TrySetException(RpcExceptions.InvalidTransaction());
                    return;

                case OperationResult.AccessDenied:
                    appendResponseSource.TrySetException(RpcExceptions.AccessDenied());
                    return;

                default:
                    appendResponseSource.TrySetException(RpcExceptions.UnknownError(completed.Result));
                    return;
                }
            }
        }
    }
            private void ReadPage(StreamRevision startRevision, ulong readCount = 0)
            {
                Guid correlationId = Guid.NewGuid();

                _bus.Publish(new ClientMessage.ReadStreamEventsForward(
                                 correlationId, correlationId, new ContinuationEnvelope(OnMessage, _semaphore, _cancellationToken),
                                 _streamName, startRevision.ToInt64(), (int)Math.Min(ReadBatchSize, _maxCount), _resolveLinks,
                                 _requiresLeader, null, _user, expires: _deadline));

                async Task OnMessage(Message message, CancellationToken ct)
                {
                    if (message is ClientMessage.NotHandled notHandled &&
                        RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                    {
                        _channel.Writer.TryComplete(ex);
                        return;
                    }

                    if (!(message is ClientMessage.ReadStreamEventsForwardCompleted completed))
                    {
                        _channel.Writer.TryComplete(
                            RpcExceptions.UnknownMessage <ClientMessage.ReadStreamEventsForwardCompleted>(message));
                        return;
                    }

                    switch (completed.Result)
                    {
                    case ReadStreamResult.Success:
                        if (readCount == 0 && _compatibility >= 1)
                        {
                            if (completed.Events.Length == 0)
                            {
                                var firstStreamPosition = StreamRevision.FromInt64(completed.NextEventNumber);
                                if (startRevision != firstStreamPosition)
                                {
                                    await _channel.Writer.WriteAsync(new() {
                                        FirstStreamPosition = firstStreamPosition
                                    }, ct).ConfigureAwait(false);
                                }
                            }
                        }

                        foreach (var @event in completed.Events)
                        {
                            if (readCount >= _maxCount)
                            {
                                break;
                            }
                            await _channel.Writer.WriteAsync(new() {
                                Event = ConvertToReadEvent(_uuidOption, @event)
                            }, ct).ConfigureAwait(false);

                            readCount++;
                        }

                        if (!completed.IsEndOfStream && readCount < _maxCount)
                        {
                            ReadPage(StreamRevision.FromInt64(completed.NextEventNumber), readCount);
                            return;
                        }

                        if (_compatibility >= 1)
                        {
                            await _channel.Writer.WriteAsync(new() {
                                LastStreamPosition = StreamRevision.FromInt64(completed.LastEventNumber)
                            }, ct).ConfigureAwait(false);
                        }

                        _channel.Writer.TryComplete();
                        return;

                    case ReadStreamResult.NoStream:
                        await _channel.Writer.WriteAsync(new() {
                            StreamNotFound = new() {
                                StreamIdentifier = _streamName
                            }
                        }, _cancellationToken).ConfigureAwait(false);

                        _channel.Writer.TryComplete();
                        return;

                    case ReadStreamResult.StreamDeleted:
                        _channel.Writer.TryComplete(RpcExceptions.StreamDeleted(_streamName));
                        return;

                    case ReadStreamResult.AccessDenied:
                        _channel.Writer.TryComplete(RpcExceptions.AccessDenied());
                        return;

                    default:
                        _channel.Writer.TryComplete(RpcExceptions.UnknownError(completed.Result));
                        return;
                    }
                }
            }
Beispiel #18
0
            private async Task Receive(ChannelWriter <BatchAppendResp> writer, ClaimsPrincipal user, bool requiresLeader,
                                       CancellationToken cancellationToken)
            {
                var pendingWrites = new ConcurrentDictionary <Guid, ClientWriteRequest>();

                try {
                    await foreach (var request in _requestStream.ReadAllAsync(cancellationToken))
                    {
                        try {
                            var correlationId = Uuid.FromDto(request.CorrelationId).ToGuid();

                            if (request.Options != null)
                            {
                                TimeSpan timeout = Min(GetRequestedTimeout(request.Options), _writeTimeout);

                                if (!await _authorizationProvider.CheckAccessAsync(user, WriteOperation.WithParameter(
                                                                                       Plugins.Authorization.Operations.Streams.Parameters.StreamId(
                                                                                           request.Options.StreamIdentifier)), cancellationToken).ConfigureAwait(false))
                                {
                                    await writer.WriteAsync(new BatchAppendResp {
                                        CorrelationId    = request.CorrelationId,
                                        StreamIdentifier = request.Options.StreamIdentifier,
                                        Error            = Status.AccessDenied
                                    }, cancellationToken).ConfigureAwait(false);

                                    continue;
                                }

                                if (request.Options.StreamIdentifier == null)
                                {
                                    await writer.WriteAsync(new BatchAppendResp {
                                        CorrelationId    = request.CorrelationId,
                                        StreamIdentifier = request.Options.StreamIdentifier,
                                        Error            = Status.BadRequest(
                                            $"Required field {nameof(request.Options.StreamIdentifier)} not set.")
                                    }, cancellationToken).ConfigureAwait(false);

                                    continue;
                                }

                                if (Max(timeout, TimeSpan.Zero) == TimeSpan.Zero)
                                {
                                    await writer.WriteAsync(new BatchAppendResp {
                                        CorrelationId    = request.CorrelationId,
                                        StreamIdentifier = request.Options.StreamIdentifier,
                                        Error            = Status.Timeout
                                    }, cancellationToken).ConfigureAwait(false);

                                    continue;
                                }

                                pendingWrites.AddOrUpdate(correlationId,
                                                          c => FromOptions(c, request.Options, timeout, cancellationToken),
                                                          (_, writeRequest) => writeRequest);
                            }

                            if (!pendingWrites.TryGetValue(correlationId, out var clientWriteRequest))
                            {
                                continue;
                            }

                            clientWriteRequest.AddEvents(request.ProposedMessages.Select(FromProposedMessage));

                            if (clientWriteRequest.Size > _maxAppendSize)
                            {
                                pendingWrites.TryRemove(correlationId, out _);
                                await writer.WriteAsync(new BatchAppendResp {
                                    CorrelationId    = request.CorrelationId,
                                    StreamIdentifier = clientWriteRequest.StreamId,
                                    Error            = Status.MaximumAppendSizeExceeded((uint)_maxAppendSize)
                                }, cancellationToken).ConfigureAwait(false);
                            }

                            if (!request.IsFinal)
                            {
                                continue;
                            }

                            if (!pendingWrites.TryRemove(correlationId, out _))
                            {
                                continue;
                            }

                            Interlocked.Increment(ref _pending);

                            _publisher.Publish(ToInternalMessage(clientWriteRequest, new CallbackEnvelope(message => {
                                try {
                                    writer.TryWrite(ConvertMessage(message));
                                } catch (Exception ex) {
                                    writer.TryComplete(ex);
                                }
                            }), requiresLeader, user, cancellationToken));

                            BatchAppendResp ConvertMessage(Message message)
                            {
                                var batchAppendResp = message switch {
                                    ClientMessage.NotHandled notHandled => new BatchAppendResp {
                                        Error = new Status {
                                            Details = Any.Pack(new Empty()),
                                            Message = (notHandled.Reason, notHandled.AdditionalInfo) switch {
                                                (NotHandledReason.NotReady, _) => "Server Is Not Ready",
                                                (NotHandledReason.TooBusy, _) => "Server Is Busy",
                                                (NotHandledReason.NotLeader or NotHandledReason.IsReadOnly,
                                                 LeaderInfo leaderInfo) =>
                                                throw RpcExceptions.LeaderInfo(leaderInfo.HttpAddress,
                                                                               leaderInfo.HttpPort),
                                                      (NotHandledReason.NotLeader or NotHandledReason.IsReadOnly, _) =>
                                                      "No leader info available in response",
                                                      _ => $"Unknown {nameof(NotHandledReason)} ({(int)notHandled.Reason})"
                                            }
                                        }
                                    },
                                    ClientMessage.WriteEventsCompleted completed => completed.Result switch {
                                        OperationResult.Success => new BatchAppendResp {
                                            Success = BatchAppendResp.Types.Success.Completed(completed.CommitPosition,
                                                                                              completed.PreparePosition, completed.LastEventNumber),
                                        },
                                        OperationResult.WrongExpectedVersion => new BatchAppendResp {
                                            Error = Status.WrongExpectedVersion(
                                                StreamRevision.FromInt64(completed.CurrentVersion),
                                                clientWriteRequest.ExpectedVersion)
                                        },
                                        OperationResult.AccessDenied => new BatchAppendResp
                                        {
                                            Error = Status.AccessDenied
                                        },
                                        OperationResult.StreamDeleted => new BatchAppendResp {
                                            Error = Status.StreamDeleted(clientWriteRequest.StreamId)
                                        },
                                        OperationResult.CommitTimeout or
                                        OperationResult.ForwardTimeout or
                                        OperationResult.PrepareTimeout => new BatchAppendResp
                                        {
                                            Error = Status.Timeout
                                        },
                                        _ => new BatchAppendResp {
                                            Error = Status.Unknown
                                        }
                                    },
                                    _ => new BatchAppendResp {
                                        Error = new Status {
                                            Details = Any.Pack(new Empty()),
                                            Message =
                                                $"Envelope callback expected either {nameof(ClientMessage.WriteEventsCompleted)} or {nameof(ClientMessage.NotHandled)}, received {message.GetType().Name} instead"
                                        }
                                    }
                                };

                                batchAppendResp.CorrelationId    = Uuid.FromGuid(correlationId).ToDto();
                                batchAppendResp.StreamIdentifier = new StreamIdentifier {
                                    StreamName = ByteString.CopyFromUtf8(clientWriteRequest.StreamId)
                                };
                                return(batchAppendResp);
                            }
                        } catch (Exception ex) {
                            await writer.WriteAsync(new BatchAppendResp {
                                CorrelationId    = request.CorrelationId,
                                StreamIdentifier = request.Options.StreamIdentifier,
                                Error            = Status.BadRequest(ex.Message)
                            }, cancellationToken).ConfigureAwait(false);
                        }
                    }
Beispiel #19
0
        public override async Task <UpdateResp> Update(UpdateReq request, ServerCallContext context)
        {
            var updatePersistentSubscriptionSource = new TaskCompletionSource <UpdateResp>();
            var settings      = request.Options.Settings;
            var correlationId = Guid.NewGuid();

            var user = context.GetHttpContext().User;

            if (!await _authorizationProvider.CheckAccessAsync(user,
                                                               UpdateOperation, context.CancellationToken).ConfigureAwait(false))
            {
                throw AccessDenied();
            }

            string streamId = null;

            switch (request.Options.StreamOptionCase)
            {
            case StreamOptionOneofCase.Stream:
            case StreamOptionOneofCase.None:                     /*for backwards compatibility*/
            {
                StreamRevision startRevision;

                if (request.Options.StreamOptionCase == StreamOptionOneofCase.Stream)
                {
                    streamId      = request.Options.Stream.StreamIdentifier;
                    startRevision = request.Options.Stream.RevisionOptionCase switch {
                        RevisionOptionOneofCase.Revision => new StreamRevision(request.Options.Stream.Revision),
                        RevisionOptionOneofCase.Start => StreamRevision.Start,
                        RevisionOptionOneofCase.End => StreamRevision.End,
                        _ => throw new InvalidOperationException()
                    };
                }
                else                             /*for backwards compatibility*/
                {
                                                #pragma warning disable 612
                    streamId      = request.Options.StreamIdentifier;
                    startRevision = new StreamRevision(request.Options.Settings.Revision);
                                                #pragma warning restore 612
                }

                _publisher.Publish(new ClientMessage.UpdatePersistentSubscriptionToStream(
                                       correlationId,
                                       correlationId,
                                       new CallbackEnvelope(HandleUpdatePersistentSubscriptionCompleted),
                                       streamId,
                                       request.Options.GroupName,
                                       settings.ResolveLinks,
                                       startRevision.ToInt64(),
                                       settings.MessageTimeoutCase switch {
                        UpdateReq.Types.Settings.MessageTimeoutOneofCase.MessageTimeoutMs => settings.MessageTimeoutMs,
                        UpdateReq.Types.Settings.MessageTimeoutOneofCase.MessageTimeoutTicks => (int)TimeSpan
                        .FromTicks(settings.MessageTimeoutTicks).TotalMilliseconds,
                        _ => 0
                    },
                                       settings.ExtraStatistics,
                                       settings.MaxRetryCount,
                                       settings.HistoryBufferSize,
                                       settings.LiveBufferSize,
                                       settings.ReadBatchSize,
                                       settings.CheckpointAfterCase switch {
                        UpdateReq.Types.Settings.CheckpointAfterOneofCase.CheckpointAfterMs => settings.CheckpointAfterMs,
                        UpdateReq.Types.Settings.CheckpointAfterOneofCase.CheckpointAfterTicks => (int)TimeSpan
                        .FromTicks(settings.CheckpointAfterTicks).TotalMilliseconds,
                        _ => 0
                    },
Beispiel #20
0
            private void CatchUp(StreamRevision startRevision)
            {
                Log.Information(
                    "Catch-up subscription {subscriptionId} to {streamName}@{streamRevision} running...",
                    _subscriptionId, _streamName, startRevision);

                ReadPage(startRevision, OnMessage);

                async Task OnMessage(Message message, CancellationToken ct)
                {
                    if (message is ClientMessage.NotHandled notHandled &&
                        RpcExceptions.TryHandleNotHandled(notHandled, out var ex))
                    {
                        Fail(ex);
                        return;
                    }

                    if (!(message is ClientMessage.ReadStreamEventsForwardCompleted completed))
                    {
                        Fail(
                            RpcExceptions.UnknownMessage <ClientMessage.ReadStreamEventsForwardCompleted>(message));
                        return;
                    }

                    switch (completed.Result)
                    {
                    case ReadStreamResult.Success:
                        ConfirmSubscription();
                        foreach (var @event in completed.Events)
                        {
                            var streamRevision = StreamRevision.FromInt64(@event.OriginalEvent.EventNumber);

                            Log.Verbose(
                                "Catch-up subscription {subscriptionId} to {streamName} received event {streamRevision}.",
                                _subscriptionId, _streamName, streamRevision);

                            await _channel.Writer.WriteAsync(@event, ct).ConfigureAwait(false);
                        }

                        if (completed.IsEndOfStream)
                        {
                            GoLive(StreamRevision.FromInt64(completed.NextEventNumber));
                            return;
                        }

                        ReadPage(StreamRevision.FromInt64(completed.NextEventNumber), OnMessage);
                        return;

                    case ReadStreamResult.NoStream:
                        ConfirmSubscription();
                        await Task.Delay(TimeSpan.FromMilliseconds(50), ct).ConfigureAwait(false);

                        ReadPage(startRevision, OnMessage);
                        return;

                    case ReadStreamResult.StreamDeleted:
                        Fail(RpcExceptions.StreamDeleted(_streamName));
                        return;

                    case ReadStreamResult.AccessDenied:
                        Fail(RpcExceptions.AccessDenied());
                        return;

                    default:
                        Fail(RpcExceptions.UnknownError(completed.Result));
                        return;
                    }
                }
            }