コード例 #1
0
        public async Task <long?> CountStreamRemainingEventsAsync(
            StreamCoordinates coordinates,
            StreamShardingSettings shardingSettings,
            CancellationToken cancellationToken = default)
        {
            try
            {
                var endCoordinates = await SeekToEndAsync(shardingSettings, cancellationToken).ConfigureAwait(false);

                var distance = coordinates.DistanceTo(endCoordinates);

                log.Debug(
                    "Stream remaining events: {Count}. Current coordinates: {CurrentCoordinates}, end coordinates: {EndCoordinates}.",
                    distance,
                    coordinates,
                    endCoordinates);

                return(distance);
            }
            catch (Exception e)
            {
                log.Warn(e, "Failed to count remaining events.");
                return(null);
            }
        }
コード例 #2
0
        // ReSharper disable once ParameterHidesMember
        protected private async Task <StreamCoordinates> SeekToEndAsync(StreamShardingSettings shardingSettings)
        {
            var query = new SeekToEndStreamQuery(settings.StreamName)
            {
                ClientShard      = shardingSettings.ClientShardIndex,
                ClientShardCount = shardingSettings.ClientShardCount
            };

            SeekToEndStreamResult result;

            using (var spanBuilder = tracer.BeginConsumerCustomOperationSpan("SeekToEnd"))
            {
                do
                {
                    result = await client.SeekToEndAsync(query, settings.ApiKeyProvider(), settings.EventsReadTimeout).ConfigureAwait(false);

                    if (!result.IsSuccessful)
                    {
                        log.Warn(
                            "Failed to seek to end for Hercules stream '{StreamName}'. " +
                            "Status: {Status}. Error: '{Error}'.",
                            settings.StreamName,
                            result.Status,
                            result.ErrorDetails);
                        await DelayOnError().ConfigureAwait(false);
                    }
                } while (!result.IsSuccessful);

                spanBuilder.SetStream(query.Name);
                spanBuilder.SetShard(query.ClientShard, query.ClientShardCount);
            }

            return(result.Payload.Next);
        }
コード例 #3
0
        private async Task <StreamCoordinates> GetShardCoordinates(
            StreamCoordinates coordinates,
            StreamShardingSettings shardingSettings,
            CancellationToken cancellationToken)
        {
            var(_, result) = await streamReader.ReadAsync(coordinates, shardingSettings, 1, cancellationToken).ConfigureAwait(false);

            var map = result.Payload.Next.ToDictionary();

            foreach (var position in coordinates.Positions)
            {
                if (map.ContainsKey(position.Partition))
                {
                    map[position.Partition] = new StreamPosition
                    {
                        Partition = position.Partition,
                        Offset    = position.Offset
                    }
                }
                ;
            }

            return(new StreamCoordinates(map.Values.ToArray()));
        }
    }
コード例 #4
0
        public async Task <(ReadStreamQuery query, ReadStreamResult result)> ReadAsync(
            StreamCoordinates coordinates,
            StreamShardingSettings shardingSettings,
            long additionalLimit,
            CancellationToken cancellationToken)
        {
            var(query, result) = await reader.ReadAsync(coordinates, shardingSettings, additionalLimit, cancellationToken).ConfigureAwait(false);

            return(query, result.FromGenericResult());
        }
コード例 #5
0
        public async Task <StreamCoordinates> SeekToEndAsync(
            StreamShardingSettings shardingSettings,
            CancellationToken cancellationToken = default)
        {
            var seekToEndQuery = new SeekToEndStreamQuery(settings.StreamName)
            {
                ClientShard      = shardingSettings.ClientShardIndex,
                ClientShardCount = shardingSettings.ClientShardCount
            };

            var end = await settings.StreamClient.SeekToEndAsync(seekToEndQuery, settings.EventsReadTimeout, cancellationToken).ConfigureAwait(false);

            return(end.Payload.Next);
        }
コード例 #6
0
        public async Task RunAsync(CancellationToken cancellationToken)
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                try
                {
                    var newShardingSettings = settings.ShardingSettingsProvider();
                    if (shardingSettings == null || !shardingSettings.Equals(newShardingSettings))
                    {
                        shardingSettings = newShardingSettings;
                        restart          = true;
                    }

                    if (restart)
                    {
                        await Restart().ConfigureAwait(false);

                        restart = false;
                    }

                    using (new OperationContextToken($"Iteration-{iteration++}"))
                        using (tracer.BeginConsumerCustomOperationSpan("Iteration"))
                            using (iterationMetric?.For("time").Measure())
                            {
                                await MakeIteration().ConfigureAwait(false);
                            }
                }
                catch (Exception error)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        return;
                    }

                    log.Error(error, "Failed to consume batch.");

                    await DelayOnError().ConfigureAwait(false);
                }
            }

            if (coordinates != null)
            {
                await settings.CoordinatesStorage.AdvanceAsync(coordinates).ConfigureAwait(false);
            }
            log.Info("Final coordinates: {StreamCoordinates}.", coordinates);
            settings.OnStop?.Invoke(coordinates);
        }
コード例 #7
0
        public async Task RunAsync(CancellationToken cancellationToken)
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                try
                {
                    // (iloktionov): Catch-up with state for other shards on any change to our sharding settings:
                    var newShardingSettings = settings.ShardingSettingsProvider();
                    if (shardingSettings == null || !shardingSettings.Equals(newShardingSettings))
                    {
                        log.Info(
                            "Observed new sharding settings: shard with index {ShardIndex} from {ShardCount}. Syncing coordinates.",
                            newShardingSettings.ClientShardIndex,
                            newShardingSettings.ClientShardCount);

                        shardingSettings = newShardingSettings;

                        restart = true;
                    }

                    if (restart)
                    {
                        await Restart(cancellationToken).ConfigureAwait(false);

                        restart = false;
                    }

                    using (iterationMetric?.For("time").Measure())
                    {
                        await MakeIteration(cancellationToken).ConfigureAwait(false);
                    }
                }
                catch (Exception error)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        return;
                    }

                    log.Error(error, "Failed to consume stream.");

                    await Task.Delay(settings.DelayOnError, cancellationToken).SilentlyContinue().ConfigureAwait(false);
                }
            }
        }
コード例 #8
0
        public async Task <(ReadStreamQuery query, ReadStreamResult <T> result)> ReadAsync(
            StreamCoordinates coordinates,
            StreamShardingSettings shardingSettings,
            long additionalLimit,
            CancellationToken cancellationToken = default)
        {
            log.Info(
                "Reading logical shard with index {ClientShard} from {ClientShardCount}.",
                shardingSettings.ClientShardIndex,
                shardingSettings.ClientShardCount);

            log.Debug("Current coordinates: {StreamCoordinates}.", coordinates);

            var eventsQuery = new ReadStreamQuery(settings.StreamName)
            {
                Coordinates      = coordinates,
                ClientShard      = shardingSettings.ClientShardIndex,
                ClientShardCount = shardingSettings.ClientShardCount,
                Limit            = (int)Math.Min(settings.EventsReadBatchSize, additionalLimit)
            };

            ReadStreamResult <T> readResult;
            var remainingAttempts = settings.EventsReadAttempts;

            do
            {
                remainingAttempts--;
                readResult = await settings.StreamClient.ReadAsync(eventsQuery, settings.EventsReadTimeout, cancellationToken).ConfigureAwait(false);

                if (!readResult.IsSuccessful && !cancellationToken.IsCancellationRequested)
                {
                    log.Warn(
                        "Failed to read events from Hercules stream '{StreamName}'. " +
                        "Status: {Status}. Error: '{Error}'. " +
                        "Will try again {RemainingAttempts} more times.",
                        settings.StreamName,
                        readResult.Status,
                        readResult.ErrorDetails,
                        remainingAttempts);
                    if (remainingAttempts > 0)
                    {
                        await Task.Delay(settings.DelayOnError, cancellationToken).SilentlyContinue().ConfigureAwait(false);
                    }
                }
            } while (!cancellationToken.IsCancellationRequested && !readResult.IsSuccessful && remainingAttempts > 0);

            if (readResult.IsSuccessful)
            {
                log.Info(
                    "Read {EventsCount} event(s) from Hercules stream '{StreamName}'.",
                    readResult.Payload.Events.Count,
                    settings.StreamName);
            }
            else if (cancellationToken.IsCancellationRequested)
            {
                log.Info("Cancelled read events request from Hercules stream '{StreamName}'.",
                         settings.StreamName);
            }
            else
            {
                log.Error(
                    "Failed to read events from Hercules stream '{StreamName}'. " +
                    "Status: {Status}. Error: '{Error}'.",
                    settings.StreamName,
                    readResult.Status,
                    readResult.ErrorDetails);
            }

            return(eventsQuery, readResult);
        }
コード例 #9
0
 public Task <(ReadStreamQuery query, ReadStreamResult <T> result)> ReadAsync(
     StreamCoordinates coordinates,
     StreamShardingSettings shardingSettings,
     CancellationToken cancellationToken = default) =>
コード例 #10
0
        public async Task <(ReadStreamQuery query, ReadStreamResult <T> result)> ReadAsync(
            StreamCoordinates coordinates,
            StreamShardingSettings shardingSettings,
            CancellationToken cancellationToken)
        {
            log.Info(
                "Reading logical shard with index {ClientShard} from {ClientShardCount}.",
                shardingSettings.ClientShardIndex,
                shardingSettings.ClientShardCount);

            log.Debug("Current coordinates: {StreamCoordinates}.", coordinates);

            coordinates = await GetShardCoordinates(coordinates, shardingSettings, cancellationToken).ConfigureAwait(false);

            log.Debug("Current shard coordinates: {StreamCoordinates}.", coordinates);

            streamPartitionsCount = streamPartitionsCount ?? await GetPartitionsCount(cancellationToken).ConfigureAwait(false);

            var current = coordinates.ToDictionary();

            foreach (var partition in coordinates.Positions.Select(p => p.Partition))
            {
                var start = current.ContainsKey(partition) ? current[partition].Offset : 0;
                var end   = settings.End.ContainsKey(partition) ? settings.End[partition].Offset : 0;

                if (start < end)
                {
                    var count = end - start;

                    log.Info("Reading {EventsCount} events from partition #{Partition}.", count, partition);

                    var(query, result) = await streamReader.ReadAsync(
                        coordinates,
                        // ReSharper disable once PossibleInvalidOperationException
                        new StreamShardingSettings(partition, streamPartitionsCount.Value),
                        count,
                        cancellationToken)
                                         .ConfigureAwait(false);

                    result.EnsureSuccess();

                    result = new ReadStreamResult <T>(
                        result.Status,
                        new ReadStreamPayload <T>(
                            result.Payload.Events,
                            coordinates.SetPosition(result.Payload.Next.Positions.Single())),
                        result.ErrorDetails);

                    query = new ReadStreamQuery(query.Name)
                    {
                        Coordinates      = coordinates,
                        ClientShard      = shardingSettings.ClientShardIndex,
                        ClientShardCount = shardingSettings.ClientShardCount
                    };

                    return(query, result);
                }
            }

            return(null, null);
        }
コード例 #11
0
 public Task <long?> CountStreamRemainingEventsAsync(
     StreamCoordinates coordinates,
     StreamShardingSettings shardingSettings,
     CancellationToken cancellationToken = default) =>
 reader.CountStreamRemainingEventsAsync(coordinates, shardingSettings, cancellationToken);
コード例 #12
0
 public Task <StreamCoordinates> SeekToEndAsync(
     StreamShardingSettings shardingSettings,
     CancellationToken cancellationToken = default) =>
 reader.SeekToEndAsync(shardingSettings, cancellationToken);