/// <summary> /// Attach configuration. /// </summary> /// <param name="options"></param> /// <returns></returns> IConsumerHooksBuilder IConsumerOptionsBuilder.WithOptions(IEventSourceConsumerOptions options) { var prms = _plan.WithOptions(options); var result = new ConsumerBuilder(prms); return(result); }
/// <summary> /// Subscribe to the channel for specific metadata. /// </summary> /// <param name="plan">The consumer plan.</param> /// <param name="func">The function.</param> /// <param name="options">The options.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns> /// When completed /// </returns> /// <exception cref="NotSupportedException">Channel must be define</exception> public ValueTask SubsribeAsync( IConsumerPlan plan, Func <Announcement, IAck, ValueTask> func, IEventSourceConsumerOptions options, CancellationToken cancellationToken) { throw new NotSupportedException("Channel must be define"); }
/// <summary> /// Subscribe to all shards under a partition. /// </summary> /// <param name="db">The database.</param> /// <param name="plan">The consumer plan.</param> /// <param name="func">The function.</param> /// <param name="options">The options.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns> /// When completed /// </returns> private async ValueTask SubsribePartitionAsync( IDatabaseAsync db, IConsumerPlan plan, Func <Announcement, IAck, ValueTask> func, IEventSourceConsumerOptions options, CancellationToken cancellationToken) { var subscriptions = new Queue <Task>(); int delay = 1; string partition = plan.Partition; int partitionSplit = partition.Length + 1; while (!cancellationToken.IsCancellationRequested) { // loop for error cases try { // infinite until cancellation var keys = _redisClientFactory.GetKeysUnsafeAsync( pattern: $"{partition}:*") .WithCancellation(cancellationToken); // TODO: [bnaya 2020-10] seem like memory leak, re-subscribe to same shard await foreach (string key in keys) { string shard = key.Substring(partitionSplit); IConsumerPlan p = plan.WithShard(shard); Task subscription = SubsribeShardAsync(db, plan, func, options, cancellationToken); subscriptions.Enqueue(subscription); } break; } catch (Exception ex) { plan.Logger.LogError(ex, "Partition subscription"); await DelayIfRetry(); } } await Task.WhenAll(subscriptions); #region DelayIfRetry async Task DelayIfRetry() { await Task.Delay(delay, cancellationToken); delay *= Max(delay, 2); delay = Min(MAX_DELAY, delay); } #endregion // DelayIfRetry }
/// <summary> /// Subscribe to the channel for specific metadata. /// </summary> /// <param name="metadata">The metadata.</param> /// <param name="func">The function.</param> /// <param name="options">The options.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>When completed</returns> public async ValueTask SubsribeAsync( IConsumerPlan metadata, Func <Announcement, IAck, ValueTask> func, IEventSourceConsumerOptions options, CancellationToken cancellationToken) { while (!_channel.Reader.Completion.IsCompleted && !cancellationToken.IsCancellationRequested) { try { var announcement = await _channel.Reader.ReadAsync(cancellationToken); await func(announcement, Ack.Empty); } catch (ChannelClosedException) { } } }
/// <summary> /// Subscribe to the channel for specific metadata. /// </summary> /// <param name="plan">The consumer plan.</param> /// <param name="func">The function.</param> /// <param name="options">The options.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns> /// When completed /// </returns> public async ValueTask SubsribeAsync( IConsumerPlan plan, Func <Announcement, IAck, ValueTask> func, IEventSourceConsumerOptions options, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { // connection errors IDatabaseAsync db = await _redisClientFactory.GetDbAsync(); if (plan.Shard != string.Empty) { await SubsribeShardAsync(db, plan, func, options, cancellationToken); } else { await SubsribePartitionAsync(db, plan, func, options, cancellationToken); } } }
/// <summary> /// Initializes a new instance. /// </summary> /// <param name="plan">The plan.</param> /// <param name="factory">The factory.</param> public Subscription( IConsumerPlan plan, Func <ConsumerMetadata, T> factory) { var channel = plan.Channel; _plan = plan; _factory = factory; if (plan.Options.KeepAlive) { _keepAlive.TryAdd(this, null); } _options = _plan.Options; _maxMessages = _options.MaxMessages; _subscriptionLifetime = channel.SubsribeAsync( plan, ConsumingAsync, plan.Options, _cancellation.Token); }
/// <summary> /// Attach the options. /// </summary> /// <param name="options">The options.</param> /// <returns></returns> internal ConsumerPlan WithOptions(IEventSourceConsumerOptions options) { return(new ConsumerPlan(this, options: options)); }
/// <summary> /// Subscribe to specific shard. /// </summary> /// <param name="db">The database.</param> /// <param name="plan">The consumer plan.</param> /// <param name="func">The function.</param> /// <param name="options">The options.</param> /// <param name="cancellationToken">The cancellation token.</param> private async Task SubsribeShardAsync( IDatabaseAsync db, IConsumerPlan plan, Func <Announcement, IAck, ValueTask> func, IEventSourceConsumerOptions options, CancellationToken cancellationToken) { string key = $"{plan.Partition}:{plan.Shard}"; bool isFirstBatchOrFailure = true; CommandFlags flags = CommandFlags.None; #region await db.CreateConsumerGroupIfNotExistsAsync(...) await db.CreateConsumerGroupIfNotExistsAsync( key, plan.ConsumerGroup, plan.Logger ?? _logger); #endregion // await db.CreateConsumerGroupIfNotExistsAsync(...) cancellationToken = CancellationTokenSource.CreateLinkedTokenSource( cancellationToken, plan.Cancellation).Token; TimeSpan delay = TimeSpan.Zero; int emptyBatchCount = 0; while (!cancellationToken.IsCancellationRequested) { await HandleBatchAsync(); } #region HandleBatchAsync // Handle single batch async ValueTask HandleBatchAsync() { StreamEntry[] results = await ReadBatchAsync(); emptyBatchCount = results.Length == 0 ? emptyBatchCount + 1 : 0; results = await ClaimStaleMessages(emptyBatchCount, results); await DelayIfEmpty(results.Length); if (results.Length == 0) { return; } try { var batchCancellation = new CancellationTokenSource(); for (int i = 0; i < results.Length && !batchCancellation.IsCancellationRequested; i++) { StreamEntry result = results[i]; #region var announcement = new Announcement(...) var entries = result.Values.ToDictionary(m => m.Name, m => m.Value); string id = entries[nameof(MetadataExtensions.Empty.MessageId)]; string segmentsKey = $"Segments~{id}"; string interceptorsKey = $"Interceptors~{id}"; string operation = entries[nameof(MetadataExtensions.Empty.Operation)]; long producedAtUnix = (long)entries[nameof(MetadataExtensions.Empty.ProducedAt)]; DateTimeOffset producedAt = DateTimeOffset.FromUnixTimeSeconds(producedAtUnix); var meta = new Metadata { MessageId = id, Partition = plan.Partition, Shard = plan.Shard, Operation = operation, ProducedAt = producedAt }; var segmentsEntities = await db.HashGetAllAsync(segmentsKey, CommandFlags.DemandMaster); // DemandMaster avoid racing var segmentsPairs = segmentsEntities.Select(m => ((string)m.Name, (byte[])m.Value)); var interceptionsEntities = await db.HashGetAllAsync(interceptorsKey, CommandFlags.DemandMaster); // DemandMaster avoid racing var interceptionsPairs = interceptionsEntities.Select(m => ((string)m.Name, (byte[])m.Value)); var segmets = Bucket.Empty.AddRange(segmentsPairs); var interceptions = Bucket.Empty.AddRange(interceptionsPairs); var announcement = new Announcement { Metadata = meta, Segments = segmets, InterceptorsData = interceptions }; #endregion // var announcement = new Announcement(...) int local = i; var cancellableIds = results[local..].Select(m => m.Id); var ack = new AckOnce( () => AckAsync(result.Id), plan.Options.AckBehavior, _logger, async() => { batchCancellation.CancelSafe(); // cancel forward await CancelAsync(cancellableIds); }); await func(announcement, ack); }