/// <summary>
/// Subscribe to specific shard.
/// </summary>
/// <param name="db">The database.</param>
/// <param name="plan">The consumer plan.</param>
/// <param name="func">The function.</param>
/// <param name="options">The options.</param>
/// <param name="cancellationToken">The cancellation token.</param>
private async Task SubsribeShardAsync(
IDatabaseAsync db,
IConsumerPlan plan,
Func <Announcement, IAck, ValueTask> func,
IEventSourceConsumerOptions options,
CancellationToken cancellationToken)
{
string key = $"{plan.Partition}:{plan.Shard}";
bool isFirstBatchOrFailure = true;
CommandFlags flags = CommandFlags.None;
#region await db.CreateConsumerGroupIfNotExistsAsync(...)
await db.CreateConsumerGroupIfNotExistsAsync(
key,
plan.ConsumerGroup,
plan.Logger ?? _logger);
#endregion // await db.CreateConsumerGroupIfNotExistsAsync(...)
cancellationToken = CancellationTokenSource.CreateLinkedTokenSource(
cancellationToken, plan.Cancellation).Token;
TimeSpan delay = TimeSpan.Zero;
int emptyBatchCount = 0;
while (!cancellationToken.IsCancellationRequested)
{
await HandleBatchAsync();
}
#region HandleBatchAsync
// Handle single batch
async ValueTask HandleBatchAsync()
{
StreamEntry[] results = await ReadBatchAsync();
emptyBatchCount = results.Length == 0 ? emptyBatchCount + 1 : 0;
results = await ClaimStaleMessages(emptyBatchCount, results);
await DelayIfEmpty(results.Length);
if (results.Length == 0)
{
return;
}
try
{
var batchCancellation = new CancellationTokenSource();
for (int i = 0; i < results.Length && !batchCancellation.IsCancellationRequested; i++)
{
StreamEntry result = results[i];
#region var announcement = new Announcement(...)
var entries = result.Values.ToDictionary(m => m.Name, m => m.Value);
string id = entries[nameof(MetadataExtensions.Empty.MessageId)];
string segmentsKey = $"Segments~{id}";
string interceptorsKey = $"Interceptors~{id}";
string operation = entries[nameof(MetadataExtensions.Empty.Operation)];
long producedAtUnix = (long)entries[nameof(MetadataExtensions.Empty.ProducedAt)];
DateTimeOffset producedAt = DateTimeOffset.FromUnixTimeSeconds(producedAtUnix);
var meta = new Metadata
{
MessageId = id,
Partition = plan.Partition,
Shard = plan.Shard,
Operation = operation,
ProducedAt = producedAt
};
var segmentsEntities = await db.HashGetAllAsync(segmentsKey, CommandFlags.DemandMaster); // DemandMaster avoid racing
var segmentsPairs = segmentsEntities.Select(m => ((string)m.Name, (byte[])m.Value));
var interceptionsEntities = await db.HashGetAllAsync(interceptorsKey, CommandFlags.DemandMaster); // DemandMaster avoid racing
var interceptionsPairs = interceptionsEntities.Select(m => ((string)m.Name, (byte[])m.Value));
var segmets = Bucket.Empty.AddRange(segmentsPairs);
var interceptions = Bucket.Empty.AddRange(interceptionsPairs);
var announcement = new Announcement
{
Metadata = meta,
Segments = segmets,
InterceptorsData = interceptions
};
#endregion // var announcement = new Announcement(...)
int local = i;
var cancellableIds = results[local..].Select(m => m.Id);
var ack = new AckOnce(
() => AckAsync(result.Id),
plan.Options.AckBehavior, _logger,
async() =>
{
batchCancellation.CancelSafe(); // cancel forward
await CancelAsync(cancellableIds);
});
await func(announcement, ack);
}
