public async Task <long?> CountStreamRemainingEventsAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken = default)
{
try
{
var endCoordinates = await SeekToEndAsync(shardingSettings, cancellationToken).ConfigureAwait(false);
var distance = coordinates.DistanceTo(endCoordinates);
log.Debug(
"Stream remaining events: {Count}. Current coordinates: {CurrentCoordinates}, end coordinates: {EndCoordinates}.",
distance,
coordinates,
endCoordinates);
return(distance);
}
catch (Exception e)
{
log.Warn(e, "Failed to count remaining events.");
return(null);
}
}
// ReSharper disable once ParameterHidesMember
protected private async Task <StreamCoordinates> SeekToEndAsync(StreamShardingSettings shardingSettings)
{
var query = new SeekToEndStreamQuery(settings.StreamName)
{
ClientShard = shardingSettings.ClientShardIndex,
ClientShardCount = shardingSettings.ClientShardCount
};
SeekToEndStreamResult result;
using (var spanBuilder = tracer.BeginConsumerCustomOperationSpan("SeekToEnd"))
{
do
{
result = await client.SeekToEndAsync(query, settings.ApiKeyProvider(), settings.EventsReadTimeout).ConfigureAwait(false);
if (!result.IsSuccessful)
{
log.Warn(
"Failed to seek to end for Hercules stream '{StreamName}'. " +
"Status: {Status}. Error: '{Error}'.",
settings.StreamName,
result.Status,
result.ErrorDetails);
await DelayOnError().ConfigureAwait(false);
}
} while (!result.IsSuccessful);
spanBuilder.SetStream(query.Name);
spanBuilder.SetShard(query.ClientShard, query.ClientShardCount);
}
return(result.Payload.Next);
}
private async Task <StreamCoordinates> GetShardCoordinates(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken)
{
var(_, result) = await streamReader.ReadAsync(coordinates, shardingSettings, 1, cancellationToken).ConfigureAwait(false);
var map = result.Payload.Next.ToDictionary();
foreach (var position in coordinates.Positions)
{
if (map.ContainsKey(position.Partition))
{
map[position.Partition] = new StreamPosition
{
Partition = position.Partition,
Offset = position.Offset
}
}
;
}
return(new StreamCoordinates(map.Values.ToArray()));
}
}
public async Task <(ReadStreamQuery query, ReadStreamResult result)> ReadAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
long additionalLimit,
CancellationToken cancellationToken)
{
var(query, result) = await reader.ReadAsync(coordinates, shardingSettings, additionalLimit, cancellationToken).ConfigureAwait(false);
return(query, result.FromGenericResult());
}
public async Task <StreamCoordinates> SeekToEndAsync(
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken = default)
{
var seekToEndQuery = new SeekToEndStreamQuery(settings.StreamName)
{
ClientShard = shardingSettings.ClientShardIndex,
ClientShardCount = shardingSettings.ClientShardCount
};
var end = await settings.StreamClient.SeekToEndAsync(seekToEndQuery, settings.EventsReadTimeout, cancellationToken).ConfigureAwait(false);
return(end.Payload.Next);
}
public async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
var newShardingSettings = settings.ShardingSettingsProvider();
if (shardingSettings == null || !shardingSettings.Equals(newShardingSettings))
{
shardingSettings = newShardingSettings;
restart = true;
}
if (restart)
{
await Restart().ConfigureAwait(false);
restart = false;
}
using (new OperationContextToken($"Iteration-{iteration++}"))
using (tracer.BeginConsumerCustomOperationSpan("Iteration"))
using (iterationMetric?.For("time").Measure())
{
await MakeIteration().ConfigureAwait(false);
}
}
catch (Exception error)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
log.Error(error, "Failed to consume batch.");
await DelayOnError().ConfigureAwait(false);
}
}
if (coordinates != null)
{
await settings.CoordinatesStorage.AdvanceAsync(coordinates).ConfigureAwait(false);
}
log.Info("Final coordinates: {StreamCoordinates}.", coordinates);
settings.OnStop?.Invoke(coordinates);
}
public async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
try
{
// (iloktionov): Catch-up with state for other shards on any change to our sharding settings:
var newShardingSettings = settings.ShardingSettingsProvider();
if (shardingSettings == null || !shardingSettings.Equals(newShardingSettings))
{
log.Info(
"Observed new sharding settings: shard with index {ShardIndex} from {ShardCount}. Syncing coordinates.",
newShardingSettings.ClientShardIndex,
newShardingSettings.ClientShardCount);
shardingSettings = newShardingSettings;
restart = true;
}
if (restart)
{
await Restart(cancellationToken).ConfigureAwait(false);
restart = false;
}
using (iterationMetric?.For("time").Measure())
{
await MakeIteration(cancellationToken).ConfigureAwait(false);
}
}
catch (Exception error)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
log.Error(error, "Failed to consume stream.");
await Task.Delay(settings.DelayOnError, cancellationToken).SilentlyContinue().ConfigureAwait(false);
}
}
}
public async Task <(ReadStreamQuery query, ReadStreamResult <T> result)> ReadAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
long additionalLimit,
CancellationToken cancellationToken = default)
{
log.Info(
"Reading logical shard with index {ClientShard} from {ClientShardCount}.",
shardingSettings.ClientShardIndex,
shardingSettings.ClientShardCount);
log.Debug("Current coordinates: {StreamCoordinates}.", coordinates);
var eventsQuery = new ReadStreamQuery(settings.StreamName)
{
Coordinates = coordinates,
ClientShard = shardingSettings.ClientShardIndex,
ClientShardCount = shardingSettings.ClientShardCount,
Limit = (int)Math.Min(settings.EventsReadBatchSize, additionalLimit)
};
ReadStreamResult <T> readResult;
var remainingAttempts = settings.EventsReadAttempts;
do
{
remainingAttempts--;
readResult = await settings.StreamClient.ReadAsync(eventsQuery, settings.EventsReadTimeout, cancellationToken).ConfigureAwait(false);
if (!readResult.IsSuccessful && !cancellationToken.IsCancellationRequested)
{
log.Warn(
"Failed to read events from Hercules stream '{StreamName}'. " +
"Status: {Status}. Error: '{Error}'. " +
"Will try again {RemainingAttempts} more times.",
settings.StreamName,
readResult.Status,
readResult.ErrorDetails,
remainingAttempts);
if (remainingAttempts > 0)
{
await Task.Delay(settings.DelayOnError, cancellationToken).SilentlyContinue().ConfigureAwait(false);
}
}
} while (!cancellationToken.IsCancellationRequested && !readResult.IsSuccessful && remainingAttempts > 0);
if (readResult.IsSuccessful)
{
log.Info(
"Read {EventsCount} event(s) from Hercules stream '{StreamName}'.",
readResult.Payload.Events.Count,
settings.StreamName);
}
else if (cancellationToken.IsCancellationRequested)
{
log.Info("Cancelled read events request from Hercules stream '{StreamName}'.",
settings.StreamName);
}
else
{
log.Error(
"Failed to read events from Hercules stream '{StreamName}'. " +
"Status: {Status}. Error: '{Error}'.",
settings.StreamName,
readResult.Status,
readResult.ErrorDetails);
}
return(eventsQuery, readResult);
}
public Task <(ReadStreamQuery query, ReadStreamResult <T> result)> ReadAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken = default) =>
public async Task <(ReadStreamQuery query, ReadStreamResult <T> result)> ReadAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken)
{
log.Info(
"Reading logical shard with index {ClientShard} from {ClientShardCount}.",
shardingSettings.ClientShardIndex,
shardingSettings.ClientShardCount);
log.Debug("Current coordinates: {StreamCoordinates}.", coordinates);
coordinates = await GetShardCoordinates(coordinates, shardingSettings, cancellationToken).ConfigureAwait(false);
log.Debug("Current shard coordinates: {StreamCoordinates}.", coordinates);
streamPartitionsCount = streamPartitionsCount ?? await GetPartitionsCount(cancellationToken).ConfigureAwait(false);
var current = coordinates.ToDictionary();
foreach (var partition in coordinates.Positions.Select(p => p.Partition))
{
var start = current.ContainsKey(partition) ? current[partition].Offset : 0;
var end = settings.End.ContainsKey(partition) ? settings.End[partition].Offset : 0;
if (start < end)
{
var count = end - start;
log.Info("Reading {EventsCount} events from partition #{Partition}.", count, partition);
var(query, result) = await streamReader.ReadAsync(
coordinates,
// ReSharper disable once PossibleInvalidOperationException
new StreamShardingSettings(partition, streamPartitionsCount.Value),
count,
cancellationToken)
.ConfigureAwait(false);
result.EnsureSuccess();
result = new ReadStreamResult <T>(
result.Status,
new ReadStreamPayload <T>(
result.Payload.Events,
coordinates.SetPosition(result.Payload.Next.Positions.Single())),
result.ErrorDetails);
query = new ReadStreamQuery(query.Name)
{
Coordinates = coordinates,
ClientShard = shardingSettings.ClientShardIndex,
ClientShardCount = shardingSettings.ClientShardCount
};
return(query, result);
}
}
return(null, null);
}
public Task <long?> CountStreamRemainingEventsAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken = default) =>
reader.CountStreamRemainingEventsAsync(coordinates, shardingSettings, cancellationToken);
public Task <StreamCoordinates> SeekToEndAsync(
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken = default) =>
reader.SeekToEndAsync(shardingSettings, cancellationToken);