public async Task RunAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { try { var newShardingSettings = settings.ShardingSettingsProvider(); if (shardingSettings == null || !shardingSettings.Equals(newShardingSettings)) { shardingSettings = newShardingSettings; restart = true; } if (restart) { await Restart().ConfigureAwait(false); restart = false; } using (new OperationContextToken($"Iteration-{iteration++}")) using (tracer.BeginConsumerCustomOperationSpan("Iteration")) using (iterationMetric?.For("time").Measure()) { await MakeIteration().ConfigureAwait(false); } } catch (Exception error) { if (cancellationToken.IsCancellationRequested) { return; } log.Error(error, "Failed to consume batch."); await DelayOnError().ConfigureAwait(false); } } if (coordinates != null) { await settings.CoordinatesStorage.AdvanceAsync(coordinates).ConfigureAwait(false); } log.Info("Final coordinates: {StreamCoordinates}.", coordinates); settings.OnStop?.Invoke(coordinates); }
private void FlushWindows() { using (new OperationContextToken("FlushEvents")) using (tracer.BeginConsumerCustomOperationSpan("FlushEvents")) using (iterationMetric?.For("flush_time").Measure()) { var result = new WindowsFlushResult(); var stale = new List <TKey>(); foreach (var pair in windows) { var flushResult = pair.Value.Flush(); result.MergeWith(flushResult); if (flushResult.EventsCount == 0 && DateTimeOffset.UtcNow - pair.Value.LastEventAdded > 1.Hours()) { stale.Add(pair.Key); } } foreach (var s in stale) { windows.Remove(s); } if (result.EventsCount > 0) { leftCoordinates = result.FirstEventCoordinates; } log.Info( "Consumer status: keys: {KeysCount}, windows: {WindowsCount}, events: {EventsCount}.", windows.Count, result.WindowsCount, result.EventsCount); stateMetric?.For("keys").Set(windows.Count); stateMetric?.For("windows").Set(result.WindowsCount); stateMetric?.For("events").Set(result.EventsCount); } }
public async Task WriteAsync(string streamName, ArraySegment <byte> bytes, int eventsCount) { if (eventsCount == 0) { LogProgress(streamName, 0); return; } using (new OperationContextToken("WriteEvents")) using (var traceBuilder = tracer.BeginConsumerCustomOperationSpan("Write")) using (iterationMetric?.For("write_time").Measure()) { traceBuilder.SetOperationDetails(eventsCount); traceBuilder.SetStream(streamName); InsertEventsResult result; do { result = await client.SendAsync( streamName, settings.ApiKeyProvider(), new ValueDisposable <Content>(new Content(bytes), new EmptyDisposable()), settings.EventsWriteTimeout, CancellationToken.None) .ConfigureAwait(false); if (!result.IsSuccessful) { log.Warn( "Failed to write events to Hercules stream '{StreamName}'. " + "Status: {Status}. Error: '{Error}'.", streamName, result.Status, result.ErrorDetails); await DelayOnError().ConfigureAwait(false); } } while (!result.IsSuccessful); LogProgress(streamName, eventsCount); } }