public static StreamCoordinates SetPosition(this StreamCoordinates coordinates, StreamPosition position) { var dict = coordinates.ToDictionary(); dict[position.Partition] = position; return(new StreamCoordinates(dict.Values.ToArray())); }
public bool AddEvent(T @event, StreamCoordinates coordinates) { var now = DateTimeOffset.Now; var timestamp = settings.TimestampProvider(@event); if (!timestamp.InInterval(now - settings.MaximumDeltaBeforeNow, now + settings.MaximumDeltaAfterNow)) { return(false); } if (timestamp < minimumAllowedTimestamp) { return(false); } if (maximumObservedTimestamp < timestamp) { maximumObservedTimestamp = timestamp; } foreach (var window in windows) { if (window.AddEvent(@event, timestamp)) { return(true); } } var newWindow = CreateWindow(@event, timestamp, coordinates); newWindow.AddEvent(@event, timestamp); windows.Add(newWindow); LastEventAdded = now.UtcDateTime; return(true); }
private async Task RestartCoordinates() { LogShardingSettings(); if (coordinates != null) { await settings.CoordinatesStorage.AdvanceAsync(coordinates).ConfigureAwait(false); } LogCoordinates("Current", coordinates); var endCoordinates = await SeekToEndAsync(shardingSettings).ConfigureAwait(false); LogCoordinates("End", endCoordinates); var storageCoordinates = await settings.CoordinatesStorage.GetCurrentAsync().ConfigureAwait(false); storageCoordinates = storageCoordinates.FilterBy(endCoordinates); LogCoordinates("Storage", storageCoordinates); if (storageCoordinates.Positions.Length < endCoordinates.Positions.Length) { log.Info("Some coordinates are missing. Returning end coordinates."); coordinates = endCoordinates; return; } log.Info("Returning storage coordinates."); coordinates = storageCoordinates; }
public async Task <long?> CountStreamRemainingEventsAsync( StreamCoordinates coordinates, StreamShardingSettings shardingSettings, CancellationToken cancellationToken = default) { try { var endCoordinates = await SeekToEndAsync(shardingSettings, cancellationToken).ConfigureAwait(false); var distance = coordinates.DistanceTo(endCoordinates); log.Debug( "Stream remaining events: {Count}. Current coordinates: {CurrentCoordinates}, end coordinates: {EndCoordinates}.", distance, coordinates, endCoordinates); return(distance); } catch (Exception e) { log.Warn(e, "Failed to count remaining events."); return(null); } }
public ReadStreamPayload( [NotNull] IList <T> events, [NotNull] StreamCoordinates next) { Events = events ?? throw new ArgumentNullException(nameof(events)); Next = next ?? throw new ArgumentNullException(nameof(next)); }
public void Distance_should_work_correctly() { var a = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 100 } }); var b = new StreamCoordinates( new[] { new StreamPosition { Partition = 1, Offset = 200 }, new StreamPosition { Partition = 2, Offset = 2 } }); a.DistanceTo(b).Should().Be(102); b.DistanceTo(a).Should().Be(-99); }
public void AdvancesOver_should_be_true_if_some_coordinates_added() { var a = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 }, new StreamPosition { Partition = 2, Offset = 1 } }); var b = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 } }); a.AdvancesOver(b).Should().BeTrue(); b.AdvancesOver(a).Should().BeFalse(); }
private async Task <StreamCoordinates> GetShardCoordinates( StreamCoordinates coordinates, StreamShardingSettings shardingSettings, CancellationToken cancellationToken) { var(_, result) = await streamReader.ReadAsync(coordinates, shardingSettings, 1, cancellationToken).ConfigureAwait(false); var map = result.Payload.Next.ToDictionary(); foreach (var position in coordinates.Positions) { if (map.ContainsKey(position.Partition)) { map[position.Partition] = new StreamPosition { Partition = position.Partition, Offset = position.Offset } } ; } return(new StreamCoordinates(map.Values.ToArray())); } }
private async Task MakeIteration(CancellationToken cancellationToken) { ReadStreamQuery query; ReadStreamResult <T> result; using (iterationMetric?.For("read_time").Measure()) { (query, result) = await streamReader.ReadAsync(coordinates, shardingSettings, cancellationToken).ConfigureAwait(false); } var events = result.Payload.Events; LogProgress(events.Count); if (events.Count != 0) { using (iterationMetric?.For("handle_time").Measure()) { await settings.EventsHandler.HandleAsync(query, result, cancellationToken).ConfigureAwait(false); } } coordinates = result.Payload.Next; if (events.Count == 0) { await Task.Delay(settings.DelayOnNoEvents, cancellationToken).ConfigureAwait(false); } Task.Run(() => settings.CoordinatesStorage.AdvanceAsync(coordinates)); }
public void MergeMin_should_work_correctly() { var a = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 }, new StreamPosition { Partition = 2, Offset = 3 }, new StreamPosition { Partition = 3, Offset = 4 } }); var b = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 2 }, new StreamPosition { Partition = 2, Offset = 2 }, new StreamPosition { Partition = 5, Offset = 4 } }); var min = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 }, new StreamPosition { Partition = 2, Offset = 2 }, new StreamPosition { Partition = 3, Offset = 4 }, new StreamPosition { Partition = 5, Offset = 4 } }); a.MergeMinWith(b).Positions.Should().BeEquivalentTo(min.Positions); b.MergeMinWith(a).Positions.Should().BeEquivalentTo(min.Positions); }
private async Task Stop(StreamCoordinates rightCoordinates) { if (leftCoordinates != null) { await settings.LeftCoordinatesStorage.AdvanceAsync(leftCoordinates).ConfigureAwait(false); } LogCoordinates("Stop", leftCoordinates, rightCoordinates); }
internal Window(WindowedStreamConsumerSettings <T, TKey> .IWindow implementation, StreamCoordinates firstEventCoordinates, DateTimeOffset start, DateTimeOffset end, TimeSpan period, TimeSpan lag) { this.implementation = implementation; FirstEventCoordinates = firstEventCoordinates; Start = start; End = end; this.period = period; this.lag = lag; lastEventAdded = DateTimeOffset.Now; }
private Window <T, TKey> CreateWindow(T @event, DateTimeOffset timestamp, StreamCoordinates coordinates) { var period = TimeSpanArithmetics.Min(settings.PeriodProvider?.Invoke(@event) ?? settings.Period, settings.MaximumAllowedPeriod); var lag = TimeSpanArithmetics.Min(settings.LagProvider?.Invoke(@event) ?? settings.Lag, settings.MaximumAllowedLag); var start = timestamp.AddTicks(-timestamp.Ticks % period.Ticks); var result = new Window <T, TKey>(settings.CreateWindow(key), coordinates, start, start + period, period, lag); return(result); }
public async Task <(ReadStreamQuery query, ReadStreamResult result)> ReadAsync( StreamCoordinates coordinates, StreamShardingSettings shardingSettings, long additionalLimit, CancellationToken cancellationToken) { var(query, result) = await reader.ReadAsync(coordinates, shardingSettings, additionalLimit, cancellationToken).ConfigureAwait(false); return(query, result.FromGenericResult()); }
private static ArraySegment <byte> CreateRequestBody([NotNull] StreamCoordinates coordinates) { var writer = new BinaryBufferWriter(sizeof(int) + coordinates.Positions.Length * (sizeof(int) + sizeof(long))) { Endianness = Endianness.Big }; StreamCoordinatesWriter.Write(coordinates, writer); return(writer.FilledSegment); }
public StreamSegmentReaderSettings( [NotNull] string streamName, [NotNull] IHerculesStreamClient <T> streamClient, [NotNull] StreamCoordinates start, [NotNull] StreamCoordinates end) { StreamName = streamName; StreamClient = streamClient; Start = start.ToDictionary(); End = end.ToDictionary(); }
public static void Write([NotNull] StreamCoordinates coordinates, [NotNull] IBinaryWriter writer) { writer.EnsureBigEndian(); writer.Write(coordinates.Positions.Length); foreach (var position in coordinates.Positions) { writer.Write(position.Partition); writer.Write(position.Offset); } }
public void FilterBy_should_work_correctly() { var a = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 }, new StreamPosition { Partition = 2, Offset = 3 }, new StreamPosition { Partition = 3, Offset = 4 } }); var b = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 2 }, new StreamPosition { Partition = 2, Offset = 2 }, new StreamPosition { Partition = 5, Offset = 4 } }); var @fixed = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 }, new StreamPosition { Partition = 2, Offset = 3 } }); a.FilterBy(b).Positions.Should().BeEquivalentTo(@fixed.Positions); }
private void AddEvent(T @event, StreamCoordinates queryCoordinates) { var key = settings.KeyProvider(@event); if (!windows.ContainsKey(key)) { windows[key] = new Windows <T, TKey>(key, settings); } if (!windows[key].AddEvent(@event, queryCoordinates) && !restart) { eventsMetric?.For("dropped").Increment(); } }
private async Task Restart(StreamCoordinates rightCoordinates) { await RestartCoordinates(rightCoordinates).ConfigureAwait(false); try { RestartWindows(rightCoordinates).GetAwaiter().GetResult(); } catch (Exception e) { log.Error(e, "Failed to restart windows."); windows.Clear(); } }
/// <summary> /// Filter by next partitions. /// </summary> public static StreamCoordinates FilterBy([NotNull] this StreamCoordinates left, [NotNull] StreamCoordinates right) { var map = left.ToDictionary(); var result = new List <StreamPosition>(); foreach (var position in right.Positions) { if (map.TryGetValue(position.Partition, out var was)) { result.Add(was); } } return(new StreamCoordinates(result.ToArray())); }
public static byte[] Serialize([NotNull] StreamCoordinates coordinates) { var builder = new StringBuilder(); foreach (var position in coordinates.Positions) { builder .Append(position.Partition) .Append(" = ") .Append(position.Offset) .AppendLine(); } return(Encoding.UTF8.GetBytes(builder.ToString())); }
public static List <HerculesEvent>[] ReadEvents( this IHerculesStreamClient client, string stream, int count, int limit, int clientShards, StreamCoordinates coordinates = null) { var timeout = 30.Seconds(); var stopwatch = Stopwatch.StartNew(); var eventsRead = 0; var clientShardTasks = Enumerable.Range(0, clientShards).Select(ReadSingleClientShard); var events = Task.WhenAll(clientShardTasks).GetAwaiter().GetResult(); events.Sum(x => x.Count).Should().Be(count); return(events); async Task <List <HerculesEvent> > ReadSingleClientShard(int clientShard) { var shardEvents = new List <HerculesEvent>(); var readQuery = new ReadStreamQuery(stream) { Limit = limit, Coordinates = coordinates ?? StreamCoordinates.Empty, ClientShard = clientShard, ClientShardCount = clientShards }; while (stopwatch.Elapsed < timeout && eventsRead < count) { var result = await client.ReadAsync(readQuery, timeout); result.IsSuccessful.Should().BeTrue(); var eventsFromResponse = result.Payload.Events; shardEvents.AddRange(eventsFromResponse); readQuery.Coordinates = result.Payload.Next; Interlocked.Add(ref eventsRead, eventsFromResponse.Count); await Task.Delay(100); } return(shardEvents); } }
private static void TestSerialization(StreamCoordinates coordinates) { var writer = new BinaryBufferWriter(1) { Endianness = Endianness.Big }; StreamCoordinatesWriter.Write(coordinates, writer); var reader = new BinaryBufferReader(writer.Buffer, 0) { Endianness = Endianness.Big }; StreamCoordinatesReader.Read(reader).Should().BeEquivalentTo(coordinates); }
private async Task RestartCoordinates(StreamCoordinates rightCoordinates) { var storageCoordinates = await settings.LeftCoordinatesStorage.GetCurrentAsync().ConfigureAwait(false); storageCoordinates = storageCoordinates.FilterBy(rightCoordinates); LogCoordinates("Storage left", storageCoordinates); if (storageCoordinates.Positions.Length < rightCoordinates.Positions.Length) { log.Info("Some coordinates are missing. Returning right coordinates."); leftCoordinates = rightCoordinates; return; } log.Info("Returning storage coordinates."); leftCoordinates = storageCoordinates; }
private void AddEvent(T @event, StreamCoordinates queryCoordinates) { var key = settings.KeyProvider(@event); var lag = DateTime.Now - settings.TimestampProvider(@event); eventLagMetric.Report(lag); if (!windows.ContainsKey(key)) { windows[key] = new Windows <T, TKey>(key, settings); } if (!windows[key].AddEvent(@event, queryCoordinates) && !restart) { settings.OnEventDrop?.Invoke(@event); eventsMetric?.For("dropped").Increment(); } }
public void AdvancesOver_should_be_false_for_equal_coordinates() { var a = new StreamCoordinates( new[] { new StreamPosition { Partition = 0, Offset = 1 }, new StreamPosition { Partition = 1, Offset = 1 }, new StreamPosition { Partition = 2, Offset = 1 } }); a.AdvancesOver(a).Should().BeFalse(); }
public static long DistanceTo([NotNull] this StreamCoordinates from, [NotNull] StreamCoordinates to) { var map = from.ToDictionary(); var result = 0L; foreach (var position in to.Positions) { if (map.TryGetValue(position.Partition, out var p)) { result += position.Offset - p.Offset; } else { result += position.Offset; } } return(result); }
private async Task RestartWindows(StreamCoordinates rightCoordinates) { LogCoordinates("Current", leftCoordinates, rightCoordinates); windows.Clear(); var partitionsCount = await GetPartitionsCount().ConfigureAwait(false); foreach (var position in leftCoordinates.Positions) { var start = position.Offset; var end = rightCoordinates.Positions.Single(p => p.Partition == position.Partition).Offset; while (start < end) { start = await RestartPartition(position.Partition, partitionsCount, start, end).ConfigureAwait(false); } } }
protected private void HandleEvents(RawReadStreamPayload result, StreamCoordinates queryCoordinates) { int count; using (new OperationContextToken("HandleEvents")) using (var operationSpan = tracer.BeginConsumerCustomOperationSpan("HandleEvents")) using (iterationMetric?.For("handle_time").Measure()) { // ReSharper disable once AssignNullToNotNullAttribute var reader = new BinaryBufferReader(result.Content.Array, result.Content.Offset) { Endianness = Endianness.Big }; count = reader.ReadInt32(); for (var i = 0; i < count; i++) { var startPosition = reader.Position; try { var @event = EventsBinaryReader.ReadEvent(reader, settings.EventBuilderProvider(reader)); settings.OnEvent?.Invoke(@event, queryCoordinates); } catch (Exception e) { log.Error(e, "Failed to read event from position {Position}.", startPosition); reader.Position = startPosition; EventsBinaryReader.ReadEvent(reader, DummyEventBuilder.Instance); } } operationSpan.SetOperationDetails(count); LogProgress(count); } if (count == 0) { Thread.Sleep(settings.DelayOnNoEvents); } }