public static StreamCoordinates SetPosition(this StreamCoordinates coordinates, StreamPosition position)
{
var dict = coordinates.ToDictionary();
dict[position.Partition] = position;
return(new StreamCoordinates(dict.Values.ToArray()));
}
public bool AddEvent(T @event, StreamCoordinates coordinates)
{
var now = DateTimeOffset.Now;
var timestamp = settings.TimestampProvider(@event);
if (!timestamp.InInterval(now - settings.MaximumDeltaBeforeNow, now + settings.MaximumDeltaAfterNow))
{
return(false);
}
if (timestamp < minimumAllowedTimestamp)
{
return(false);
}
if (maximumObservedTimestamp < timestamp)
{
maximumObservedTimestamp = timestamp;
}
foreach (var window in windows)
{
if (window.AddEvent(@event, timestamp))
{
return(true);
}
}
var newWindow = CreateWindow(@event, timestamp, coordinates);
newWindow.AddEvent(@event, timestamp);
windows.Add(newWindow);
LastEventAdded = now.UtcDateTime;
return(true);
}
private async Task RestartCoordinates()
{
LogShardingSettings();
if (coordinates != null)
{
await settings.CoordinatesStorage.AdvanceAsync(coordinates).ConfigureAwait(false);
}
LogCoordinates("Current", coordinates);
var endCoordinates = await SeekToEndAsync(shardingSettings).ConfigureAwait(false);
LogCoordinates("End", endCoordinates);
var storageCoordinates = await settings.CoordinatesStorage.GetCurrentAsync().ConfigureAwait(false);
storageCoordinates = storageCoordinates.FilterBy(endCoordinates);
LogCoordinates("Storage", storageCoordinates);
if (storageCoordinates.Positions.Length < endCoordinates.Positions.Length)
{
log.Info("Some coordinates are missing. Returning end coordinates.");
coordinates = endCoordinates;
return;
}
log.Info("Returning storage coordinates.");
coordinates = storageCoordinates;
}
public async Task <long?> CountStreamRemainingEventsAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken = default)
{
try
{
var endCoordinates = await SeekToEndAsync(shardingSettings, cancellationToken).ConfigureAwait(false);
var distance = coordinates.DistanceTo(endCoordinates);
log.Debug(
"Stream remaining events: {Count}. Current coordinates: {CurrentCoordinates}, end coordinates: {EndCoordinates}.",
distance,
coordinates,
endCoordinates);
return(distance);
}
catch (Exception e)
{
log.Warn(e, "Failed to count remaining events.");
return(null);
}
}
public ReadStreamPayload(
[NotNull] IList <T> events,
[NotNull] StreamCoordinates next)
{
Events = events ?? throw new ArgumentNullException(nameof(events));
Next = next ?? throw new ArgumentNullException(nameof(next));
}
public void Distance_should_work_correctly()
{
var a = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 100
}
});
var b = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 1, Offset = 200
},
new StreamPosition {
Partition = 2, Offset = 2
}
});
a.DistanceTo(b).Should().Be(102);
b.DistanceTo(a).Should().Be(-99);
}
public void AdvancesOver_should_be_true_if_some_coordinates_added()
{
var a = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
},
new StreamPosition {
Partition = 2, Offset = 1
}
});
var b = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
}
});
a.AdvancesOver(b).Should().BeTrue();
b.AdvancesOver(a).Should().BeFalse();
}
private async Task <StreamCoordinates> GetShardCoordinates(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
CancellationToken cancellationToken)
{
var(_, result) = await streamReader.ReadAsync(coordinates, shardingSettings, 1, cancellationToken).ConfigureAwait(false);
var map = result.Payload.Next.ToDictionary();
foreach (var position in coordinates.Positions)
{
if (map.ContainsKey(position.Partition))
{
map[position.Partition] = new StreamPosition
{
Partition = position.Partition,
Offset = position.Offset
}
}
;
}
return(new StreamCoordinates(map.Values.ToArray()));
}
}
private async Task MakeIteration(CancellationToken cancellationToken)
{
ReadStreamQuery query;
ReadStreamResult <T> result;
using (iterationMetric?.For("read_time").Measure())
{
(query, result) = await streamReader.ReadAsync(coordinates, shardingSettings, cancellationToken).ConfigureAwait(false);
}
var events = result.Payload.Events;
LogProgress(events.Count);
if (events.Count != 0)
{
using (iterationMetric?.For("handle_time").Measure())
{
await settings.EventsHandler.HandleAsync(query, result, cancellationToken).ConfigureAwait(false);
}
}
coordinates = result.Payload.Next;
if (events.Count == 0)
{
await Task.Delay(settings.DelayOnNoEvents, cancellationToken).ConfigureAwait(false);
}
Task.Run(() => settings.CoordinatesStorage.AdvanceAsync(coordinates));
}
public void MergeMin_should_work_correctly()
{
var a = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
},
new StreamPosition {
Partition = 2, Offset = 3
},
new StreamPosition {
Partition = 3, Offset = 4
}
});
var b = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 2
},
new StreamPosition {
Partition = 2, Offset = 2
},
new StreamPosition {
Partition = 5, Offset = 4
}
});
var min = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
},
new StreamPosition {
Partition = 2, Offset = 2
},
new StreamPosition {
Partition = 3, Offset = 4
},
new StreamPosition {
Partition = 5, Offset = 4
}
});
a.MergeMinWith(b).Positions.Should().BeEquivalentTo(min.Positions);
b.MergeMinWith(a).Positions.Should().BeEquivalentTo(min.Positions);
}
private async Task Stop(StreamCoordinates rightCoordinates)
{
if (leftCoordinates != null)
{
await settings.LeftCoordinatesStorage.AdvanceAsync(leftCoordinates).ConfigureAwait(false);
}
LogCoordinates("Stop", leftCoordinates, rightCoordinates);
}
internal Window(WindowedStreamConsumerSettings <T, TKey> .IWindow implementation, StreamCoordinates firstEventCoordinates, DateTimeOffset start, DateTimeOffset end, TimeSpan period, TimeSpan lag)
{
this.implementation = implementation;
FirstEventCoordinates = firstEventCoordinates;
Start = start;
End = end;
this.period = period;
this.lag = lag;
lastEventAdded = DateTimeOffset.Now;
}
private Window <T, TKey> CreateWindow(T @event, DateTimeOffset timestamp, StreamCoordinates coordinates)
{
var period = TimeSpanArithmetics.Min(settings.PeriodProvider?.Invoke(@event) ?? settings.Period, settings.MaximumAllowedPeriod);
var lag = TimeSpanArithmetics.Min(settings.LagProvider?.Invoke(@event) ?? settings.Lag, settings.MaximumAllowedLag);
var start = timestamp.AddTicks(-timestamp.Ticks % period.Ticks);
var result = new Window <T, TKey>(settings.CreateWindow(key), coordinates, start, start + period, period, lag);
return(result);
}
public async Task <(ReadStreamQuery query, ReadStreamResult result)> ReadAsync(
StreamCoordinates coordinates,
StreamShardingSettings shardingSettings,
long additionalLimit,
CancellationToken cancellationToken)
{
var(query, result) = await reader.ReadAsync(coordinates, shardingSettings, additionalLimit, cancellationToken).ConfigureAwait(false);
return(query, result.FromGenericResult());
}
private static ArraySegment <byte> CreateRequestBody([NotNull] StreamCoordinates coordinates)
{
var writer = new BinaryBufferWriter(sizeof(int) + coordinates.Positions.Length * (sizeof(int) + sizeof(long)))
{
Endianness = Endianness.Big
};
StreamCoordinatesWriter.Write(coordinates, writer);
return(writer.FilledSegment);
}
public StreamSegmentReaderSettings(
[NotNull] string streamName,
[NotNull] IHerculesStreamClient <T> streamClient,
[NotNull] StreamCoordinates start,
[NotNull] StreamCoordinates end)
{
StreamName = streamName;
StreamClient = streamClient;
Start = start.ToDictionary();
End = end.ToDictionary();
}
public static void Write([NotNull] StreamCoordinates coordinates, [NotNull] IBinaryWriter writer)
{
writer.EnsureBigEndian();
writer.Write(coordinates.Positions.Length);
foreach (var position in coordinates.Positions)
{
writer.Write(position.Partition);
writer.Write(position.Offset);
}
}
public void FilterBy_should_work_correctly()
{
var a = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
},
new StreamPosition {
Partition = 2, Offset = 3
},
new StreamPosition {
Partition = 3, Offset = 4
}
});
var b = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 2
},
new StreamPosition {
Partition = 2, Offset = 2
},
new StreamPosition {
Partition = 5, Offset = 4
}
});
var @fixed = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
},
new StreamPosition {
Partition = 2, Offset = 3
}
});
a.FilterBy(b).Positions.Should().BeEquivalentTo(@fixed.Positions);
}
private void AddEvent(T @event, StreamCoordinates queryCoordinates)
{
var key = settings.KeyProvider(@event);
if (!windows.ContainsKey(key))
{
windows[key] = new Windows <T, TKey>(key, settings);
}
if (!windows[key].AddEvent(@event, queryCoordinates) && !restart)
{
eventsMetric?.For("dropped").Increment();
}
}
private async Task Restart(StreamCoordinates rightCoordinates)
{
await RestartCoordinates(rightCoordinates).ConfigureAwait(false);
try
{
RestartWindows(rightCoordinates).GetAwaiter().GetResult();
}
catch (Exception e)
{
log.Error(e, "Failed to restart windows.");
windows.Clear();
}
}
/// <summary>
/// Filter by next partitions.
/// </summary>
public static StreamCoordinates FilterBy([NotNull] this StreamCoordinates left, [NotNull] StreamCoordinates right)
{
var map = left.ToDictionary();
var result = new List <StreamPosition>();
foreach (var position in right.Positions)
{
if (map.TryGetValue(position.Partition, out var was))
{
result.Add(was);
}
}
return(new StreamCoordinates(result.ToArray()));
}
public static byte[] Serialize([NotNull] StreamCoordinates coordinates)
{
var builder = new StringBuilder();
foreach (var position in coordinates.Positions)
{
builder
.Append(position.Partition)
.Append(" = ")
.Append(position.Offset)
.AppendLine();
}
return(Encoding.UTF8.GetBytes(builder.ToString()));
}
public static List <HerculesEvent>[] ReadEvents(
this IHerculesStreamClient client,
string stream,
int count,
int limit,
int clientShards,
StreamCoordinates coordinates = null)
{
var timeout = 30.Seconds();
var stopwatch = Stopwatch.StartNew();
var eventsRead = 0;
var clientShardTasks = Enumerable.Range(0, clientShards).Select(ReadSingleClientShard);
var events = Task.WhenAll(clientShardTasks).GetAwaiter().GetResult();
events.Sum(x => x.Count).Should().Be(count);
return(events);
async Task <List <HerculesEvent> > ReadSingleClientShard(int clientShard)
{
var shardEvents = new List <HerculesEvent>();
var readQuery = new ReadStreamQuery(stream)
{
Limit = limit,
Coordinates = coordinates ?? StreamCoordinates.Empty,
ClientShard = clientShard,
ClientShardCount = clientShards
};
while (stopwatch.Elapsed < timeout && eventsRead < count)
{
var result = await client.ReadAsync(readQuery, timeout);
result.IsSuccessful.Should().BeTrue();
var eventsFromResponse = result.Payload.Events;
shardEvents.AddRange(eventsFromResponse);
readQuery.Coordinates = result.Payload.Next;
Interlocked.Add(ref eventsRead, eventsFromResponse.Count);
await Task.Delay(100);
}
return(shardEvents);
}
}
private static void TestSerialization(StreamCoordinates coordinates)
{
var writer = new BinaryBufferWriter(1)
{
Endianness = Endianness.Big
};
StreamCoordinatesWriter.Write(coordinates, writer);
var reader = new BinaryBufferReader(writer.Buffer, 0)
{
Endianness = Endianness.Big
};
StreamCoordinatesReader.Read(reader).Should().BeEquivalentTo(coordinates);
}
private async Task RestartCoordinates(StreamCoordinates rightCoordinates)
{
var storageCoordinates = await settings.LeftCoordinatesStorage.GetCurrentAsync().ConfigureAwait(false);
storageCoordinates = storageCoordinates.FilterBy(rightCoordinates);
LogCoordinates("Storage left", storageCoordinates);
if (storageCoordinates.Positions.Length < rightCoordinates.Positions.Length)
{
log.Info("Some coordinates are missing. Returning right coordinates.");
leftCoordinates = rightCoordinates;
return;
}
log.Info("Returning storage coordinates.");
leftCoordinates = storageCoordinates;
}
private void AddEvent(T @event, StreamCoordinates queryCoordinates)
{
var key = settings.KeyProvider(@event);
var lag = DateTime.Now - settings.TimestampProvider(@event);
eventLagMetric.Report(lag);
if (!windows.ContainsKey(key))
{
windows[key] = new Windows <T, TKey>(key, settings);
}
if (!windows[key].AddEvent(@event, queryCoordinates) && !restart)
{
settings.OnEventDrop?.Invoke(@event);
eventsMetric?.For("dropped").Increment();
}
}
public void AdvancesOver_should_be_false_for_equal_coordinates()
{
var a = new StreamCoordinates(
new[]
{
new StreamPosition {
Partition = 0, Offset = 1
},
new StreamPosition {
Partition = 1, Offset = 1
},
new StreamPosition {
Partition = 2, Offset = 1
}
});
a.AdvancesOver(a).Should().BeFalse();
}
public static long DistanceTo([NotNull] this StreamCoordinates from, [NotNull] StreamCoordinates to)
{
var map = from.ToDictionary();
var result = 0L;
foreach (var position in to.Positions)
{
if (map.TryGetValue(position.Partition, out var p))
{
result += position.Offset - p.Offset;
}
else
{
result += position.Offset;
}
}
return(result);
}
private async Task RestartWindows(StreamCoordinates rightCoordinates)
{
LogCoordinates("Current", leftCoordinates, rightCoordinates);
windows.Clear();
var partitionsCount = await GetPartitionsCount().ConfigureAwait(false);
foreach (var position in leftCoordinates.Positions)
{
var start = position.Offset;
var end = rightCoordinates.Positions.Single(p => p.Partition == position.Partition).Offset;
while (start < end)
{
start = await RestartPartition(position.Partition, partitionsCount, start, end).ConfigureAwait(false);
}
}
}
protected private void HandleEvents(RawReadStreamPayload result, StreamCoordinates queryCoordinates)
{
int count;
using (new OperationContextToken("HandleEvents"))
using (var operationSpan = tracer.BeginConsumerCustomOperationSpan("HandleEvents"))
using (iterationMetric?.For("handle_time").Measure())
{
// ReSharper disable once AssignNullToNotNullAttribute
var reader = new BinaryBufferReader(result.Content.Array, result.Content.Offset)
{
Endianness = Endianness.Big
};
count = reader.ReadInt32();
for (var i = 0; i < count; i++)
{
var startPosition = reader.Position;
try
{
var @event = EventsBinaryReader.ReadEvent(reader, settings.EventBuilderProvider(reader));
settings.OnEvent?.Invoke(@event, queryCoordinates);
}
catch (Exception e)
{
log.Error(e, "Failed to read event from position {Position}.", startPosition);
reader.Position = startPosition;
EventsBinaryReader.ReadEvent(reader, DummyEventBuilder.Instance);
}
}
operationSpan.SetOperationDetails(count);
LogProgress(count);
}
if (count == 0)
{
Thread.Sleep(settings.DelayOnNoEvents);
}
}