/// <inheritdoc />
public MemoryContentLocationEventStore(
MemoryContentLocationEventStoreConfiguration configuration,
IContentLocationEventHandler handler,
CentralStorage centralStorage,
Interfaces.FileSystem.AbsolutePath workingDirectory)
: base(configuration, nameof(MemoryContentLocationEventStore), handler, centralStorage, workingDirectory)
{
_hub = configuration.Hub;
_hub.OnEvent += HubOnEvent;
}
public async Task LargeInstanceEventsTest(EventKind kind, string kindName)
{
BuildXL.Utilities.Analysis.IgnoreArgument(kindName, "Kind name is only specified so enum value name shows up in test name");
if (kind == EventKind.Blob)
{
// Blob events don't have a large equivalent since they just have a storage id
return;
}
var context = new OperationContext(new Context(Logger));
var harness = new LargeEventTestHarness();
var sender = harness.Sender;
var configuration = new MemoryContentLocationEventStoreConfiguration();
var memoryEventHub = new MemoryEventHubClient(configuration);
var eventStore = new EventHubContentLocationEventStore(configuration,
harness,
"TestMachine",
harness,
TestRootDirectoryPath / "eventwork",
SystemClock.Instance);
await eventStore.StartupAsync(context).ThrowIfFailure();
eventStore.StartProcessing(context, new EventSequencePoint(DateTime.UtcNow)).ThrowIfFailure();
var serializer = CreateContentLocationEventDataSerializer();
await sendAndVerifyLargeEvent(kind);
using var stream = new MemoryStream();
using var writer = BuildXL.Utilities.BuildXLWriter.Create(stream);
serializer.SerializeEvents(writer, harness.Events);
stream.Position.Should().BeGreaterThan(ContentLocationEventDataSerializer.MaxEventDataPayloadSize,
"Event should be larger than max event payload size to properly test serialization logic");
bool canSplit = kind == EventKind.AddLocation ||
kind == EventKind.AddLocationWithoutTouching ||
kind == EventKind.RemoveLocation ||
kind == EventKind.RemoveLocation ||
kind == EventKind.Touch;
foreach (var eventData in harness.Events)
{
eventData.SerializationKind.Should().Be(kind);
}
configuration.Hub.EventStream.Count.Should().BeGreaterThan(0);
foreach (var rawEvent in configuration.Hub.EventStream)
{
rawEvent.Body.Count.Should().BeLessOrEqualTo(ContentLocationEventDataSerializer.MaxEventDataPayloadSize);
}
if (canSplit)
{
// Events should be split
harness.Events.Count.Should().BeGreaterThan(1);
configuration.Hub.EventStream.Count.Should().BeGreaterThan(1);
// No uploads/downloads should happen for splittable events
harness.State.DownloadedCount.Should().Be(0);
harness.State.UploadedCount.Should().Be(0);
harness.State.UploadedSize.Should().Be(0);
harness.State.DownloadedSize.Should().Be(0);
}
else
{
harness.Events.Count.Should().Be(1);
configuration.Hub.EventStream.Count.Should().Be(1);
harness.State.DownloadedCount.Should().Be(1);
harness.State.UploadedCount.Should().Be(1);
harness.State.UploadedSize.Should().BeGreaterOrEqualTo(stream.Position);
harness.State.UploadedSize.Should().Be(harness.State.DownloadedSize);
}
async Task sendAndVerifyLargeEvent(EventKind kind)
{
const int largeEventContentCount = 50000;
switch (kind)
{
case EventKind.AddLocation:
case EventKind.AddLocationWithoutTouching:
{
var sent = Enumerable.Range(0, largeEventContentCount).Select(_ => new ContentHashWithSize(ContentHash.Random(), ThreadSafeRandom.Generator.Next(0, 100000))).ToList();
eventStore.AddLocations(
context,
sender,
sent,
touch: kind == EventKind.AddLocation).ThrowIfFailure();
var received = harness.Events.Cast <AddContentLocationEventData>().SelectMany(e => e.ContentHashes.SelectList((hash, index) => (hash, size: e.ContentSizes[index]))).ToList();
received.Count.Should().Be(sent.Count);
for (int i = 0; i < received.Count; i++)
{
received[i].hash.Should().Be(new ShortHash(sent[i].Hash));
received[i].size.Should().Be(sent[i].Size);
}
return;
}
case EventKind.RemoveLocation:
{
var sent = Enumerable.Range(0, largeEventContentCount).Select(_ => ContentHash.Random()).ToList();
eventStore.RemoveLocations(
context,
sender,
sent).ThrowIfFailure();
var received = harness.Events.Cast <RemoveContentLocationEventData>().SelectMany(e => e.ContentHashes).ToList();
received.Count.Should().Be(sent.Count);
for (int i = 0; i < received.Count; i++)
{
received[i].Should().Be(new ShortHash(sent[i]));
}
return;
}
case EventKind.Touch:
{
var sent = Enumerable.Range(0, largeEventContentCount).Select(_ => ContentHash.Random()).ToList();
eventStore.Touch(
context,
sender,
sent,
DateTime.UtcNow).ThrowIfFailure();
var received = harness.Events.Cast <TouchContentLocationEventData>().SelectMany(e => e.ContentHashes).ToList();
received.Count.Should().Be(sent.Count);
for (int i = 0; i < received.Count; i++)
{
received[i].Should().Be(new ShortHash(sent[i]));
}
return;
}
case EventKind.UpdateMetadataEntry:
{
var contentHashes = Enumerable.Range(0, largeEventContentCount).Select(_ => ContentHash.Random()).ToArray();
var sent = new UpdateMetadataEntryEventData(
sender,
StrongFingerprint.Random(),
new MetadataEntry(
new ContentHashListWithDeterminism(
new ContentHashList(contentHashes),
CacheDeterminism.None),
DateTime.UtcNow.ToFileTimeUtc()));
await eventStore.UpdateMetadataEntryAsync(context, sent).ShouldBeSuccess();
var received = harness.Events.Cast <UpdateMetadataEntryEventData>().SelectMany(e => e.Entry.ContentHashListWithDeterminism.ContentHashList.Hashes).ToList();
received.Count.Should().Be(contentHashes.Length);
for (int i = 0; i < received.Count; i++)
{
received[i].Should().Be(contentHashes[i]);
}
return;
}
default:
throw Contract.AssertFailure($"No large event for kind {kind}. Add large event to ensure this case is tested");
}
}
}
/// <nodoc />
public MemoryEventHubClient(MemoryContentLocationEventStoreConfiguration configuration)
{
_hub = configuration.Hub;
}