Ejemplo n.º 1
0
 /// <inheritdoc />
 public MemoryContentLocationEventStore(
     MemoryContentLocationEventStoreConfiguration configuration,
     IContentLocationEventHandler handler,
     CentralStorage centralStorage,
     Interfaces.FileSystem.AbsolutePath workingDirectory)
     : base(configuration, nameof(MemoryContentLocationEventStore), handler, centralStorage, workingDirectory)
 {
     _hub          = configuration.Hub;
     _hub.OnEvent += HubOnEvent;
 }
Ejemplo n.º 2
0
        public async Task LargeInstanceEventsTest(EventKind kind, string kindName)
        {
            BuildXL.Utilities.Analysis.IgnoreArgument(kindName, "Kind name is only specified so enum value name shows up in test name");
            if (kind == EventKind.Blob)
            {
                // Blob events don't have a large equivalent since they just have a storage id
                return;
            }

            var context = new OperationContext(new Context(Logger));
            var harness = new LargeEventTestHarness();
            var sender  = harness.Sender;

            var configuration  = new MemoryContentLocationEventStoreConfiguration();
            var memoryEventHub = new MemoryEventHubClient(configuration);
            var eventStore     = new EventHubContentLocationEventStore(configuration,
                                                                       harness,
                                                                       "TestMachine",
                                                                       harness,
                                                                       TestRootDirectoryPath / "eventwork",
                                                                       SystemClock.Instance);

            await eventStore.StartupAsync(context).ThrowIfFailure();

            eventStore.StartProcessing(context, new EventSequencePoint(DateTime.UtcNow)).ThrowIfFailure();

            var serializer = CreateContentLocationEventDataSerializer();

            await sendAndVerifyLargeEvent(kind);

            using var stream = new MemoryStream();
            using var writer = BuildXL.Utilities.BuildXLWriter.Create(stream);

            serializer.SerializeEvents(writer, harness.Events);

            stream.Position.Should().BeGreaterThan(ContentLocationEventDataSerializer.MaxEventDataPayloadSize,
                                                   "Event should be larger than max event payload size to properly test serialization logic");

            bool canSplit = kind == EventKind.AddLocation ||
                            kind == EventKind.AddLocationWithoutTouching ||
                            kind == EventKind.RemoveLocation ||
                            kind == EventKind.RemoveLocation ||
                            kind == EventKind.Touch;

            foreach (var eventData in harness.Events)
            {
                eventData.SerializationKind.Should().Be(kind);
            }

            configuration.Hub.EventStream.Count.Should().BeGreaterThan(0);
            foreach (var rawEvent in configuration.Hub.EventStream)
            {
                rawEvent.Body.Count.Should().BeLessOrEqualTo(ContentLocationEventDataSerializer.MaxEventDataPayloadSize);
            }

            if (canSplit)
            {
                // Events should be split
                harness.Events.Count.Should().BeGreaterThan(1);
                configuration.Hub.EventStream.Count.Should().BeGreaterThan(1);

                // No uploads/downloads should happen for splittable events
                harness.State.DownloadedCount.Should().Be(0);
                harness.State.UploadedCount.Should().Be(0);
                harness.State.UploadedSize.Should().Be(0);
                harness.State.DownloadedSize.Should().Be(0);
            }
            else
            {
                harness.Events.Count.Should().Be(1);
                configuration.Hub.EventStream.Count.Should().Be(1);

                harness.State.DownloadedCount.Should().Be(1);
                harness.State.UploadedCount.Should().Be(1);
                harness.State.UploadedSize.Should().BeGreaterOrEqualTo(stream.Position);
                harness.State.UploadedSize.Should().Be(harness.State.DownloadedSize);
            }

            async Task sendAndVerifyLargeEvent(EventKind kind)
            {
                const int largeEventContentCount = 50000;

                switch (kind)
                {
                case EventKind.AddLocation:
                case EventKind.AddLocationWithoutTouching:
                {
                    var sent = Enumerable.Range(0, largeEventContentCount).Select(_ => new ContentHashWithSize(ContentHash.Random(), ThreadSafeRandom.Generator.Next(0, 100000))).ToList();

                    eventStore.AddLocations(
                        context,
                        sender,
                        sent,
                        touch: kind == EventKind.AddLocation).ThrowIfFailure();

                    var received = harness.Events.Cast <AddContentLocationEventData>().SelectMany(e => e.ContentHashes.SelectList((hash, index) => (hash, size: e.ContentSizes[index]))).ToList();

                    received.Count.Should().Be(sent.Count);

                    for (int i = 0; i < received.Count; i++)
                    {
                        received[i].hash.Should().Be(new ShortHash(sent[i].Hash));
                        received[i].size.Should().Be(sent[i].Size);
                    }

                    return;
                }

                case EventKind.RemoveLocation:
                {
                    var sent = Enumerable.Range(0, largeEventContentCount).Select(_ => ContentHash.Random()).ToList();

                    eventStore.RemoveLocations(
                        context,
                        sender,
                        sent).ThrowIfFailure();

                    var received = harness.Events.Cast <RemoveContentLocationEventData>().SelectMany(e => e.ContentHashes).ToList();

                    received.Count.Should().Be(sent.Count);

                    for (int i = 0; i < received.Count; i++)
                    {
                        received[i].Should().Be(new ShortHash(sent[i]));
                    }

                    return;
                }

                case EventKind.Touch:
                {
                    var sent = Enumerable.Range(0, largeEventContentCount).Select(_ => ContentHash.Random()).ToList();

                    eventStore.Touch(
                        context,
                        sender,
                        sent,
                        DateTime.UtcNow).ThrowIfFailure();

                    var received = harness.Events.Cast <TouchContentLocationEventData>().SelectMany(e => e.ContentHashes).ToList();

                    received.Count.Should().Be(sent.Count);

                    for (int i = 0; i < received.Count; i++)
                    {
                        received[i].Should().Be(new ShortHash(sent[i]));
                    }

                    return;
                }

                case EventKind.UpdateMetadataEntry:
                {
                    var contentHashes = Enumerable.Range(0, largeEventContentCount).Select(_ => ContentHash.Random()).ToArray();
                    var sent          = new UpdateMetadataEntryEventData(
                        sender,
                        StrongFingerprint.Random(),
                        new MetadataEntry(
                            new ContentHashListWithDeterminism(
                                new ContentHashList(contentHashes),
                                CacheDeterminism.None),
                            DateTime.UtcNow.ToFileTimeUtc()));

                    await eventStore.UpdateMetadataEntryAsync(context, sent).ShouldBeSuccess();

                    var received = harness.Events.Cast <UpdateMetadataEntryEventData>().SelectMany(e => e.Entry.ContentHashListWithDeterminism.ContentHashList.Hashes).ToList();

                    received.Count.Should().Be(contentHashes.Length);

                    for (int i = 0; i < received.Count; i++)
                    {
                        received[i].Should().Be(contentHashes[i]);
                    }

                    return;
                }

                default:
                    throw Contract.AssertFailure($"No large event for kind {kind}. Add large event to ensure this case is tested");
                }
            }
        }
Ejemplo n.º 3
0
 /// <nodoc />
 public MemoryEventHubClient(MemoryContentLocationEventStoreConfiguration configuration)
 {
     _hub = configuration.Hub;
 }