private async Task BloatDbAsync(Context context, IMemoizationSession session)
        {
            uint dummyFingerprintsToAdd = 40; // generates a ~52KB DB file
            var  addBlock = new ActionBlock <int>(
                async _ =>
            {
                var strongFingerprint = StrongFingerprint.Random();
                var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
                    ContentHashList.Random(), Determinism[DeterminismNone]);

                var result = await session.AddOrGetContentHashListAsync(
                    context, strongFingerprint, contentHashListWithDeterminism, Token);
                Assert.True(result.Succeeded);
            },
                new ExecutionDataflowBlockOptions {
                MaxDegreeOfParallelism = System.Environment.ProcessorCount
            });

            while (--dummyFingerprintsToAdd > 0)
            {
                await addBlock.SendAsync(0);
            }

            addBlock.Complete();
            await addBlock.Completion;
        }
예제 #2
0
        public Task GetSelectorsGivesSelectorsInReverseLruOrderAfterGet()
        {
            var context            = new Context(Logger);
            var weakFingerprint    = Fingerprint.Random();
            var selector1          = Selector.Random();
            var selector2          = Selector.Random();
            var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
            var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
            var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
            var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);

            return(RunTestAsync(context, async(store, session) =>
            {
                await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
                _clock.Increment();
                await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
                _clock.Increment();
                await session.GetContentHashListAsync(context, strongFingerprint1, Token).ShouldBeSuccess();
                _clock.Increment();

                List <GetSelectorResult> getSelectorResults = await session.GetSelectors(context, weakFingerprint, Token).ToListAsync();
                Assert.Equal(2, getSelectorResults.Count);

                GetSelectorResult r1 = getSelectorResults[0];
                Assert.True(r1.Succeeded);
                Assert.True(r1.Selector == selector1);

                GetSelectorResult r2 = getSelectorResults[1];
                Assert.True(r2.Succeeded);
                Assert.True(r2.Selector == selector2);
            }));
        }
예제 #3
0
        public Task PreferSharedUrgencyHintIsRespected()
        {
            var context            = new Context(Logger);
            var weakFingerprint    = Fingerprint.Random();
            var selector1          = Selector.Random();
            var selector2          = Selector.Random();
            var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
            var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
            var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
            var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);

            return(RunTestAsync(context, async session =>
            {
                await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
                _clock.Increment();
                await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
                _clock.Increment();

                var contentHashList = session.GetContentHashListAsync(context, strongFingerprint1, Token, UrgencyHint.PreferShared);

                List <GetSelectorResult> getSelectorResults = await session.GetSelectors(context, weakFingerprint, Token).ToListAsync(CancellationToken.None);
                Assert.Equal(2, getSelectorResults.Count);

                GetSelectorResult r1 = getSelectorResults[0];
                Assert.True(r1.Succeeded);
                Assert.True(r1.Selector == selector2);

                GetSelectorResult r2 = getSelectorResults[1];
                Assert.True(r2.Succeeded);
                Assert.True(r2.Selector == selector1);
            }));
        }
 /// <inheritdoc />
 public override AddOrGetContentHashListResult AddOrGetContentHashList(
     OperationContext context,
     StrongFingerprint strongFingerprint,
     ContentHashListWithDeterminism contentHashListWithDeterminism)
 {
     throw new NotImplementedException();
 }
예제 #5
0
 private void SealIfNecessaryAfterUnbackedAddOrGet(
     Context context,
     StrongFingerprint strongFingerprint,
     ContentHashListWithDeterminism addedValue,
     ContentHashListWithCacheMetadata cacheMetadata)
 {
     if (cacheMetadata == null)
     {
         // Our unbacked Add won the race, so the value is implicitly unbacked in VSTS
         // Seal the added value
         SealInTheBackground(context, strongFingerprint, addedValue);
     }
     else if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null)
     {
         // Value is explicitly unbacked in VSTS
         if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null)
         {
             // Our Add lost the race, so seal the existing value
             SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism);
         }
         else
         {
             // Our Add won the race, so seal the added value
             var valueToSeal = new ContentHashListWithDeterminism(
                 addedValue.ContentHashList, cacheMetadata.ContentHashListWithDeterminism.Determinism);
             SealInTheBackground(
                 context,
                 strongFingerprint,
                 valueToSeal);
         }
     }
 }
예제 #6
0
        public void ContentHashListWithDeterminismProperty()
        {
            var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
                ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));

            Assert.Equal(contentHashListWithDeterminism, new GetContentHashListResult(contentHashListWithDeterminism).ContentHashListWithDeterminism);
        }
예제 #7
0
        public Task GetSelectorsGivesSelectors()
        {
            var weakFingerprint    = Fingerprint.Random();
            var selector1          = Selector.Random();
            var selector2          = Selector.Random();
            var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
            var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
            var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
            var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);

            return(RunTest((context, contentLocationDatabase) =>
            {
                contentLocationDatabase.AddOrGetContentHashList(
                    context, strongFingerprint1, contentHashListWithDeterminism1).ShouldBeSuccess();
                contentLocationDatabase.AddOrGetContentHashList(
                    context, strongFingerprint2, contentHashListWithDeterminism2).ShouldBeSuccess();

                List <GetSelectorResult> getSelectorResults = contentLocationDatabase.GetSelectors(context, weakFingerprint).ToList();
                Assert.Equal(2, getSelectorResults.Count);

                GetSelectorResult r1 = getSelectorResults[0].ShouldBeSuccess();
                Assert.True(r1.Selector == selector1 || r1.Selector == selector2);

                GetSelectorResult r2 = getSelectorResults[1].ShouldBeSuccess();
                Assert.True(r2.Selector == selector1 || r2.Selector == selector2);
            }));
        }
예제 #8
0
        public void SerializeEmptyContentHashList()
        {
            var input        = new ContentHashListWithDeterminism();
            var deserialized = input.ToGrpc().FromGrpc();

            Assert.Equal(input, deserialized);
        }
예제 #9
0
        private void TestContentHashListWithDeterminismRoundtrip(ContentHashListWithDeterminism contentHashListWithDeterminism)
        {
            var redisValue        = _serializer.ToRedisValue(contentHashListWithDeterminism);
            var roundtrippedValue = _serializer.AsContentHashList(redisValue);

            Assert.Equal(contentHashListWithDeterminism, roundtrippedValue);
        }
        public async Task UpgradeFromBeforeSerializedDeterminism(int oldDeterminism, int shouldBecomeDeterminism)
        {
            var context           = new Context(Logger);
            var strongFingerprint = StrongFingerprint.Random();
            var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
                ContentHashList.Random(), Determinism[oldDeterminism]);

            using (var testDirectory = new DisposableDirectory(FileSystem))
            {
                await RunTestAsync(context, testDirectory, async (store, session) =>
                {
                    var result = await session.AddOrGetContentHashListAsync(
                        context, strongFingerprint, contentHashListWithDeterminism, Token);
                    Assert.True(result.Succeeded);

                    await((TestSQLiteMemoizationStore)store).DeleteColumnAsync("ContentHashLists", "SerializedDeterminism");
                });

                await RunTestAsync(context, testDirectory, async (store, session) =>
                {
                    var result = await session.GetContentHashListAsync(context, strongFingerprint, Token);
                    Assert.Equal(
                        new GetContentHashListResult(new ContentHashListWithDeterminism(
                                                         contentHashListWithDeterminism.ContentHashList, Determinism[shouldBecomeDeterminism])), result);
                });
            }
        }
예제 #11
0
 public AddOrGetContentHashListResult(
     ResultCode code,
     ContentHashListWithDeterminism contentHashListWithDeterminism)
 {
     Code = code;
     ContentHashListWithDeterminism = contentHashListWithDeterminism;
 }
예제 #12
0
        /// <nodoc />
        public static MetadataEntry Deserialize(BuildXLReader reader)
        {
            var lastUpdateTimeUtc = reader.ReadInt64Compact();
            var contentHashListWithDeterminism = ContentHashListWithDeterminism.Deserialize(reader);

            return(new MetadataEntry(contentHashListWithDeterminism, lastUpdateTimeUtc));
        }
예제 #13
0
 public void ContentHashListWithDeterminismRoundTrip(int contentHashCount, int determinism)
 {
     ContentHashListWithDeterminism contentHashListWithDeterminism = new ContentHashListWithDeterminism(
         ContentHashList.Random(HashingType, contentHashCount),
         m_memoizationDeterminism[determinism]);
     ContentHashListWithDeterminism roundTrip = contentHashListWithDeterminism.FromMemoization().ToMemoization();
 }
예제 #14
0
            public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
                Context context,
                StrongFingerprint strongFingerprint,
                ContentHashListWithDeterminism contentHashListWithDeterminism,
                CancellationToken cts,
                UrgencyHint urgencyHint = UrgencyHint.Nominal)
            {
                foreach (var hash in contentHashListWithDeterminism.ContentHashList.Hashes)
                {
                    Assert.True(_storedHashes.Contains(hash));
                }

                if (_forceUpdate)
                {
                    _storedHashLists.Remove(strongFingerprint);
                }

                var added = !_storedHashLists.ContainsKey(strongFingerprint);

                if (added)
                {
                    _storedHashLists[strongFingerprint] = contentHashListWithDeterminism;
                    return(Task.FromResult(new AddOrGetContentHashListResult(default(ContentHashListWithDeterminism))));
                }

                return(Task.FromResult(new AddOrGetContentHashListResult(_storedHashLists[strongFingerprint])));
            }
        private async Task <StrongFingerprint> PublishValueForRandomStrongFingerprintAsync(
            Context context,
            string cacheNamespace,
            BackingOption backingOption,
            TimeSpan expiryMinimum,
            TimeSpan expiryRange)
        {
            StrongFingerprint publishedFingerprint = await CreateBackedRandomStrongFingerprintAsync(context, cacheNamespace);

            Func <DisposableDirectory, ICache> createCache = dir => CreateBareBuildCache(
                dir, cacheNamespace, FileSystem, Logger, backingOption, ItemStorageOption, expiryMinimum, expiryRange);

            Func <ICacheSession, Task> publishAsync = async(ICacheSession session) =>
            {
                ContentHashListWithDeterminism valueToPublish =
                    await CreateRandomContentHashListWithDeterminismAsync(context, true, session);

                AddOrGetContentHashListResult addOrGetResult = await session.AddOrGetContentHashListAsync(
                    context, publishedFingerprint, valueToPublish, Token);

                // Ensure the new value was successfully published and that it was the winning value for that key
                Assert.True(addOrGetResult.Succeeded);
                Assert.Null(addOrGetResult.ContentHashListWithDeterminism.ContentHashList);
            };

            await RunTestAsync(context, publishAsync, createCache);

            return(publishedFingerprint);
        }
예제 #16
0
        public async Task TestGetOrAddStrongFingerprintAsyncExisting()
        {
            var strongFp = StrongFingerprint.Random();
            var hashList = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);

            var initialData = new Dictionary <RedisKey, RedisValue>
            {
                { _redisSerializer.ToRedisKey(strongFp).Prepend(RedisNameSpace), _redisSerializer.ToRedisValue(hashList) },
            };

            using (var mockDb = new MockRedisDatabase(SystemClock.Instance, initialData))
            {
                await RunTest(
                    mockDb,
                    async (context, metadataCache, redisDb) =>
                {
                    var getContentHashListResult = await metadataCache.GetOrAddContentHashListAsync(
                        context,
                        strongFp,
                        fp =>
                    {
                        throw new InvalidOperationException(
                            "GetFunc not expected to be called since data is already present in the cache");
                    });

                    // Check result
                    Assert.True(getContentHashListResult.Succeeded);
                    Assert.Equal(hashList, getContentHashListResult.ContentHashListWithDeterminism);
                });
            }
        }
예제 #17
0
        /// <summary>
        /// Tries to get a content hash list for a specific strong fingerprint in the cache.
        /// </summary>
        public bool TryGetValue(string cacheNamespace, StrongFingerprint strongFingerprint, out ContentHashListWithDeterminism value)
        {
            ContentHashListWithDeterminism existingValue;

            // Try to get the existing value
            var contentHashListCacheDictionary = _cacheNamespaceToContentHashListCacheDictionary.GetOrAdd(cacheNamespace, (key) => new StrongFingerprintToHashListDictionary());

            while (contentHashListCacheDictionary.TryGetValue(strongFingerprint, out existingValue))
            {
                if (!existingValue.Determinism.IsDeterministic)
                {
                    // The value is either tool deterministic or it's cache deterministic and has not expired, so it's usable
                    value = existingValue;
                    return(true);
                }

                if (
                    ((ICollection <KeyValuePair <StrongFingerprint, ContentHashListWithDeterminism> >)contentHashListCacheDictionary).Remove(
                        new KeyValuePair <StrongFingerprint, ContentHashListWithDeterminism>(strongFingerprint, existingValue)))
                {
                    // Removal was successful, so nothing usable
                    value = new ContentHashListWithDeterminism(null, CacheDeterminism.None);
                    return(false);
                }
            }

            // Nothing usable
            value = new ContentHashListWithDeterminism(null, CacheDeterminism.None);
            return(false);
        }
예제 #18
0
        public void EqualsTrue()
        {
            var contentHashListWithDeterminism = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);
            var v1 = new GetContentHashListResult(contentHashListWithDeterminism);
            var v2 = new GetContentHashListResult(contentHashListWithDeterminism);

            Assert.True(v1.Equals(v2));
        }
예제 #19
0
 /// <inheritdoc />
 public override Possible <bool> TryUpsert(
     OperationContext context,
     StrongFingerprint strongFingerprint,
     ContentHashListWithDeterminism replacement,
     Func <MetadataEntry, bool> shouldReplace)
 {
     throw new NotImplementedException();
 }
 /// <summary>
 ///     Queue a seal operation in the background. Attempts to upload all content to VSTS before updating the metadata as backed.
 ///     Lost races will be ignored and any failures will be logged and reported on shutdown.
 /// </summary>
 protected void SealInTheBackground(
     Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism)
 {
     if (_sealUnbackedContentHashLists)
     {
         _taskTracker.Add(Task.Run(() => SealAsync(context, strongFingerprint, contentHashListWithDeterminism)));
     }
 }
예제 #21
0
        public void GetHashCodeSameWhenEqual()
        {
            var contentHashListWithDeterminism = new ContentHashListWithDeterminism(
                ContentHashList.Random(), CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires));
            var v1 = new GetContentHashListResult(contentHashListWithDeterminism);
            var v2 = new GetContentHashListResult(contentHashListWithDeterminism);

            Assert.Equal(v1.GetHashCode(), v2.GetHashCode());
        }
예제 #22
0
 /// <inheritdoc />
 public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
     Context context,
     StrongFingerprint strongFingerprint,
     ContentHashListWithDeterminism contentHashListWithDeterminism,
     CancellationToken cts,
     UrgencyHint urgencyHint = UrgencyHint.Nominal)
 {
     throw new System.NotImplementedException();
 }
예제 #23
0
        public Task GarbageCollectionDeletesInLruOrder()
        {
            var context         = new Context(Logger);
            var weakFingerprint = Fingerprint.Random();

            var selector1          = Selector.Random();
            var strongFingerprint1 = new StrongFingerprint(weakFingerprint, selector1);
            var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);

            var selector2          = Selector.Random();
            var strongFingerprint2 = new StrongFingerprint(weakFingerprint, selector2);
            var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None);

            return(RunTestAsync(context,
                                funcAsync: async(store, session) =>
            {
                await session.AddOrGetContentHashListAsync(context, strongFingerprint1, contentHashListWithDeterminism1, Token).ShouldBeSuccess();
                _clock.Increment();

                await session.AddOrGetContentHashListAsync(context, strongFingerprint2, contentHashListWithDeterminism2, Token).ShouldBeSuccess();
                _clock.Increment();

                // Force update the last access time of the first fingerprint
                await session.GetContentHashListAsync(context, strongFingerprint1, Token).ShouldBeSuccess();
                _clock.Increment();

                RocksDbContentLocationDatabase database = (store as RocksDbMemoizationStore)?.RocksDbDatabase;
                Contract.Assert(database != null);

                var ctx = new OperationContext(context);
                await database.GarbageCollectAsync(ctx).ShouldBeSuccess();

                var r1 = database.GetContentHashList(ctx, strongFingerprint1).ShouldBeSuccess().ContentHashListWithDeterminism;
                r1.Should().BeEquivalentTo(contentHashListWithDeterminism1);

                var r2 = database.GetContentHashList(ctx, strongFingerprint2).ShouldBeSuccess().ContentHashListWithDeterminism;
                r2.ContentHashList.Should().BeNull();
                r2.Determinism.Should().Be(CacheDeterminism.None);

                database.Counters[ContentLocationDatabaseCounters.GarbageCollectMetadataEntriesRemoved].Value.Should().Be(1);
                database.Counters[ContentLocationDatabaseCounters.GarbageCollectMetadataEntriesScanned].Value.Should().Be(2);
            },
                                createStoreFunc: createStoreInternal));

            // This is needed because type errors arise if you inline
            IMemoizationStore createStoreInternal(DisposableDirectory disposableDirectory)
            {
                return(CreateStore(testDirectory: disposableDirectory, configMutator: (configuration) =>
                {
                    configuration.MetadataGarbageCollectionEnabled = true;
                    configuration.MetadataGarbageCollectionMaximumNumberOfEntriesToKeep = 1;
                    // Disables automatic GC
                    configuration.GarbageCollectionInterval = Timeout.InfiniteTimeSpan;
                }));
            }
        }
        public async Task Roundtrip()
        {
            var fileName = $"{Guid.NewGuid()}.json";

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var sessionId        = 42;
                var serializedConfig = "Foo";
                var pat = Guid.NewGuid().ToString();

                var numOperations = 3;
                var operations    = Enumerable.Range(0, numOperations)
                                    .Select(_ => generateRandomOperation())
                                    .ToList();

                var sessionInfo = new HibernatedCacheSessionInfo(sessionId, serializedConfig, pat, operations);
                var sessions1   = new HibernatedSessions <HibernatedCacheSessionInfo>(new List <HibernatedCacheSessionInfo> {
                    sessionInfo
                });
                await sessions1.WriteAsync(FileSystem, directory.Path, fileName);

                FileSystem.HibernatedSessionsExists(directory.Path, fileName).Should().BeTrue();

                var fileSize = FileSystem.GetFileSize(directory.Path / fileName);
                fileSize.Should().BeGreaterThan(0);

                var sessions2 = await FileSystem.ReadHibernatedSessionsAsync <HibernatedCacheSessionInfo>(directory.Path, fileName);

                sessions2.Sessions.Count.Should().Be(1);
                sessions2.Sessions[0].Id.Should().Be(sessionId);
                sessions2.Sessions[0].SerializedSessionConfiguration.Should().Be(serializedConfig);
                sessions2.Sessions[0].Pat.Should().Be(pat);
                sessions2.Sessions[0].PendingPublishingOperations.Should().BeEquivalentTo(operations);

                await FileSystem.DeleteHibernatedSessions(directory.Path, fileName);

                PublishingOperation generateRandomOperation()
                {
                    var amountOfHashes  = 3;
                    var hashes          = Enumerable.Range(0, amountOfHashes).Select(_ => ContentHash.Random()).ToArray();
                    var contentHashList = new ContentHashList(hashes);
                    var determinism     = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddMilliseconds(ThreadSafeRandom.Generator.Next()));
                    var contentHashListWithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);

                    var fingerprint       = new Fingerprint(ContentHash.Random().ToByteArray());
                    var selector          = new Selector(ContentHash.Random());
                    var strongFingerprint = new StrongFingerprint(fingerprint, selector);

                    return(new PublishingOperation
                    {
                        ContentHashListWithDeterminism = contentHashListWithDeterminism,
                        StrongFingerprint = strongFingerprint
                    });
                }
            }
        }
예제 #25
0
 /// <inheritdoc />
 public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
     Context context,
     StrongFingerprint strongFingerprint,
     ContentHashListWithDeterminism contentHashListWithDeterminism,
     CancellationToken cts,
     UrgencyHint urgencyHint)
 {
     return(MemoizationStore.AddOrGetContentHashListAsync(
                context, strongFingerprint, contentHashListWithDeterminism, _contentSession, cts));
 }
예제 #26
0
 /// <summary>
 ///     Load a ContentHashList.
 /// </summary>
 internal Task <GetContentHashListResult> GetContentHashListAsync(
     Context context, StrongFingerprint strongFingerprint, CancellationToken cts)
 {
     return(GetContentHashListCall.RunAsync(Tracer, context, strongFingerprint, async() =>
     {
         ContentHashListWithDeterminism contentHashListWithDeterminism =
             await RunConcurrentAsync(() => GetContentHashListAsync(strongFingerprint)).ConfigureAwait(false);
         UpdateLruOnGet(strongFingerprint);
         return new GetContentHashListResult(contentHashListWithDeterminism);
     }));
 }
예제 #27
0
        public void EqualsTrueNotReferenceEqualContentHashList()
        {
            var contentHashList = ContentHashList.Random();
            var determinism     = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), CacheDeterminism.NeverExpires);
            var contentHashListWithDeterminism1 = new ContentHashListWithDeterminism(contentHashList, determinism);
            var contentHashListWithDeterminism2 = new ContentHashListWithDeterminism(contentHashList, determinism);
            var v1 = new GetContentHashListResult(contentHashListWithDeterminism1);
            var v2 = new GetContentHashListResult(contentHashListWithDeterminism2);

            Assert.True(v1.Equals(v2));
        }
예제 #28
0
 /// <summary>
 /// Performs a compare exchange operation on metadata, while ensuring all invariants are kept. If the
 /// fingerprint is not present, then it is inserted.
 /// </summary>
 public Task <Result <bool> > CompareExchange(
     OperationContext context,
     StrongFingerprint strongFingerprint,
     string expectedReplacementToken,
     ContentHashListWithDeterminism expected,
     ContentHashListWithDeterminism replacement)
 {
     return(context.PerformOperationAsync(Tracer, () => CompareExchangeCore(context, strongFingerprint, expectedReplacementToken, expected, replacement),
                                          extraEndMessage: _ => $"StrongFingerprint=[{strongFingerprint}]",
                                          traceOperationStarted: false,
                                          traceErrorsOnly: true));
 }
예제 #29
0
        public Task <BoolResult> PublishContentHashListAsync(
            Context context,
            StrongFingerprint fingerprint,
            ContentHashListWithDeterminism contentHashList,
            CancellationToken token)
        {
            Contract.Check(_publisher != null)?.Assert("Startup should be run before attempting to publish.");

            var operationContext = new OperationContext(context, token);

            Tracer.Debug(operationContext, $"Enqueueing publish request for StrongFingerprint=[{fingerprint}], CHL=[{contentHashList.ToTraceString()}]");

            return(_publishingGate.GatedOperationAsync(
                       (timeSpentWaiting, gateCount) =>
            {
                ContentHashList?hashListInRemote = null;
                return operationContext.PerformOperationAsync(
                    Tracer,
                    async() =>
                {
                    // Make sure to push the blob in the selector if it exists.
                    var hashesToPush = new List <ContentHash>(contentHashList.ContentHashList.Hashes);
                    if (!fingerprint.Selector.ContentHash.IsZero())
                    {
                        hashesToPush.Add(fingerprint.Selector.ContentHash);
                    }

                    var remotePinResults = await Task.WhenAll(await _publisher.PinAsync(operationContext, hashesToPush, token));
                    var missingFromRemote = remotePinResults
                                            .Where(r => !r.Item.Succeeded)
                                            .Select(r => hashesToPush[r.Index])
                                            .ToArray();

                    if (missingFromRemote.Length > 0)
                    {
                        await PushToRemoteAsync(operationContext, missingFromRemote).ThrowIfFailure();
                    }

                    var addOrGetResult = await _publisher.AddOrGetContentHashListAsync(operationContext, fingerprint, contentHashList, token).ThrowIfFailure();
                    hashListInRemote = addOrGetResult.ContentHashListWithDeterminism.ContentHashList;

                    return BoolResult.Success;
                },
                    traceOperationStarted: false,
                    extraEndMessage: result =>
                    $"Added=[{result.Succeeded && hashListInRemote is null}], " +
                    $"StrongFingerprint=[{fingerprint}], " +
                    $"ContentHashList=[{contentHashList.ToTraceString()}], " +
                    $"TimeSpentWaiting=[{timeSpentWaiting}], " +
                    $"GateCount=[{gateCount}]");
            },
                       token));
        }
예제 #30
0
        private async Task <Result <bool> > CompareExchangeInternalAsync(
            OperationContext context,
            StrongFingerprint strongFingerprint,
            string expectedReplacementToken,
            ContentHashListWithDeterminism replacement,
            string newReplacementToken)
        {
            var key = GetKey(strongFingerprint.WeakFingerprint);
            var replacementMetadata = new MetadataEntry(replacement, DateTime.UtcNow);

            using var replacementBytes    = SerializeMetadataEntry(replacementMetadata);
            using var selectorBytes       = SerializeSelector(strongFingerprint.Selector, isReplacementToken: false);
            using var tokenFieldNameBytes = SerializeSelector(strongFingerprint.Selector, isReplacementToken: true);

            var(primaryResult, secondaryResult) = await _redis.ExecuteRaidedAsync <bool>(
                context,
                async (redis, cancellationToken) =>
            {
                using var nestedContext = new CancellableOperationContext(context, cancellationToken);

                return(await redis.ExecuteBatchAsync(
                           nestedContext,
                           batch =>
                {
                    var task = batch.CompareExchangeAsync(
                        key,
                        (ReadOnlyMemory <byte>)selectorBytes,
                        (ReadOnlyMemory <byte>)tokenFieldNameBytes,
                        expectedReplacementToken,
                        (ReadOnlyMemory <byte>)replacementBytes,
                        newReplacementToken);
                    batch.KeyExpireAsync(key, Configuration.ExpiryTime).FireAndForget(nestedContext);
                    return task;
                },
                           RedisOperation.CompareExchange));
            },
                retryWindow : Configuration.SlowOperationCancellationTimeout);

            Contract.Assert(primaryResult != null || secondaryResult != null);

            if (primaryResult?.Succeeded == true || secondaryResult?.Succeeded == true)
            {
                // One of the operations is successful.
                return((primaryResult?.Value ?? false) || (secondaryResult?.Value ?? false));
            }

            // All operations failed, propagating the error back to the caller.
            var failure = primaryResult ?? secondaryResult;

            Contract.Assert(!failure.Succeeded);
            return(new Result <bool>(failure));
        }