private async Task <BoolResult> PublishContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashList, CancellationToken token) { Contract.Assert(!_hasBeenMarkedForShutdown, "Should not queue publish operations after the session has been marked for shutdown"); var publishingOperation = new PublishingOperation { StrongFingerprint = strongFingerprint, ContentHashListWithDeterminism = contentHashList }; var addResult = _pendingPublishingOperations.TryAdd(publishingOperation, Interlocked.Increment(ref _orderNumber)); if (!addResult) { // Since the operation is already pending, we return success. It's important to note that ordering matters, since // publishing can overwrite the remote. // Example of how ordering matters: // These operations are queued in this order. // Publish SF(A) with CHL(A) // Publish SF(A) with CHL(B) // Publish SF(A) with CHL(A) // If we followed the queuing order and the remote was being overwritten by every call, the end result is SF(A)->CHL(A). // If we eliminate duplicates, and the first operation is still pending while we queue the third one, the result is SF(A)->CHL(B). // For now, we are willing to take chanses and ocassionally publish CHLs out of order. return(BoolResult.Success); } var result = await _remote.PublishContentHashListAsync( context, strongFingerprint, contentHashList, token); _pendingPublishingOperations.TryRemove(publishingOperation, out _); // Make sure that we unblock shutdown once all publishing operations have been completed. if (_hasBeenMarkedForShutdown && _pendingPublishingOperations.Count == 0) { _readyForShutdown.TrySetResult(true); } return(result); }
private async Task <BoolResult> PublishContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashList, CancellationToken token) { var publishingOperation = new PublishingOperation { StrongFingerprint = strongFingerprint, ContentHashListWithDeterminism = contentHashList }; var addResult = _pendingPublishingOperations.TryAdd(publishingOperation, Interlocked.Increment(ref _orderNumber)); if (!addResult) { // Since the operation is already pending, we return success. It's important to note that ordering matters, since // publishing can overwrite the remote. // Example of how ordering matters: // These operations are queued in this order. // Publish SF(A) with CHL(A) // Publish SF(A) with CHL(B) // Publish SF(A) with CHL(A) // If we followed the queuing order and the remote was being overwritten by every call, the end result is SF(A)->CHL(A). // If we eliminate duplicates, and the first operation is still pending while we queue the third one, the result is SF(A)->CHL(B). // For now, we are willing to take chanses and ocassionally publish CHLs out of order. return(BoolResult.Success); } var result = await _remote.PublishContentHashListAsync( context, strongFingerprint, contentHashList, token); _pendingPublishingOperations.TryRemove(publishingOperation, out _); return(result); }
public async Task CacheSessionDataIsHibernated() { using var testDirectory = new DisposableDirectory(FileSystem); var cacheDirectory = testDirectory.Path / "Service"; var cacheName = "theCache"; var namedCacheRoots = new Dictionary <string, AbsolutePath> { [cacheName] = cacheDirectory / "Root" }; var grpcPort = PortExtensions.GetNextAvailablePort(); var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem) { GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it. }; var scenario = "Default"; var server = new LocalCacheServer( FileSystem, TestGlobal.Logger, scenario, cacheFactory: CreateBlockingPublishingCache, serverConfiguration, Capabilities.All); var context = new OperationContext(new Context(Logger)); await server.StartupAsync(context).ShouldBeSuccess(); var pat = Guid.NewGuid().ToString(); var publishingConfig = new PublishingConfigDummy { PublishAsynchronously = true, }; using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario); await clientCache.StartupAsync(context).ShouldBeSuccess(); var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session; await clientSession.StartupAsync(context).ShouldBeSuccess(); var piecesOfContent = 3; var putResults = await Task.WhenAll( Enumerable.Range(0, piecesOfContent) .Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess())); var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray()); var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1)); var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism); var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray()); var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 }); var strongFingerprint = new StrongFingerprint(fingerprint, selector); var cts = new CancellationTokenSource(); // Even though publishing is blocking, this should succeed because we're publishing asynchronously. await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess(); // Allow for the publishing operation to be registered. await Task.Delay(TimeSpan.FromSeconds(1)); // Simulate a restart. await server.ShutdownAsync(context).ShouldBeSuccess(); server.Dispose(); server = new LocalCacheServer( FileSystem, TestGlobal.Logger, scenario: scenario, cacheFactory: CreateBlockingPublishingCache, serverConfiguration, Capabilities.All); await server.StartupAsync(context).ShouldBeSuccess(); // Session should have been persisted. var sessionsAndDatas = server.GetCurrentSessions(); sessionsAndDatas.Length.Should().Be(1); var serverSession = sessionsAndDatas[0].session; var data = sessionsAndDatas[0].data; var operation = new PublishingOperation { ContentHashListWithDeterminism = contentHashListwithDeterminism, StrongFingerprint = strongFingerprint }; var operations = new[] { operation }; data.Name.Should().Be(clientSession.Name); data.Pat.Should().Be(pat); data.Capabilities.Should().Be(Capabilities.All); data.ImplicitPin.Should().Be(ImplicitPin.None); data.Pins.Should().BeEquivalentTo(new List <string>()); data.PublishingConfig.Should().BeEquivalentTo(publishingConfig); data.PendingPublishingOperations.Should().BeEquivalentTo(operations); var hibernateSession = serverSession as IHibernateCacheSession; hibernateSession.Should().NotBeNull(); var actualPending = hibernateSession.GetPendingPublishingOperations(); actualPending.Should().BeEquivalentTo(operations); await server.ShutdownAsync(context).ShouldBeSuccess(); await clientSession.ShutdownAsync(context).ShouldBeSuccess(); }