private async Task <BoolResult> PublishContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashList,
CancellationToken token)
{
Contract.Assert(!_hasBeenMarkedForShutdown, "Should not queue publish operations after the session has been marked for shutdown");
var publishingOperation = new PublishingOperation
{
StrongFingerprint = strongFingerprint,
ContentHashListWithDeterminism = contentHashList
};
var addResult = _pendingPublishingOperations.TryAdd(publishingOperation, Interlocked.Increment(ref _orderNumber));
if (!addResult)
{
// Since the operation is already pending, we return success. It's important to note that ordering matters, since
// publishing can overwrite the remote.
// Example of how ordering matters:
// These operations are queued in this order.
// Publish SF(A) with CHL(A)
// Publish SF(A) with CHL(B)
// Publish SF(A) with CHL(A)
// If we followed the queuing order and the remote was being overwritten by every call, the end result is SF(A)->CHL(A).
// If we eliminate duplicates, and the first operation is still pending while we queue the third one, the result is SF(A)->CHL(B).
// For now, we are willing to take chanses and ocassionally publish CHLs out of order.
return(BoolResult.Success);
}
var result = await _remote.PublishContentHashListAsync(
context,
strongFingerprint,
contentHashList,
token);
_pendingPublishingOperations.TryRemove(publishingOperation, out _);
// Make sure that we unblock shutdown once all publishing operations have been completed.
if (_hasBeenMarkedForShutdown && _pendingPublishingOperations.Count == 0)
{
_readyForShutdown.TrySetResult(true);
}
return(result);
}
private async Task <BoolResult> PublishContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashList,
CancellationToken token)
{
var publishingOperation = new PublishingOperation
{
StrongFingerprint = strongFingerprint,
ContentHashListWithDeterminism = contentHashList
};
var addResult = _pendingPublishingOperations.TryAdd(publishingOperation, Interlocked.Increment(ref _orderNumber));
if (!addResult)
{
// Since the operation is already pending, we return success. It's important to note that ordering matters, since
// publishing can overwrite the remote.
// Example of how ordering matters:
// These operations are queued in this order.
// Publish SF(A) with CHL(A)
// Publish SF(A) with CHL(B)
// Publish SF(A) with CHL(A)
// If we followed the queuing order and the remote was being overwritten by every call, the end result is SF(A)->CHL(A).
// If we eliminate duplicates, and the first operation is still pending while we queue the third one, the result is SF(A)->CHL(B).
// For now, we are willing to take chanses and ocassionally publish CHLs out of order.
return(BoolResult.Success);
}
var result = await _remote.PublishContentHashListAsync(
context,
strongFingerprint,
contentHashList,
token);
_pendingPublishingOperations.TryRemove(publishingOperation, out _);
return(result);
}
public async Task CacheSessionDataIsHibernated()
{
using var testDirectory = new DisposableDirectory(FileSystem);
var cacheDirectory = testDirectory.Path / "Service";
var cacheName = "theCache";
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
[cacheName] = cacheDirectory / "Root"
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem)
{
GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
};
var scenario = "Default";
var server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
var context = new OperationContext(new Context(Logger));
await server.StartupAsync(context).ShouldBeSuccess();
var pat = Guid.NewGuid().ToString();
var publishingConfig = new PublishingConfigDummy
{
PublishAsynchronously = true,
};
using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario);
await clientCache.StartupAsync(context).ShouldBeSuccess();
var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session;
await clientSession.StartupAsync(context).ShouldBeSuccess();
var piecesOfContent = 3;
var putResults = await Task.WhenAll(
Enumerable.Range(0, piecesOfContent)
.Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess()));
var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray());
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1));
var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 });
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
var cts = new CancellationTokenSource();
// Even though publishing is blocking, this should succeed because we're publishing asynchronously.
await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess();
// Allow for the publishing operation to be registered.
await Task.Delay(TimeSpan.FromSeconds(1));
// Simulate a restart.
await server.ShutdownAsync(context).ShouldBeSuccess();
server.Dispose();
server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario: scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
await server.StartupAsync(context).ShouldBeSuccess();
// Session should have been persisted.
var sessionsAndDatas = server.GetCurrentSessions();
sessionsAndDatas.Length.Should().Be(1);
var serverSession = sessionsAndDatas[0].session;
var data = sessionsAndDatas[0].data;
var operation = new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListwithDeterminism,
StrongFingerprint = strongFingerprint
};
var operations = new[] { operation };
data.Name.Should().Be(clientSession.Name);
data.Pat.Should().Be(pat);
data.Capabilities.Should().Be(Capabilities.All);
data.ImplicitPin.Should().Be(ImplicitPin.None);
data.Pins.Should().BeEquivalentTo(new List <string>());
data.PublishingConfig.Should().BeEquivalentTo(publishingConfig);
data.PendingPublishingOperations.Should().BeEquivalentTo(operations);
var hibernateSession = serverSession as IHibernateCacheSession;
hibernateSession.Should().NotBeNull();
var actualPending = hibernateSession.GetPendingPublishingOperations();
actualPending.Should().BeEquivalentTo(operations);
await server.ShutdownAsync(context).ShouldBeSuccess();
await clientSession.ShutdownAsync(context).ShouldBeSuccess();
}
