private async Task LongCallCanceledOnShutdownAsync <T>(
string scenario, bool respectsCancellationToken, Func <OperationContext, AbsolutePath, IRpcClient, Task <T> > unresponsiveFunc)
{
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
// Use a store which gets stuck forever on the session operations (pin/place/put/open) unless canceled,
// and configurably ignoring the cancellation token to simulate long operations that might not check
// the token quickly enough.
var unresponsivenessHasStartedSemaphore = new SemaphoreSlim(0, 1);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path => new TestHangingContentStore(respectsCancellationToken, unresponsivenessHasStartedSemaphore);
IRpcClient rpcClient;
Task <T> unresponsiveTask;
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario);
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
// Start the task which we expect to become unresponsive
unresponsiveTask = unresponsiveFunc(context, directory.CreateRandomFileName(), rpcClient);
// Synchronize with the store's unresponsiveness.
await unresponsivenessHasStartedSemaphore.WaitAsync();
await server.ShutdownAsync(context).ShouldBeSuccess();
}
// Make sure that the client gets a retryable exception
Func <Task> awaitUnresponsiveTaskFunc = async() => await unresponsiveTask;
awaitUnresponsiveTaskFunc.Should().Throw <ClientCanRetryException>();
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
rpcClient.Dispose();
}
}
private async Task RunServerTestAsync(Context context, string scenario, Func <Context, ServiceConfiguration, IRpcClient, Task> funcAsync)
{
using (var directory = new DisposableDirectory(FileSystem))
{
var storeConfig = new ContentStoreConfiguration(new MaxSizeQuota($"{1 * 1024 * 1024}"));
await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);
var serviceConfig =
new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, directory.Path }
},
directory.Path,
ServiceConfiguration.DefaultMaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
Guid.NewGuid().ToString())
{
TraceGrpcOperation = true
};
using (var server = new LocalContentServer(FileSystem, Logger, scenario, path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path), new LocalServerConfiguration(serviceConfig)))
{
BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);
r.ShouldBeSuccess();
var configuration = new ServiceClientRpcConfiguration()
{
GrpcPort = (int)serviceConfig.GrpcPort,
};
using (var rpcClient = new GrpcContentClient(new ServiceClientContentSessionTracer(scenario), FileSystem, configuration, scenario))
{
try
{
var createSessionResult = await rpcClient.CreateSessionAsync(new OperationContext(context), SessionName, CacheName, ImplicitPin.None);
createSessionResult.ShouldBeSuccess();
await funcAsync(context, serviceConfig, rpcClient);
}
finally
{
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
}
}
r = await server.ShutdownAsync(context);
r.ShouldBeSuccess();
}
}
}
public async Task CreateSessionThrowsClientCanRetryExceptionWhenServiceOffline()
{
string scenario = nameof(CreateSessionThrowsClientCanRetryExceptionWhenServiceOffline);
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1KB"))));
IRpcClient rpcClient;
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);
r.ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario);
r = await server.ShutdownAsync(context);
r.ShouldBeSuccess();
}
Func <Task <BoolResult> > createSessionFunc = () => rpcClient.CreateSessionAsync(context, SessionName, CacheName, ImplicitPin.None);
createSessionFunc.Should().Throw <ClientCanRetryException>();
rpcClient.Dispose();
}
}
public async Task PutStreamRetriesWhenTempFileDisappears()
{
string scenario = nameof(PutStreamRetriesWhenTempFileDisappears);
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory = path => new TestTempFileDeletingContentStore(FileSystem);
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
using (IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario))
{
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
using (var stream = new MemoryStream())
{
Func <Task <PutResult> > putFunc = () => rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory: false);
putFunc.Should().Throw <ClientCanRetryException>();
}
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
}
await server.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
public async Task DoesNotRespectFileReplacementMode(FileReplacementMode requestedReplacementMode)
{
string scenario = nameof(DoesNotRespectFileReplacementMode) + requestedReplacementMode;
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1MB"))));
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, port, scenario);
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
ContentHash contentHash;
using (var stream = new MemoryStream())
{
PutResult putResult = await rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory : false);
putResult.ShouldBeSuccess();
putResult.ContentSize.Should().Be(0);
contentHash = putResult.ContentHash;
}
var tempPath = directory.CreateRandomFileName();
FileSystem.WriteAllBytes(tempPath, new byte[] {});
var placeResult = await rpcClient.PlaceFileAsync(
context, contentHash, tempPath, FileAccessMode.ReadOnly, requestedReplacementMode, FileRealizationMode.Any);
placeResult.Succeeded.Should().BeTrue();
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
rpcClient.Dispose();
await server.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
