private async Task LongCallCanceledOnShutdownAsync <T>(
string scenario, bool respectsCancellationToken, Func <OperationContext, AbsolutePath, IRpcClient, Task <T> > unresponsiveFunc)
{
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
// Use a store which gets stuck forever on the session operations (pin/place/put/open) unless canceled,
// and configurably ignoring the cancellation token to simulate long operations that might not check
// the token quickly enough.
var unresponsivenessHasStartedSemaphore = new SemaphoreSlim(0, 1);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path => new TestHangingContentStore(respectsCancellationToken, unresponsivenessHasStartedSemaphore);
IRpcClient rpcClient;
Task <T> unresponsiveTask;
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario);
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
// Start the task which we expect to become unresponsive
unresponsiveTask = unresponsiveFunc(context, directory.CreateRandomFileName(), rpcClient);
// Synchronize with the store's unresponsiveness.
await unresponsivenessHasStartedSemaphore.WaitAsync();
await server.ShutdownAsync(context).ShouldBeSuccess();
}
// Make sure that the client gets a retryable exception
Func <Task> awaitUnresponsiveTaskFunc = async() => await unresponsiveTask;
awaitUnresponsiveTaskFunc.Should().Throw <ClientCanRetryException>();
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
rpcClient.Dispose();
}
}
protected override TestServiceClientContentStore CreateStore(
AbsolutePath rootPath,
ContentStoreConfiguration configuration,
LocalServerConfiguration localContentServerConfiguration,
TimeSpan?heartbeatOverride)
{
configuration.Write(FileSystem, rootPath);
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName, null, Scenario),
heartbeatOverride,
serviceConfiguration,
localContentServerConfiguration: localContentServerConfiguration
));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path;
configuration.Write(FileSystem, rootPath).Wait();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
MaxConnections,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestInProcessServiceClientContentStore(
FileSystem,
Logger,
CacheName,
Scenario,
null,
serviceConfiguration
));
}
private async Task RunTestCase(Func <AbsolutePath, IContentSession, GrpcCopyClient, Task> testAct, [CallerMemberName] string testName = null)
{
var cacheName = testName + "_cache";
testName += Guid.NewGuid(); // Using a guid to disambiguate scenario name.
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ cacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
42,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName)
{
CopyRequestHandlingCountLimit = _copyToLimit, ProactivePushCountLimit = _proactivePushCountLimit
};
var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
Func <AbsolutePath, IContentStore> contentStoreFactory = (path) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
directory.Path,
new ConfigurationModel(storeConfig));
var server = new LocalContentServer(FileSystem, Logger, testName, contentStoreFactory, new LocalServerConfiguration(configuration));
await server.StartupAsync(_context).ShouldBeSuccess();
var sessionData = new LocalContentServerSessionData(testName, Capabilities.ContentOnly, ImplicitPin.PutAndGet, pins: null);
var createSessionResult = await server.CreateSessionAsync(new OperationContext(_context), sessionData, cacheName);
createSessionResult.ShouldBeSuccess();
(int sessionId, _) = createSessionResult.Value;
using var sessionReference = server.GetSession(sessionId);
var session = sessionReference.Session;
// Create a GRPC client to connect to the server
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
await _clientCache.UseAsync(new OperationContext(_context), LocalHost, port, async (nestedContext, grpcCopyClient) =>
{
await testAct(rootPath, session, grpcCopyClient);
return(Unit.Void);
});
await server.ShutdownAsync(_context).ShouldBeSuccess();
}
}
private DistributedServiceConfiguration CreateRandomDistributedConfig(DisposableDirectory d, string stampId, string ringId)
{
return(new DistributedServiceConfiguration(
dataRootPath: d.Path / GetRandomFileName(),
gracefulShutdownSeconds: 5,
grpcPort: (uint)PortExtensions.GetNextAvailablePort(),
cacheName: ThreadSafeRandom.Generator.Next().ToString(),
cachePath: d.Path / GetRandomFileName(),
stampId: stampId,
ringId: ringId));
}
private async Task RunServerTestAsync(Context context, string scenario, Func <Context, ServiceConfiguration, IRpcClient, Task> funcAsync)
{
using (var directory = new DisposableDirectory(FileSystem))
{
var storeConfig = new ContentStoreConfiguration(new MaxSizeQuota($"{1 * 1024 * 1024}"));
await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);
var serviceConfig =
new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, directory.Path }
},
directory.Path,
ServiceConfiguration.DefaultMaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
Guid.NewGuid().ToString())
{
TraceGrpcOperation = true
};
using (var server = new LocalContentServer(FileSystem, Logger, scenario, path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path), new LocalServerConfiguration(serviceConfig)))
{
BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);
r.ShouldBeSuccess();
var configuration = new ServiceClientRpcConfiguration()
{
GrpcPort = (int)serviceConfig.GrpcPort,
};
using (var rpcClient = new GrpcContentClient(new ServiceClientContentSessionTracer(scenario), FileSystem, configuration, scenario))
{
try
{
var createSessionResult = await rpcClient.CreateSessionAsync(new OperationContext(context), SessionName, CacheName, ImplicitPin.None);
createSessionResult.ShouldBeSuccess();
await funcAsync(context, serviceConfig, rpcClient);
}
finally
{
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
}
}
r = await server.ShutdownAsync(context);
r.ShouldBeSuccess();
}
}
}
public async Task CreateSessionThrowsClientCanRetryExceptionWhenServiceOffline()
{
string scenario = nameof(CreateSessionThrowsClientCanRetryExceptionWhenServiceOffline);
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1KB"))));
IRpcClient rpcClient;
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);
r.ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario);
r = await server.ShutdownAsync(context);
r.ShouldBeSuccess();
}
Func <Task <BoolResult> > createSessionFunc = () => rpcClient.CreateSessionAsync(context, SessionName, CacheName, ImplicitPin.None);
createSessionFunc.Should().Throw <ClientCanRetryException>();
rpcClient.Dispose();
}
}
private async Task RunTestCase(string testName, Func <AbsolutePath, IContentSession, GrpcCopyClient, Task> testAct)
{
var cacheName = testName + "_cache";
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ cacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
42,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
Func <AbsolutePath, IContentStore> contentStoreFactory = (path) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
directory.Path,
new ConfigurationModel(storeConfig));
var server = new LocalContentServer(FileSystem, Logger, testName, contentStoreFactory, new LocalServerConfiguration(configuration));
await server.StartupAsync(_context).ShouldBeSuccess();
var createSessionResult = await server.CreateSessionAsync(new OperationContext(_context), testName, cacheName, ImplicitPin.PutAndGet, Capabilities.ContentOnly);
createSessionResult.ShouldBeSuccess();
(int sessionId, AbsolutePath tempDir) = createSessionResult.Value;
var session = server.GetSession(sessionId);
// Create a GRPC client to connect to the server
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
using (var clientWrapper = await _clientCache.CreateAsync(LocalHost, port, false))
{
// Run validation
await testAct(rootPath, session, clientWrapper.Value);
}
await server.ShutdownAsync(_context).ShouldBeSuccess();
}
}
public async Task WrongPort()
{
await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
{
// Copy fake file out via GRPC
var host = "localhost";
var bogusPort = PortExtensions.GetNextAvailablePort();
using (client = GrpcCopyClient.Create(host, bogusPort))
{
var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
}
});
}
public Task WrongPort()
{
return(RunTestCase(async(rootPath, session, client) =>
{
// Copy fake file out via GRPC
var bogusPort = PortExtensions.GetNextAvailablePort();
await _clientCache.UseAsync(new OperationContext(_context), LocalHost, bogusPort, async(nestedContext, client) =>
{
var copyFileResult = await client.CopyFileAsync(nestedContext, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), new CopyOptions(bandwidthConfiguration: null));
Assert.Equal(CopyResultCode.ServerUnavailable, copyFileResult.Code);
return Unit.Void;
});
}));
}
public async Task HibernationDataNotLoadedIfStoreStartupFails()
{
const string scenario = nameof(HibernationDataNotLoadedIfStoreStartupFails);
var fileName = $"{Guid.NewGuid()}.json";
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var contentHash = ContentHash.Random();
var pins = new List <string> {
contentHash.Serialize()
};
var hibernatedSessionInfo = new HibernatedContentSessionInfo(SessionId, SessionName, ImplicitPin.None, CacheName, pins, 0, Capabilities.None);
var hibernatedSessions = new HibernatedSessions <HibernatedContentSessionInfo>(new List <HibernatedContentSessionInfo> {
hibernatedSessionInfo
});
hibernatedSessions.Write(FileSystem, rootPath, fileName);
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(namedCacheRoots, rootPath, MaxConnections, GracefulShutdownSeconds, grpcPort, grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory =
(path) => new TestFailingContentStore();
var localContentServerConfiguration = new LocalServerConfiguration(configuration)
{
UnusedSessionHeartbeatTimeout = TimeSpan.FromSeconds(TimeoutSecs),
UnusedSessionTimeout = TimeSpan.FromSeconds(TimeoutSecs * 4),
RequestCallTokensPerCompletionQueue = 10,
};
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, localContentServerConfiguration))
{
var r = await server.StartupAsync(context);
r.ShouldBeError(TestFailingContentStore.FailureMessage);
FileSystem.HibernatedSessionsExists(rootPath, fileName).Should().BeTrue("The hibernation data should not have been read/deleted");
}
}
}
public async Task PutStreamRetriesWhenTempFileDisappears()
{
string scenario = nameof(PutStreamRetriesWhenTempFileDisappears);
var tracingContext = new Context(Logger);
var context = new OperationContext(tracingContext);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory = path => new TestTempFileDeletingContentStore(FileSystem);
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
using (IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario))
{
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
using (var stream = new MemoryStream())
{
Func <Task <PutResult> > putFunc = () => rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory: false);
putFunc.Should().Throw <ClientCanRetryException>();
}
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
}
await server.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
public async Task WrongPort()
{
await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
{
// Copy fake file out via GRPC
var bogusPort = PortExtensions.GetNextAvailablePort();
using (var clientWrapper = await _clientCache.CreateAsync(LocalHost, bogusPort, true))
{
// Replace the given client with a bogus one
client = clientWrapper.Value;
var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
}
});
}
protected override IContentStore CreateStore(AbsolutePath rootPath, string cacheName, ContentStoreConfiguration configuration)
{
configuration.Write(FileSystem, rootPath);
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfig = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ cacheName, rootPath }
},
rootPath,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestInProcessServiceClientContentStore(
FileSystem, Logger, cacheName, _scenario, null, serviceConfig));
}
public async Task WrongPort()
{
await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
{
// Write a random file to put into the cache
var content = ThreadSafeRandom.GetBytes(FileSize);
var contentHash = HashingExtensions.CalculateHash(content, HashType.Vso0);
// Copy the file out via GRPC
var host = "localhost";
var bogusPort = PortExtensions.GetNextAvailablePort();
using (client = GrpcCopyClient.Create(host, bogusPort))
{
var copyFileResult = await client.CopyFileAsync(_context, contentHash, rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
}
});
}
/// <inheritdoc />
protected override ICache CreateCache(DisposableDirectory testDirectory)
{
var backendCacheDirectory = testDirectory.Path / "Backend";
FileSystem.CreateDirectory(backendCacheDirectory);
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
[CacheName] = backendCacheDirectory / "Root"
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var serverConfiguration = new LocalServerConfiguration(backendCacheDirectory, namedCacheRoots, grpcPort, FileSystem)
{
GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
};
var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(CacheName, new ServiceClientRpcConfiguration(serverConfiguration.GrpcPort), "Scenario-" + Guid.NewGuid());
Func <AbsolutePath, ICache> contentStoreFactory = CreateBackendCache;
var serviceClient = new TestInProcessServiceClientCache(Logger, FileSystem, contentStoreFactory, serverConfiguration, serviceClientConfiguration);
return(serviceClient);
}
private async Task RunClientWhenServiceDownTestAsync(Context context, Func <Context, GrpcRepairClient, Task> funcAsync)
{
var exceptionThrown = false;
using (var directory = new DisposableDirectory(FileSystem))
{
var configuration = CreateServiceConfiguration(directory.Path, PortExtensions.GetNextAvailablePort(), Guid.NewGuid().ToString());
var client = new GrpcRepairClient(configuration.GrpcPort);
try
{
await funcAsync(context, client);
}
catch (ClientCanRetryException)
{
exceptionThrown = true;
}
Assert.True(exceptionThrown);
}
}
protected override IContentStore CreateStore(AbsolutePath rootPath, ContentStoreConfiguration configuration)
{
configuration.Write(FileSystem, rootPath).Wait();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
MaxConnections,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName, null, Scenario),
null,
serviceConfiguration));
}
private async Task RunServerTestAsync(Context context, Func <Context, ServiceConfiguration, Task> funcAsync)
{
using (var directory = new DisposableDirectory(FileSystem))
{
var storeConfig = CreateStoreConfiguration();
await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);
var serviceConfig = CreateServiceConfiguration(directory.Path, PortExtensions.GetNextAvailablePort(), Guid.NewGuid().ToString());
using (var server = new ServiceProcess(serviceConfig, LocalContentServerConfiguration, Scenario, WaitForServerReadyTimeoutMs, WaitForExitTimeoutMs))
{
BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);
r.ShouldBeSuccess();
await funcAsync(context, serviceConfig);
r = await server.ShutdownAsync(context);
r.ShouldBeSuccess();
}
}
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path;
configuration.Write(FileSystem, rootPath);
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfig = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName, rpcConfiguration: null, scenario: Scenario),
heartbeatInterval: null,
serviceConfiguration: serviceConfig));
}
public async Task CacheSessionDataIsHibernated()
{
using var testDirectory = new DisposableDirectory(FileSystem);
var cacheDirectory = testDirectory.Path / "Service";
var cacheName = "theCache";
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
[cacheName] = cacheDirectory / "Root"
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem)
{
GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
};
var scenario = "Default";
var server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
var context = new OperationContext(new Context(Logger));
await server.StartupAsync(context).ShouldBeSuccess();
var pat = Guid.NewGuid().ToString();
var publishingConfig = new PublishingConfigDummy
{
PublishAsynchronously = true,
};
using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario);
await clientCache.StartupAsync(context).ShouldBeSuccess();
var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session;
await clientSession.StartupAsync(context).ShouldBeSuccess();
var piecesOfContent = 3;
var putResults = await Task.WhenAll(
Enumerable.Range(0, piecesOfContent)
.Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess()));
var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray());
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1));
var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 });
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
var cts = new CancellationTokenSource();
// Even though publishing is blocking, this should succeed because we're publishing asynchronously.
await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess();
// Allow for the publishing operation to be registered.
await Task.Delay(TimeSpan.FromSeconds(1));
// Simulate a restart.
await server.ShutdownAsync(context).ShouldBeSuccess();
server.Dispose();
server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario: scenario,
cacheFactory: CreateBlockingPublishingCache,
serverConfiguration,
Capabilities.All);
await server.StartupAsync(context).ShouldBeSuccess();
// Session should have been persisted.
var sessionsAndDatas = server.GetCurrentSessions();
sessionsAndDatas.Length.Should().Be(1);
var serverSession = sessionsAndDatas[0].session;
var data = sessionsAndDatas[0].data;
var operation = new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListwithDeterminism,
StrongFingerprint = strongFingerprint
};
var operations = new[] { operation };
data.Name.Should().Be(clientSession.Name);
data.Pat.Should().Be(pat);
data.Capabilities.Should().Be(Capabilities.All);
data.ImplicitPin.Should().Be(ImplicitPin.None);
data.Pins.Should().BeEquivalentTo(new List <string>());
data.PublishingConfig.Should().BeEquivalentTo(publishingConfig);
data.PendingPublishingOperations.Should().BeEquivalentTo(operations);
var hibernateSession = serverSession as IHibernateCacheSession;
hibernateSession.Should().NotBeNull();
var actualPending = hibernateSession.GetPendingPublishingOperations();
actualPending.Should().BeEquivalentTo(operations);
await server.ShutdownAsync(context).ShouldBeSuccess();
await clientSession.ShutdownAsync(context).ShouldBeSuccess();
}
// Setup all grpc ports before creating stores so they know how to talk to each other.
protected override object PrepareAdditionalCreateStoreArgs(int storeCount)
{
return(Enumerable.Range(0, storeCount).Select(i => PortExtensions.GetNextAvailablePort()).ToArray());
}
public async Task ServiceTestAsync()
{
using var fileSystem = new PassThroughFileSystem(Logger);
using var dir = new DisposableDirectory(fileSystem);
var cacheDir = dir.Path / "cache";
var dataPath = dir.Path / "data";
var port = PortExtensions.GetNextAvailablePort();
var args = new Dictionary <string, string>
{
["paths"] = cacheDir.Path,
["names"] = "Default",
["grpcPort"] = port.ToString(),
["LogSeverity"] = "Diagnostic",
["dataRootPath"] = dataPath.Path,
["Scenario"] = "AppTests",
["grpcPortFileName"] = "AppTestsMMF"
};
var serviceProcess = RunService("Service", args, Logger);
try
{
await RunAppAsync("ServiceRunning", new Dictionary <string, string> {
{ "waitSeconds", "5" }, { "Scenario", "AppTests" }
}, Logger);
var context = new Context(Logger);
var config = new ServiceClientContentStoreConfiguration("Default", new ServiceClientRpcConfiguration {
GrpcPort = port
}, scenario: "AppTests");
using var store = new ServiceClientContentStore(Logger, fileSystem, config);
await store.StartupAsync(context).ShouldBeSuccess();
var sessionResult = store.CreateSession(context, "Default", ImplicitPin.None).ShouldBeSuccess();
using var session = sessionResult.Session;
await session.StartupAsync(context).ShouldBeSuccess();
var source = dir.Path / "source.txt";
var contents = new byte[1024];
Random.NextBytes(contents);
fileSystem.WriteAllBytes(source, contents);
var putResult = await session.PutFileAsync(context, HashType.MD5, source, FileRealizationMode.Any, CancellationToken.None).ShouldBeSuccess();
var hash = putResult.ContentHash;
await session.PinAsync(context, hash, CancellationToken.None).ShouldBeSuccess();
var destination = dir.Path / "destination.txt";
await session.PlaceFileAsync(
context,
hash,
destination,
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.Any,
CancellationToken.None).ShouldBeSuccess();
fileSystem.ReadAllBytes(destination).Should().BeEquivalentTo(contents);
}
finally
{
if (!serviceProcess.HasExited)
{
serviceProcess.Kill();
#pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead
serviceProcess.WaitForExit();
#pragma warning restore AsyncFixer02
}
}
}
[Trait("Category", "QTestSkip")] // Skipped
public async Task RestoredSessionReleasedAfterInactivity()
{
const string scenario = nameof(RestoredSessionReleasedAfterInactivity);
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var contentHash = ContentHash.Random();
var pins = new List <string> {
contentHash.Serialize()
};
var hibernatedSessionInfo = new HibernatedSessionInfo(SessionId, SessionName, ImplicitPin.None, CacheName, pins, DateTime.UtcNow.Ticks, Capabilities.None);
var hibernatedSessions = new HibernatedSessions(new List <HibernatedSessionInfo> {
hibernatedSessionInfo
});
await hibernatedSessions.WriteAsync(FileSystem, rootPath);
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(namedCacheRoots, rootPath, MaxConnections, GracefulShutdownSeconds, grpcPort, grpcPortFileName);
var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
Func <AbsolutePath, IContentStore> contentStoreFactory = (path) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
directory.Path,
new ConfigurationModel(storeConfig));
var localContentServerConfiguration = new LocalServerConfiguration(configuration)
{
UnusedSessionHeartbeatTimeout = TimeSpan.FromSeconds(TimeoutSecs),
RequestCallTokensPerCompletionQueue = 10,
};
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, localContentServerConfiguration))
{
var r1 = await server.StartupAsync(context);
r1.ShouldBeSuccess();
var beforeIds = server.GetSessionIds();
beforeIds.Should().Contain(SessionId);
// Wait one period to ensure that it times out, another to ensure that the checker finds it, and another to give it time to release it.
await Task.Delay(TimeSpan.FromSeconds(TimeoutSecs * 3));
var afterIds = server.GetSessionIds();
afterIds.Count.Should().Be(0);
var r2 = await server.ShutdownAsync(context);
r2.ShouldBeSuccess();
}
}
}
public async Task MultipleCaches()
{
const string CacheName1 = "test1";
const string CacheName2 = "test2";
using (var testDirectory0 = new DisposableDirectory(FileSystem))
using (var testDirectory1 = new DisposableDirectory(FileSystem))
using (var testDirectory2 = new DisposableDirectory(FileSystem))
{
var config = CreateStoreConfiguration();
var rootPath1 = testDirectory1.Path;
config.Write(FileSystem, rootPath1);
var rootPath2 = testDirectory2.Path;
config.Write(FileSystem, rootPath2);
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName1, rootPath1 }, { CacheName2, rootPath2 }
},
testDirectory0.Path,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
using (var server = CreateServer(serviceConfiguration))
{
var factory = new MemoryMappedFileGrpcPortSharingFactory(Logger, grpcPortFileName);
var reader = factory.GetPortReader();
var port = reader.ReadPort();
var rpcConfig = new ServiceClientRpcConfiguration(port);
using (var store1 = new ServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName1, rpcConfig, Scenario)))
using (var store2 = new ServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName2, rpcConfig, Scenario)))
{
try
{
var rs = await server.StartupAsync(_context);
rs.ShouldBeSuccess();
var storeBoolResult1 = await store1.StartupAsync(_context);
storeBoolResult1.ShouldBeSuccess();
var storeBoolResult2 = await store2.StartupAsync(_context);
storeBoolResult2.ShouldBeSuccess();
IContentSession session1 = null;
IContentSession session2 = null;
try
{
var createSessionResult1 = store1.CreateSession(_context, "session1", ImplicitPin.None);
createSessionResult1.ShouldBeSuccess();
var createSessionResult2 = store2.CreateSession(_context, "session2", ImplicitPin.None);
createSessionResult2.ShouldBeSuccess();
using (createSessionResult1.Session)
using (createSessionResult2.Session)
{
var r1 = await createSessionResult1.Session.StartupAsync(_context);
r1.ShouldBeSuccess();
session1 = createSessionResult1.Session;
var r2 = await createSessionResult2.Session.StartupAsync(_context);
r2.ShouldBeSuccess();
session2 = createSessionResult2.Session;
var r3 = await session1.PutRandomAsync(
_context,
ContentHashType,
false,
RandomContentByteCount,
Token);
var pinResult = await session1.PinAsync(_context, r3.ContentHash, Token);
pinResult.ShouldBeSuccess();
r3 = await session2.PutRandomAsync(
_context,
ContentHashType,
false,
RandomContentByteCount,
Token);
pinResult = await session2.PinAsync(_context, r3.ContentHash, Token);
pinResult.ShouldBeSuccess();
}
}
finally
{
if (session2 != null)
{
await session2.ShutdownAsync(_context).ShouldBeSuccess();
}
if (session1 != null)
{
await session1.ShutdownAsync(_context).ShouldBeSuccess();
}
}
}
finally
{
BoolResult r1 = null;
BoolResult r2 = null;
if (store1.StartupCompleted)
{
r1 = await store1.ShutdownAsync(_context);
}
if (store2.StartupCompleted)
{
r2 = await store2.ShutdownAsync(_context);
}
var r3 = await server.ShutdownAsync(_context);
r1?.ShouldBeSuccess();
r2?.ShouldBeSuccess();
r3?.ShouldBeSuccess();
}
}
}
}
}
public async Task DoesNotRespectFileReplacementMode(FileReplacementMode requestedReplacementMode)
{
string scenario = nameof(DoesNotRespectFileReplacementMode) + requestedReplacementMode;
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var rootPath = directory.Path;
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var grpcPortFileName = Guid.NewGuid().ToString();
var configuration = new ServiceConfiguration(
namedCacheRoots,
rootPath,
MaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPort,
grpcPortFileName);
Func <AbsolutePath, IContentStore> contentStoreFactory =
path =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1MB"))));
using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
{
var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
await server.StartupAsync(context).ShouldBeSuccess();
var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, port, scenario);
await rpcClient.CreateSessionAsync(
context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();
ContentHash contentHash;
using (var stream = new MemoryStream())
{
PutResult putResult = await rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory : false);
putResult.ShouldBeSuccess();
putResult.ContentSize.Should().Be(0);
contentHash = putResult.ContentHash;
}
var tempPath = directory.CreateRandomFileName();
FileSystem.WriteAllBytes(tempPath, new byte[] {});
var placeResult = await rpcClient.PlaceFileAsync(
context, contentHash, tempPath, FileAccessMode.ReadOnly, requestedReplacementMode, FileRealizationMode.Any);
placeResult.Succeeded.Should().BeTrue();
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
rpcClient.Dispose();
await server.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
private void StartRedisServerIfNeeded()
{
// Can reuse an existing process only when this instance successfully created a connection to it.
// Otherwise the test will fail with NRE.
if (_process != null && _connectionMultiplexer != null)
{
_logger.Debug("Redis process is already running. Reusing an existing instance.");
return;
}
_logger.Debug("Starting a redis server.");
var redisName = OperatingSystemHelper.IsWindowsOS ? "redis-server.exe" : "redis-server";
string redisServerPath = Path.GetFullPath(Path.Combine("redisServer", redisName));
if (!File.Exists(redisServerPath))
{
throw new InvalidOperationException($"Could not find {redisName} at {redisServerPath}");
}
int portNumber = 0;
const int maxRetries = 10;
for (int i = 0; i < maxRetries; i++)
{
var fileName = _tempDirectory.CreateRandomFileName();
var redisServerLogsPath = _tempDirectory.CreateRandomFileName();
_fileSystem.CreateDirectory(redisServerLogsPath);
portNumber = PortExtensions.GetNextAvailablePort();
string newConfig = $@"
timeout 0
tcp-keepalive 0
dir {redisServerLogsPath}
port {portNumber}";
File.WriteAllText(fileName.Path, newConfig);
var args = $" {fileName}";
_logger.Debug($"Running cmd=[{redisServerPath} {args}]");
const bool createNoWindow = true;
_process = new ProcessUtility(redisServerPath, args, createNoWindow, workingDirectory: Path.GetDirectoryName(redisServerPath));
_process.Start();
string processOutput;
if (_process == null)
{
processOutput = "[Process could not start]";
throw new InvalidOperationException(processOutput);
}
if (createNoWindow)
{
if (_process.HasExited)
{
if (_process.WaitForExit(5000))
{
throw new InvalidOperationException(_process.GetLogs());
}
throw new InvalidOperationException("Process or either wait handle timed out. " + _process.GetLogs());
}
processOutput = $"[Process {_process.Id} is still running]";
}
_logger.Debug("Process output: " + processOutput);
ConnectionString = $"localhost:{portNumber}";
ConfigurationOptions options = ConfigurationOptions.Parse(ConnectionString);
options.ConnectTimeout = 2000;
options.SyncTimeout = 2000;
try
{
_connectionMultiplexer = ConnectionMultiplexer.Connect(options);
break;
}
catch (RedisConnectionException ex)
{
SafeKillProcess();
_logger.Debug($"Retrying for exception connecting to redis process {_process.Id} with port {portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");
if (i != maxRetries - 1)
{
Thread.Sleep(300);
}
else
{
throw;
}
}
catch (Exception ex)
{
SafeKillProcess();
_logger.Error(
$"Exception connecting to redis process {_process.Id} with port {portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");
throw;
}
}
_logger.Debug($"Redis server {_process.Id} is up and running at port {portNumber}.");
}
private void Start()
{
// Can reuse an existing process only when this instance successfully created a connection to it.
// Otherwise the test will fail with NRE.
if (_process != null)
{
_logger.Debug("Storage process is already running. Reusing an existing instance.");
return;
}
_logger.Debug("Starting a storage server.");
var storageName = (OperatingSystemHelper.IsWindowsOS ? "tools/win-x64/blob.exe"
: (OperatingSystemHelper.IsLinuxOS ? "tools/linux-x64/blob"
: "tools/osx-x64/blob"));
string storageServerPath = Path.GetFullPath(Path.Combine("azurite", storageName));
if (!File.Exists(storageServerPath))
{
throw new InvalidOperationException($"Could not find {storageName} at {storageServerPath}");
}
_portNumber = 0;
const int maxRetries = 10;
for (int i = 0; i < maxRetries; i++)
{
var storageServerWorkspacePath = _tempDirectory.CreateRandomFileName();
_fileSystem.CreateDirectory(storageServerWorkspacePath);
_portNumber = PortExtensions.GetNextAvailablePort();
var args = $"--blobPort {_portNumber} --location {storageServerWorkspacePath}";
_logger.Debug($"Running cmd=[{storageServerPath} {args}]");
_process = new ProcessUtility(storageServerPath, args, createNoWindow: true, workingDirectory: Path.GetDirectoryName(storageServerPath));
_process.Start();
string processOutput;
if (_process == null)
{
processOutput = "[Process could not start]";
throw new InvalidOperationException(processOutput);
}
if (_process.HasExited)
{
if (_process.WaitForExit(5000))
{
throw new InvalidOperationException(_process.GetLogs());
}
throw new InvalidOperationException("Process or either wait handle timed out. " + _process.GetLogs());
}
processOutput = $"[Process {_process.Id} is still running]";
_logger.Debug("Process output: " + processOutput);
ConnectionString = $"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:{_portNumber}/devstoreaccount1;";
AzureBlobStorageCredentials creds = new AzureBlobStorageCredentials(ConnectionString);
var client = creds.CreateCloudBlobClient();
try
{
bool exists = client.GetContainerReference("test").ExistsAsync(DefaultBlobStorageRequestOptions, null).GetAwaiter().GetResult();
break;
}
catch (StorageException ex)
{
SafeKillProcess();
_logger.Debug($"Retrying for exception connecting to storage process {_process.Id} with port {_portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");
if (i != maxRetries - 1)
{
Thread.Sleep(300);
}
else
{
throw;
}
}
catch (Exception ex)
{
SafeKillProcess();
_logger.Error(
$"Exception connecting to storage process {_process.Id} with port {_portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");
throw;
}
}
_logger.Debug($"Storage server {_process.Id} is up and running at port {_portNumber}.");
}