private async Task LongCallCanceledOnShutdownAsync <T>(
            string scenario, bool respectsCancellationToken, Func <OperationContext, AbsolutePath, IRpcClient, Task <T> > unresponsiveFunc)
        {
            var tracingContext = new Context(Logger);
            var context        = new OperationContext(tracingContext);

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath = directory.Path;

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { CacheName, rootPath }
                };
                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();
                var configuration    = new ServiceConfiguration(
                    namedCacheRoots,
                    rootPath,
                    MaxConnections,
                    ServiceConfiguration.DefaultGracefulShutdownSeconds,
                    grpcPort,
                    grpcPortFileName);

                // Use a store which gets stuck forever on the session operations (pin/place/put/open) unless canceled,
                // and configurably ignoring the cancellation token to simulate long operations that might not check
                // the token quickly enough.
                var unresponsivenessHasStartedSemaphore = new SemaphoreSlim(0, 1);
                Func <AbsolutePath, IContentStore> contentStoreFactory =
                    path => new TestHangingContentStore(respectsCancellationToken, unresponsivenessHasStartedSemaphore);

                IRpcClient rpcClient;
                Task <T>   unresponsiveTask;
                using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
                {
                    var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
                    await server.StartupAsync(context).ShouldBeSuccess();

                    var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
                    rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario);

                    await rpcClient.CreateSessionAsync(
                        context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();

                    // Start the task which we expect to become unresponsive
                    unresponsiveTask = unresponsiveFunc(context, directory.CreateRandomFileName(), rpcClient);

                    // Synchronize with the store's unresponsiveness.
                    await unresponsivenessHasStartedSemaphore.WaitAsync();

                    await server.ShutdownAsync(context).ShouldBeSuccess();
                }

                // Make sure that the client gets a retryable exception
                Func <Task> awaitUnresponsiveTaskFunc = async() => await unresponsiveTask;
                awaitUnresponsiveTaskFunc.Should().Throw <ClientCanRetryException>();

                (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
                rpcClient.Dispose();
            }
        }
Exemple #2
0
        protected override TestServiceClientContentStore CreateStore(
            AbsolutePath rootPath,
            ContentStoreConfiguration configuration,
            LocalServerConfiguration localContentServerConfiguration,
            TimeSpan?heartbeatOverride)
        {
            configuration.Write(FileSystem, rootPath);

            var grpcPortFileName     = Guid.NewGuid().ToString();
            var serviceConfiguration = new ServiceConfiguration(
                new Dictionary <string, AbsolutePath> {
                { CacheName, rootPath }
            },
                rootPath,
                GracefulShutdownSeconds,
                PortExtensions.GetNextAvailablePort(),
                grpcPortFileName);

            return(new TestServiceClientContentStore(
                       Logger,
                       FileSystem,
                       new ServiceClientContentStoreConfiguration(CacheName, null, Scenario),
                       heartbeatOverride,
                       serviceConfiguration,
                       localContentServerConfiguration: localContentServerConfiguration
                       ));
        }
Exemple #3
0
        protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
        {
            var rootPath = testDirectory.Path;

            configuration.Write(FileSystem, rootPath).Wait();

            var grpcPortFileName = Guid.NewGuid().ToString();

            var serviceConfiguration = new ServiceConfiguration(
                new Dictionary <string, AbsolutePath> {
                { CacheName, rootPath }
            },
                rootPath,
                MaxConnections,
                GracefulShutdownSeconds,
                PortExtensions.GetNextAvailablePort(),
                grpcPortFileName);

            return(new TestInProcessServiceClientContentStore(
                       FileSystem,
                       Logger,
                       CacheName,
                       Scenario,
                       null,
                       serviceConfiguration
                       ));
        }
        private async Task RunTestCase(Func <AbsolutePath, IContentSession, GrpcCopyClient, Task> testAct, [CallerMemberName] string testName = null)
        {
            var cacheName = testName + "_cache";

            testName += Guid.NewGuid(); // Using a guid to disambiguate scenario name.

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath = directory.Path;

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { cacheName, rootPath }
                };
                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();
                var configuration    = new ServiceConfiguration(
                    namedCacheRoots,
                    rootPath,
                    42,
                    ServiceConfiguration.DefaultGracefulShutdownSeconds,
                    grpcPort,
                    grpcPortFileName)
                {
                    CopyRequestHandlingCountLimit = _copyToLimit, ProactivePushCountLimit = _proactivePushCountLimit
                };

                var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
                Func <AbsolutePath, IContentStore> contentStoreFactory = (path) =>
                                                                         new FileSystemContentStore(
                    FileSystem,
                    SystemClock.Instance,
                    directory.Path,
                    new ConfigurationModel(storeConfig));

                var server = new LocalContentServer(FileSystem, Logger, testName, contentStoreFactory, new LocalServerConfiguration(configuration));

                await server.StartupAsync(_context).ShouldBeSuccess();

                var sessionData         = new LocalContentServerSessionData(testName, Capabilities.ContentOnly, ImplicitPin.PutAndGet, pins: null);
                var createSessionResult = await server.CreateSessionAsync(new OperationContext(_context), sessionData, cacheName);

                createSessionResult.ShouldBeSuccess();

                (int sessionId, _)         = createSessionResult.Value;
                using var sessionReference = server.GetSession(sessionId);
                var session = sessionReference.Session;

                // Create a GRPC client to connect to the server
                var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
                await _clientCache.UseAsync(new OperationContext(_context), LocalHost, port, async (nestedContext, grpcCopyClient) =>
                {
                    await testAct(rootPath, session, grpcCopyClient);
                    return(Unit.Void);
                });

                await server.ShutdownAsync(_context).ShouldBeSuccess();
            }
        }
Exemple #5
0
 private DistributedServiceConfiguration CreateRandomDistributedConfig(DisposableDirectory d, string stampId, string ringId)
 {
     return(new DistributedServiceConfiguration(
                dataRootPath: d.Path / GetRandomFileName(),
                gracefulShutdownSeconds: 5,
                grpcPort: (uint)PortExtensions.GetNextAvailablePort(),
                cacheName: ThreadSafeRandom.Generator.Next().ToString(),
                cachePath: d.Path / GetRandomFileName(),
                stampId: stampId,
                ringId: ringId));
 }
Exemple #6
0
        private async Task RunServerTestAsync(Context context, string scenario, Func <Context, ServiceConfiguration, IRpcClient, Task> funcAsync)
        {
            using (var directory = new DisposableDirectory(FileSystem))
            {
                var storeConfig = new ContentStoreConfiguration(new MaxSizeQuota($"{1 * 1024 * 1024}"));
                await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);

                var serviceConfig =
                    new ServiceConfiguration(
                        new Dictionary <string, AbsolutePath> {
                    { CacheName, directory.Path }
                },
                        directory.Path,
                        ServiceConfiguration.DefaultMaxConnections,
                        ServiceConfiguration.DefaultGracefulShutdownSeconds,
                        PortExtensions.GetNextAvailablePort(),
                        Guid.NewGuid().ToString())
                {
                    TraceGrpcOperation = true
                };

                using (var server = new LocalContentServer(FileSystem, Logger, scenario, path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path), new LocalServerConfiguration(serviceConfig)))
                {
                    BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);

                    r.ShouldBeSuccess();
                    var configuration = new ServiceClientRpcConfiguration()
                    {
                        GrpcPort = (int)serviceConfig.GrpcPort,
                    };

                    using (var rpcClient = new GrpcContentClient(new ServiceClientContentSessionTracer(scenario), FileSystem, configuration, scenario))
                    {
                        try
                        {
                            var createSessionResult = await rpcClient.CreateSessionAsync(new OperationContext(context), SessionName, CacheName, ImplicitPin.None);

                            createSessionResult.ShouldBeSuccess();

                            await funcAsync(context, serviceConfig, rpcClient);
                        }
                        finally
                        {
                            (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
                        }
                    }

                    r = await server.ShutdownAsync(context);

                    r.ShouldBeSuccess();
                }
            }
        }
        public async Task CreateSessionThrowsClientCanRetryExceptionWhenServiceOffline()
        {
            string scenario       = nameof(CreateSessionThrowsClientCanRetryExceptionWhenServiceOffline);
            var    tracingContext = new Context(Logger);
            var    context        = new OperationContext(tracingContext);

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath = directory.Path;

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { CacheName, rootPath }
                };
                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();
                var configuration    = new ServiceConfiguration(
                    namedCacheRoots,
                    rootPath,
                    MaxConnections,
                    ServiceConfiguration.DefaultGracefulShutdownSeconds,
                    grpcPort,
                    grpcPortFileName);
                Func <AbsolutePath, IContentStore> contentStoreFactory =
                    path =>
                    new FileSystemContentStore(
                        FileSystem,
                        SystemClock.Instance,
                        rootPath,
                        new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1KB"))));

                IRpcClient rpcClient;
                using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
                {
                    var        tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
                    BoolResult r      = await server.StartupAsync(context).ConfigureAwait(false);

                    r.ShouldBeSuccess();

                    var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
                    rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario);

                    r = await server.ShutdownAsync(context);

                    r.ShouldBeSuccess();
                }

                Func <Task <BoolResult> > createSessionFunc = () => rpcClient.CreateSessionAsync(context, SessionName, CacheName, ImplicitPin.None);
                createSessionFunc.Should().Throw <ClientCanRetryException>();

                rpcClient.Dispose();
            }
        }
Exemple #8
0
        private async Task RunTestCase(string testName, Func <AbsolutePath, IContentSession, GrpcCopyClient, Task> testAct)
        {
            var cacheName = testName + "_cache";

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath = directory.Path;

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { cacheName, rootPath }
                };
                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();
                var configuration    = new ServiceConfiguration(
                    namedCacheRoots,
                    rootPath,
                    42,
                    ServiceConfiguration.DefaultGracefulShutdownSeconds,
                    grpcPort,
                    grpcPortFileName);

                var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
                Func <AbsolutePath, IContentStore> contentStoreFactory = (path) =>
                                                                         new FileSystemContentStore(
                    FileSystem,
                    SystemClock.Instance,
                    directory.Path,
                    new ConfigurationModel(storeConfig));

                var server = new LocalContentServer(FileSystem, Logger, testName, contentStoreFactory, new LocalServerConfiguration(configuration));

                await server.StartupAsync(_context).ShouldBeSuccess();

                var createSessionResult = await server.CreateSessionAsync(new OperationContext(_context), testName, cacheName, ImplicitPin.PutAndGet, Capabilities.ContentOnly);

                createSessionResult.ShouldBeSuccess();

                (int sessionId, AbsolutePath tempDir) = createSessionResult.Value;
                var session = server.GetSession(sessionId);

                // Create a GRPC client to connect to the server
                var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
                using (var clientWrapper = await _clientCache.CreateAsync(LocalHost, port, false))
                {
                    // Run validation
                    await testAct(rootPath, session, clientWrapper.Value);
                }

                await server.ShutdownAsync(_context).ShouldBeSuccess();
            }
        }
 public async Task WrongPort()
 {
     await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
     {
         // Copy fake file out via GRPC
         var host = "localhost";
         var bogusPort = PortExtensions.GetNextAvailablePort();
         using (client = GrpcCopyClient.Create(host, bogusPort))
         {
             var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
             Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
         }
     });
 }
        public Task WrongPort()
        {
            return(RunTestCase(async(rootPath, session, client) =>
            {
                // Copy fake file out via GRPC
                var bogusPort = PortExtensions.GetNextAvailablePort();

                await _clientCache.UseAsync(new OperationContext(_context), LocalHost, bogusPort, async(nestedContext, client) =>
                {
                    var copyFileResult = await client.CopyFileAsync(nestedContext, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), new CopyOptions(bandwidthConfiguration: null));
                    Assert.Equal(CopyResultCode.ServerUnavailable, copyFileResult.Code);
                    return Unit.Void;
                });
            }));
        }
        public async Task HibernationDataNotLoadedIfStoreStartupFails()
        {
            const string scenario = nameof(HibernationDataNotLoadedIfStoreStartupFails);
            var          fileName = $"{Guid.NewGuid()}.json";

            var context = new Context(Logger);

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath    = directory.Path;
                var contentHash = ContentHash.Random();

                var pins = new List <string> {
                    contentHash.Serialize()
                };
                var hibernatedSessionInfo = new HibernatedContentSessionInfo(SessionId, SessionName, ImplicitPin.None, CacheName, pins, 0, Capabilities.None);
                var hibernatedSessions    = new HibernatedSessions <HibernatedContentSessionInfo>(new List <HibernatedContentSessionInfo> {
                    hibernatedSessionInfo
                });
                hibernatedSessions.Write(FileSystem, rootPath, fileName);

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { CacheName, rootPath }
                };

                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();

                var configuration = new ServiceConfiguration(namedCacheRoots, rootPath, MaxConnections, GracefulShutdownSeconds, grpcPort, grpcPortFileName);

                Func <AbsolutePath, IContentStore> contentStoreFactory =
                    (path) => new TestFailingContentStore();
                var localContentServerConfiguration = new LocalServerConfiguration(configuration)
                {
                    UnusedSessionHeartbeatTimeout       = TimeSpan.FromSeconds(TimeoutSecs),
                    UnusedSessionTimeout                = TimeSpan.FromSeconds(TimeoutSecs * 4),
                    RequestCallTokensPerCompletionQueue = 10,
                };

                using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, localContentServerConfiguration))
                {
                    var r = await server.StartupAsync(context);

                    r.ShouldBeError(TestFailingContentStore.FailureMessage);
                    FileSystem.HibernatedSessionsExists(rootPath, fileName).Should().BeTrue("The hibernation data should not have been read/deleted");
                }
            }
        }
        public async Task PutStreamRetriesWhenTempFileDisappears()
        {
            string scenario       = nameof(PutStreamRetriesWhenTempFileDisappears);
            var    tracingContext = new Context(Logger);
            var    context        = new OperationContext(tracingContext);

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath = directory.Path;

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { CacheName, rootPath }
                };
                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();
                var configuration    = new ServiceConfiguration(
                    namedCacheRoots,
                    rootPath,
                    MaxConnections,
                    ServiceConfiguration.DefaultGracefulShutdownSeconds,
                    grpcPort,
                    grpcPortFileName);
                Func <AbsolutePath, IContentStore> contentStoreFactory = path => new TestTempFileDeletingContentStore(FileSystem);

                using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
                {
                    var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
                    await server.StartupAsync(context).ShouldBeSuccess();

                    var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
                    using (IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, new ServiceClientRpcConfiguration(grpcPort), scenario))
                    {
                        await rpcClient.CreateSessionAsync(
                            context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();

                        using (var stream = new MemoryStream())
                        {
                            Func <Task <PutResult> > putFunc = () => rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory: false);
                            putFunc.Should().Throw <ClientCanRetryException>();
                        }

                        (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
                    }

                    await server.ShutdownAsync(context).ShouldBeSuccess();
                }
            }
        }
Exemple #13
0
        public async Task WrongPort()
        {
            await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
            {
                // Copy fake file out via GRPC
                var bogusPort = PortExtensions.GetNextAvailablePort();
                using (var clientWrapper = await _clientCache.CreateAsync(LocalHost, bogusPort, true))
                {
                    // Replace the given client with a bogus one
                    client = clientWrapper.Value;

                    var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
                    Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
                }
            });
        }
Exemple #14
0
        protected override IContentStore CreateStore(AbsolutePath rootPath, string cacheName, ContentStoreConfiguration configuration)
        {
            configuration.Write(FileSystem, rootPath);
            var grpcPortFileName = Guid.NewGuid().ToString();

            var serviceConfig = new ServiceConfiguration(
                new Dictionary <string, AbsolutePath> {
                { cacheName, rootPath }
            },
                rootPath,
                GracefulShutdownSeconds,
                PortExtensions.GetNextAvailablePort(),
                grpcPortFileName);

            return(new TestInProcessServiceClientContentStore(
                       FileSystem, Logger, cacheName, _scenario, null, serviceConfig));
        }
Exemple #15
0
        public async Task WrongPort()
        {
            await RunTestCase(nameof(WrongPort), async (rootPath, session, client) =>
            {
                // Write a random file to put into the cache
                var content     = ThreadSafeRandom.GetBytes(FileSize);
                var contentHash = HashingExtensions.CalculateHash(content, HashType.Vso0);

                // Copy the file out via GRPC
                var host      = "localhost";
                var bogusPort = PortExtensions.GetNextAvailablePort();
                using (client = GrpcCopyClient.Create(host, bogusPort))
                {
                    var copyFileResult = await client.CopyFileAsync(_context, contentHash, rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None);
                    Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code);
                }
            });
        }
Exemple #16
0
        /// <inheritdoc />
        protected override ICache CreateCache(DisposableDirectory testDirectory)
        {
            var backendCacheDirectory = testDirectory.Path / "Backend";

            FileSystem.CreateDirectory(backendCacheDirectory);

            var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                [CacheName] = backendCacheDirectory / "Root"
            };

            var grpcPort            = PortExtensions.GetNextAvailablePort();
            var serverConfiguration = new LocalServerConfiguration(backendCacheDirectory, namedCacheRoots, grpcPort, FileSystem)
            {
                GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
            };
            var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(CacheName, new ServiceClientRpcConfiguration(serverConfiguration.GrpcPort), "Scenario-" + Guid.NewGuid());
            Func <AbsolutePath, ICache> contentStoreFactory = CreateBackendCache;
            var serviceClient = new TestInProcessServiceClientCache(Logger, FileSystem, contentStoreFactory, serverConfiguration, serviceClientConfiguration);

            return(serviceClient);
        }
Exemple #17
0
        private async Task RunClientWhenServiceDownTestAsync(Context context, Func <Context, GrpcRepairClient, Task> funcAsync)
        {
            var exceptionThrown = false;

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var configuration = CreateServiceConfiguration(directory.Path, PortExtensions.GetNextAvailablePort(), Guid.NewGuid().ToString());

                var client = new GrpcRepairClient(configuration.GrpcPort);

                try
                {
                    await funcAsync(context, client);
                }
                catch (ClientCanRetryException)
                {
                    exceptionThrown = true;
                }

                Assert.True(exceptionThrown);
            }
        }
Exemple #18
0
        protected override IContentStore CreateStore(AbsolutePath rootPath, ContentStoreConfiguration configuration)
        {
            configuration.Write(FileSystem, rootPath).Wait();
            var grpcPortFileName = Guid.NewGuid().ToString();

            var serviceConfiguration = new ServiceConfiguration(
                new Dictionary <string, AbsolutePath> {
                { CacheName, rootPath }
            },
                rootPath,
                MaxConnections,
                GracefulShutdownSeconds,
                PortExtensions.GetNextAvailablePort(),
                grpcPortFileName);

            return(new TestServiceClientContentStore(
                       Logger,
                       FileSystem,
                       new ServiceClientContentStoreConfiguration(CacheName, null, Scenario),
                       null,
                       serviceConfiguration));
        }
Exemple #19
0
        private async Task RunServerTestAsync(Context context, Func <Context, ServiceConfiguration, Task> funcAsync)
        {
            using (var directory = new DisposableDirectory(FileSystem))
            {
                var storeConfig = CreateStoreConfiguration();
                await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);

                var serviceConfig = CreateServiceConfiguration(directory.Path, PortExtensions.GetNextAvailablePort(), Guid.NewGuid().ToString());

                using (var server = new ServiceProcess(serviceConfig, LocalContentServerConfiguration, Scenario, WaitForServerReadyTimeoutMs, WaitForExitTimeoutMs))
                {
                    BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);

                    r.ShouldBeSuccess();

                    await funcAsync(context, serviceConfig);

                    r = await server.ShutdownAsync(context);

                    r.ShouldBeSuccess();
                }
            }
        }
Exemple #20
0
        protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
        {
            var rootPath = testDirectory.Path;

            configuration.Write(FileSystem, rootPath);

            var grpcPortFileName = Guid.NewGuid().ToString();

            var serviceConfig = new ServiceConfiguration(
                new Dictionary <string, AbsolutePath> {
                { CacheName, rootPath }
            },
                rootPath,
                GracefulShutdownSeconds,
                PortExtensions.GetNextAvailablePort(),
                grpcPortFileName);

            return(new TestServiceClientContentStore(
                       Logger,
                       FileSystem,
                       new ServiceClientContentStoreConfiguration(CacheName, rpcConfiguration: null, scenario: Scenario),
                       heartbeatInterval: null,
                       serviceConfiguration: serviceConfig));
        }
        public async Task CacheSessionDataIsHibernated()
        {
            using var testDirectory = new DisposableDirectory(FileSystem);
            var cacheDirectory  = testDirectory.Path / "Service";
            var cacheName       = "theCache";
            var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                [cacheName] = cacheDirectory / "Root"
            };
            var grpcPort            = PortExtensions.GetNextAvailablePort();
            var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem)
            {
                GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
            };
            var scenario = "Default";

            var server = new LocalCacheServer(
                FileSystem,
                TestGlobal.Logger,
                scenario,
                cacheFactory: CreateBlockingPublishingCache,
                serverConfiguration,
                Capabilities.All);

            var context = new OperationContext(new Context(Logger));
            await server.StartupAsync(context).ShouldBeSuccess();

            var pat = Guid.NewGuid().ToString();
            var publishingConfig = new PublishingConfigDummy
            {
                PublishAsynchronously = true,
            };

            using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario);
            await clientCache.StartupAsync(context).ShouldBeSuccess();

            var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session;

            await clientSession.StartupAsync(context).ShouldBeSuccess();

            var piecesOfContent = 3;
            var putResults      = await Task.WhenAll(
                Enumerable.Range(0, piecesOfContent)
                .Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess()));

            var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray());
            var determinism     = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1));
            var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);

            var fingerprint       = new Fingerprint(ContentHash.Random().ToByteArray());
            var selector          = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 });
            var strongFingerprint = new StrongFingerprint(fingerprint, selector);

            var cts = new CancellationTokenSource();
            // Even though publishing is blocking, this should succeed because we're publishing asynchronously.
            await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess();

            // Allow for the publishing operation to be registered.
            await Task.Delay(TimeSpan.FromSeconds(1));

            // Simulate a restart.
            await server.ShutdownAsync(context).ShouldBeSuccess();

            server.Dispose();

            server = new LocalCacheServer(
                FileSystem,
                TestGlobal.Logger,
                scenario: scenario,
                cacheFactory: CreateBlockingPublishingCache,
                serverConfiguration,
                Capabilities.All);

            await server.StartupAsync(context).ShouldBeSuccess();

            // Session should have been persisted.
            var sessionsAndDatas = server.GetCurrentSessions();

            sessionsAndDatas.Length.Should().Be(1);
            var serverSession = sessionsAndDatas[0].session;
            var data          = sessionsAndDatas[0].data;

            var operation = new PublishingOperation
            {
                ContentHashListWithDeterminism = contentHashListwithDeterminism,
                StrongFingerprint = strongFingerprint
            };
            var operations = new[] { operation };

            data.Name.Should().Be(clientSession.Name);
            data.Pat.Should().Be(pat);
            data.Capabilities.Should().Be(Capabilities.All);
            data.ImplicitPin.Should().Be(ImplicitPin.None);
            data.Pins.Should().BeEquivalentTo(new List <string>());
            data.PublishingConfig.Should().BeEquivalentTo(publishingConfig);
            data.PendingPublishingOperations.Should().BeEquivalentTo(operations);

            var hibernateSession = serverSession as IHibernateCacheSession;

            hibernateSession.Should().NotBeNull();
            var actualPending = hibernateSession.GetPendingPublishingOperations();

            actualPending.Should().BeEquivalentTo(operations);

            await server.ShutdownAsync(context).ShouldBeSuccess();

            await clientSession.ShutdownAsync(context).ShouldBeSuccess();
        }
Exemple #22
0
 // Setup all grpc ports before creating stores so they know how to talk to each other.
 protected override object PrepareAdditionalCreateStoreArgs(int storeCount)
 {
     return(Enumerable.Range(0, storeCount).Select(i => PortExtensions.GetNextAvailablePort()).ToArray());
 }
Exemple #23
0
        public async Task ServiceTestAsync()
        {
            using var fileSystem = new PassThroughFileSystem(Logger);
            using var dir        = new DisposableDirectory(fileSystem);
            var cacheDir = dir.Path / "cache";
            var dataPath = dir.Path / "data";

            var port = PortExtensions.GetNextAvailablePort();

            var args = new Dictionary <string, string>
            {
                ["paths"]            = cacheDir.Path,
                ["names"]            = "Default",
                ["grpcPort"]         = port.ToString(),
                ["LogSeverity"]      = "Diagnostic",
                ["dataRootPath"]     = dataPath.Path,
                ["Scenario"]         = "AppTests",
                ["grpcPortFileName"] = "AppTestsMMF"
            };

            var serviceProcess = RunService("Service", args, Logger);

            try
            {
                await RunAppAsync("ServiceRunning", new Dictionary <string, string> {
                    { "waitSeconds", "5" }, { "Scenario", "AppTests" }
                }, Logger);

                var context = new Context(Logger);

                var config = new ServiceClientContentStoreConfiguration("Default", new ServiceClientRpcConfiguration {
                    GrpcPort = port
                }, scenario: "AppTests");
                using var store = new ServiceClientContentStore(Logger, fileSystem, config);
                await store.StartupAsync(context).ShouldBeSuccess();

                var sessionResult = store.CreateSession(context, "Default", ImplicitPin.None).ShouldBeSuccess();
                using var session = sessionResult.Session;
                await session.StartupAsync(context).ShouldBeSuccess();

                var source   = dir.Path / "source.txt";
                var contents = new byte[1024];
                Random.NextBytes(contents);
                fileSystem.WriteAllBytes(source, contents);

                var putResult = await session.PutFileAsync(context, HashType.MD5, source, FileRealizationMode.Any, CancellationToken.None).ShouldBeSuccess();

                var hash = putResult.ContentHash;

                await session.PinAsync(context, hash, CancellationToken.None).ShouldBeSuccess();

                var destination = dir.Path / "destination.txt";
                await session.PlaceFileAsync(
                    context,
                    hash,
                    destination,
                    FileAccessMode.ReadOnly,
                    FileReplacementMode.FailIfExists,
                    FileRealizationMode.Any,
                    CancellationToken.None).ShouldBeSuccess();

                fileSystem.ReadAllBytes(destination).Should().BeEquivalentTo(contents);
            }
            finally
            {
                if (!serviceProcess.HasExited)
                {
                    serviceProcess.Kill();
#pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead
                    serviceProcess.WaitForExit();
#pragma warning restore AsyncFixer02
                }
            }
        }
Exemple #24
0
        [Trait("Category", "QTestSkip")] // Skipped
        public async Task RestoredSessionReleasedAfterInactivity()
        {
            const string scenario = nameof(RestoredSessionReleasedAfterInactivity);

            var context = new Context(Logger);

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath    = directory.Path;
                var contentHash = ContentHash.Random();

                var pins = new List <string> {
                    contentHash.Serialize()
                };
                var hibernatedSessionInfo = new HibernatedSessionInfo(SessionId, SessionName, ImplicitPin.None, CacheName, pins, DateTime.UtcNow.Ticks, Capabilities.None);
                var hibernatedSessions    = new HibernatedSessions(new List <HibernatedSessionInfo> {
                    hibernatedSessionInfo
                });
                await hibernatedSessions.WriteAsync(FileSystem, rootPath);

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { CacheName, rootPath }
                };

                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();

                var configuration = new ServiceConfiguration(namedCacheRoots, rootPath, MaxConnections, GracefulShutdownSeconds, grpcPort, grpcPortFileName);
                var storeConfig   = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
                Func <AbsolutePath, IContentStore> contentStoreFactory = (path) =>
                                                                         new FileSystemContentStore(
                    FileSystem,
                    SystemClock.Instance,
                    directory.Path,
                    new ConfigurationModel(storeConfig));
                var localContentServerConfiguration = new LocalServerConfiguration(configuration)
                {
                    UnusedSessionHeartbeatTimeout       = TimeSpan.FromSeconds(TimeoutSecs),
                    RequestCallTokensPerCompletionQueue = 10,
                };

                using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, localContentServerConfiguration))
                {
                    var r1 = await server.StartupAsync(context);

                    r1.ShouldBeSuccess();

                    var beforeIds = server.GetSessionIds();
                    beforeIds.Should().Contain(SessionId);

                    // Wait one period to ensure that it times out, another to ensure that the checker finds it, and another to give it time to release it.
                    await Task.Delay(TimeSpan.FromSeconds(TimeoutSecs * 3));

                    var afterIds = server.GetSessionIds();
                    afterIds.Count.Should().Be(0);

                    var r2 = await server.ShutdownAsync(context);

                    r2.ShouldBeSuccess();
                }
            }
        }
        public async Task MultipleCaches()
        {
            const string CacheName1 = "test1";
            const string CacheName2 = "test2";

            using (var testDirectory0 = new DisposableDirectory(FileSystem))
                using (var testDirectory1 = new DisposableDirectory(FileSystem))
                    using (var testDirectory2 = new DisposableDirectory(FileSystem))
                    {
                        var config = CreateStoreConfiguration();

                        var rootPath1 = testDirectory1.Path;
                        config.Write(FileSystem, rootPath1);

                        var rootPath2 = testDirectory2.Path;
                        config.Write(FileSystem, rootPath2);

                        var grpcPort         = PortExtensions.GetNextAvailablePort();
                        var grpcPortFileName = Guid.NewGuid().ToString();

                        var serviceConfiguration = new ServiceConfiguration(
                            new Dictionary <string, AbsolutePath> {
                            { CacheName1, rootPath1 }, { CacheName2, rootPath2 }
                        },
                            testDirectory0.Path,
                            ServiceConfiguration.DefaultGracefulShutdownSeconds,
                            grpcPort,
                            grpcPortFileName);

                        using (var server = CreateServer(serviceConfiguration))
                        {
                            var factory   = new MemoryMappedFileGrpcPortSharingFactory(Logger, grpcPortFileName);
                            var reader    = factory.GetPortReader();
                            var port      = reader.ReadPort();
                            var rpcConfig = new ServiceClientRpcConfiguration(port);

                            using (var store1 = new ServiceClientContentStore(
                                       Logger,
                                       FileSystem,
                                       new ServiceClientContentStoreConfiguration(CacheName1, rpcConfig, Scenario)))
                                using (var store2 = new ServiceClientContentStore(
                                           Logger,
                                           FileSystem,
                                           new ServiceClientContentStoreConfiguration(CacheName2, rpcConfig, Scenario)))
                                {
                                    try
                                    {
                                        var rs = await server.StartupAsync(_context);

                                        rs.ShouldBeSuccess();

                                        var storeBoolResult1 = await store1.StartupAsync(_context);

                                        storeBoolResult1.ShouldBeSuccess();

                                        var storeBoolResult2 = await store2.StartupAsync(_context);

                                        storeBoolResult2.ShouldBeSuccess();

                                        IContentSession session1 = null;
                                        IContentSession session2 = null;

                                        try
                                        {
                                            var createSessionResult1 = store1.CreateSession(_context, "session1", ImplicitPin.None);
                                            createSessionResult1.ShouldBeSuccess();

                                            var createSessionResult2 = store2.CreateSession(_context, "session2", ImplicitPin.None);
                                            createSessionResult2.ShouldBeSuccess();

                                            using (createSessionResult1.Session)
                                                using (createSessionResult2.Session)
                                                {
                                                    var r1 = await createSessionResult1.Session.StartupAsync(_context);

                                                    r1.ShouldBeSuccess();
                                                    session1 = createSessionResult1.Session;

                                                    var r2 = await createSessionResult2.Session.StartupAsync(_context);

                                                    r2.ShouldBeSuccess();
                                                    session2 = createSessionResult2.Session;

                                                    var r3 = await session1.PutRandomAsync(
                                                        _context,
                                                        ContentHashType,
                                                        false,
                                                        RandomContentByteCount,
                                                        Token);

                                                    var pinResult = await session1.PinAsync(_context, r3.ContentHash, Token);

                                                    pinResult.ShouldBeSuccess();

                                                    r3 = await session2.PutRandomAsync(
                                                        _context,
                                                        ContentHashType,
                                                        false,
                                                        RandomContentByteCount,
                                                        Token);

                                                    pinResult = await session2.PinAsync(_context, r3.ContentHash, Token);

                                                    pinResult.ShouldBeSuccess();
                                                }
                                        }
                                        finally
                                        {
                                            if (session2 != null)
                                            {
                                                await session2.ShutdownAsync(_context).ShouldBeSuccess();
                                            }

                                            if (session1 != null)
                                            {
                                                await session1.ShutdownAsync(_context).ShouldBeSuccess();
                                            }
                                        }
                                    }
                                    finally
                                    {
                                        BoolResult r1 = null;
                                        BoolResult r2 = null;

                                        if (store1.StartupCompleted)
                                        {
                                            r1 = await store1.ShutdownAsync(_context);
                                        }

                                        if (store2.StartupCompleted)
                                        {
                                            r2 = await store2.ShutdownAsync(_context);
                                        }

                                        var r3 = await server.ShutdownAsync(_context);

                                        r1?.ShouldBeSuccess();
                                        r2?.ShouldBeSuccess();
                                        r3?.ShouldBeSuccess();
                                    }
                                }
                        }
                    }
        }
Exemple #26
0
        public async Task DoesNotRespectFileReplacementMode(FileReplacementMode requestedReplacementMode)
        {
            string scenario = nameof(DoesNotRespectFileReplacementMode) + requestedReplacementMode;
            var    context  = new Context(Logger);

            using (var directory = new DisposableDirectory(FileSystem))
            {
                var rootPath = directory.Path;

                var namedCacheRoots = new Dictionary <string, AbsolutePath> {
                    { CacheName, rootPath }
                };
                var grpcPort         = PortExtensions.GetNextAvailablePort();
                var grpcPortFileName = Guid.NewGuid().ToString();
                var configuration    = new ServiceConfiguration(
                    namedCacheRoots,
                    rootPath,
                    MaxConnections,
                    ServiceConfiguration.DefaultGracefulShutdownSeconds,
                    grpcPort,
                    grpcPortFileName);
                Func <AbsolutePath, IContentStore> contentStoreFactory =
                    path =>
                    new FileSystemContentStore(
                        FileSystem,
                        SystemClock.Instance,
                        rootPath,
                        new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("1MB"))));

                using (var server = new LocalContentServer(FileSystem, Logger, scenario, contentStoreFactory, new LocalServerConfiguration(configuration)))
                {
                    var tracer = new ServiceClientContentSessionTracer("TestTracerForRpcClient");
                    await server.StartupAsync(context).ShouldBeSuccess();

                    var        port      = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort();
                    IRpcClient rpcClient = new GrpcContentClient(tracer, FileSystem, port, scenario);

                    await rpcClient.CreateSessionAsync(
                        context, SessionName, CacheName, ImplicitPin.None).ShouldBeSuccess();

                    ContentHash contentHash;
                    using (var stream = new MemoryStream())
                    {
                        PutResult putResult = await rpcClient.PutStreamAsync(context, HashType.Vso0, stream, createDirectory : false);

                        putResult.ShouldBeSuccess();
                        putResult.ContentSize.Should().Be(0);
                        contentHash = putResult.ContentHash;
                    }

                    var tempPath = directory.CreateRandomFileName();
                    FileSystem.WriteAllBytes(tempPath, new byte[] {});

                    var placeResult = await rpcClient.PlaceFileAsync(
                        context, contentHash, tempPath, FileAccessMode.ReadOnly, requestedReplacementMode, FileRealizationMode.Any);

                    placeResult.Succeeded.Should().BeTrue();

                    (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
                    rpcClient.Dispose();

                    await server.ShutdownAsync(context).ShouldBeSuccess();
                }
            }
        }
Exemple #27
0
        private void StartRedisServerIfNeeded()
        {
            // Can reuse an existing process only when this instance successfully created a connection to it.
            // Otherwise the test will fail with NRE.
            if (_process != null && _connectionMultiplexer != null)
            {
                _logger.Debug("Redis process is already running. Reusing an existing instance.");
                return;
            }

            _logger.Debug("Starting a redis server.");

            var    redisName       = OperatingSystemHelper.IsWindowsOS ? "redis-server.exe" : "redis-server";
            string redisServerPath = Path.GetFullPath(Path.Combine("redisServer", redisName));

            if (!File.Exists(redisServerPath))
            {
                throw new InvalidOperationException($"Could not find {redisName} at {redisServerPath}");
            }

            int portNumber = 0;

            const int maxRetries = 10;

            for (int i = 0; i < maxRetries; i++)
            {
                var fileName            = _tempDirectory.CreateRandomFileName();
                var redisServerLogsPath = _tempDirectory.CreateRandomFileName();
                _fileSystem.CreateDirectory(redisServerLogsPath);
                portNumber = PortExtensions.GetNextAvailablePort();
                string newConfig = $@"
timeout 0
tcp-keepalive 0
dir {redisServerLogsPath}
port {portNumber}";

                File.WriteAllText(fileName.Path, newConfig);

                var args = $" {fileName}";
                _logger.Debug($"Running cmd=[{redisServerPath} {args}]");

                const bool createNoWindow = true;
                _process = new ProcessUtility(redisServerPath, args, createNoWindow, workingDirectory: Path.GetDirectoryName(redisServerPath));

                _process.Start();

                string processOutput;
                if (_process == null)
                {
                    processOutput = "[Process could not start]";
                    throw new InvalidOperationException(processOutput);
                }

                if (createNoWindow)
                {
                    if (_process.HasExited)
                    {
                        if (_process.WaitForExit(5000))
                        {
                            throw new InvalidOperationException(_process.GetLogs());
                        }

                        throw new InvalidOperationException("Process or either wait handle timed out. " + _process.GetLogs());
                    }

                    processOutput = $"[Process {_process.Id} is still running]";
                }

                _logger.Debug("Process output: " + processOutput);

                ConnectionString = $"localhost:{portNumber}";
                ConfigurationOptions options = ConfigurationOptions.Parse(ConnectionString);
                options.ConnectTimeout = 2000;
                options.SyncTimeout    = 2000;
                try
                {
                    _connectionMultiplexer = ConnectionMultiplexer.Connect(options);
                    break;
                }
                catch (RedisConnectionException ex)
                {
                    SafeKillProcess();
                    _logger.Debug($"Retrying for exception connecting to redis process {_process.Id} with port {portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");

                    if (i != maxRetries - 1)
                    {
                        Thread.Sleep(300);
                    }
                    else
                    {
                        throw;
                    }
                }
                catch (Exception ex)
                {
                    SafeKillProcess();
                    _logger.Error(
                        $"Exception connecting to redis process {_process.Id} with port {portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");
                    throw;
                }
            }

            _logger.Debug($"Redis server {_process.Id} is up and running at port {portNumber}.");
        }
        private void Start()
        {
            // Can reuse an existing process only when this instance successfully created a connection to it.
            // Otherwise the test will fail with NRE.
            if (_process != null)
            {
                _logger.Debug("Storage process is already running. Reusing an existing instance.");
                return;
            }

            _logger.Debug("Starting a storage server.");


            var storageName = (OperatingSystemHelper.IsWindowsOS ? "tools/win-x64/blob.exe"
                : (OperatingSystemHelper.IsLinuxOS ? "tools/linux-x64/blob"
                : "tools/osx-x64/blob"));
            string storageServerPath = Path.GetFullPath(Path.Combine("azurite", storageName));

            if (!File.Exists(storageServerPath))
            {
                throw new InvalidOperationException($"Could not find {storageName} at {storageServerPath}");
            }

            _portNumber = 0;

            const int maxRetries = 10;

            for (int i = 0; i < maxRetries; i++)
            {
                var storageServerWorkspacePath = _tempDirectory.CreateRandomFileName();
                _fileSystem.CreateDirectory(storageServerWorkspacePath);
                _portNumber = PortExtensions.GetNextAvailablePort();

                var args = $"--blobPort {_portNumber} --location {storageServerWorkspacePath}";
                _logger.Debug($"Running cmd=[{storageServerPath} {args}]");

                _process = new ProcessUtility(storageServerPath, args, createNoWindow: true, workingDirectory: Path.GetDirectoryName(storageServerPath));

                _process.Start();

                string processOutput;
                if (_process == null)
                {
                    processOutput = "[Process could not start]";
                    throw new InvalidOperationException(processOutput);
                }

                if (_process.HasExited)
                {
                    if (_process.WaitForExit(5000))
                    {
                        throw new InvalidOperationException(_process.GetLogs());
                    }

                    throw new InvalidOperationException("Process or either wait handle timed out. " + _process.GetLogs());
                }

                processOutput = $"[Process {_process.Id} is still running]";

                _logger.Debug("Process output: " + processOutput);

                ConnectionString = $"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:{_portNumber}/devstoreaccount1;";

                AzureBlobStorageCredentials creds = new AzureBlobStorageCredentials(ConnectionString);

                var client = creds.CreateCloudBlobClient();
                try
                {
                    bool exists = client.GetContainerReference("test").ExistsAsync(DefaultBlobStorageRequestOptions, null).GetAwaiter().GetResult();
                    break;
                }
                catch (StorageException ex)
                {
                    SafeKillProcess();
                    _logger.Debug($"Retrying for exception connecting to storage process {_process.Id} with port {_portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");

                    if (i != maxRetries - 1)
                    {
                        Thread.Sleep(300);
                    }
                    else
                    {
                        throw;
                    }
                }
                catch (Exception ex)
                {
                    SafeKillProcess();
                    _logger.Error(
                        $"Exception connecting to storage process {_process.Id} with port {_portNumber}: {ex.ToString()}. Has process exited {_process.HasExited} with output {_process.GetLogs()}");
                    throw;
                }
            }

            _logger.Debug($"Storage server {_process.Id} is up and running at port {_portNumber}.");
        }