private static IRpcClient GetRpcClient( IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration) { return(new GrpcCacheClient(sessionTracer, fileSystem, configuration.RpcConfiguration, configuration.Scenario)); }
/// <nodoc /> public ServiceClientCache( ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentStoreConfiguration configuration) : base(logger, fileSystem, configuration) { }
/// <summary> /// Both content and metadata are entirely backed by an out-of-proc cache. /// </summary> public static ICache CreateRpcCache( ILogger logger, ServiceClientContentStoreConfiguration serviceClientCacheConfiguration) { var fileSystem = new PassThroughFileSystem(logger); return(new ServiceClientCache(logger, fileSystem, serviceClientCacheConfiguration)); }
private static IRpcClient GetRpcClient( IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration) { var rpcConfiguration = configuration.RpcConfiguration; return(new GrpcCacheClient(sessionTracer, fileSystem, rpcConfiguration.GrpcPort, configuration.Scenario, rpcConfiguration.HeartbeatInterval)); }
private static MemoizationStoreAdapterCache CreateCache(Config cacheConfig, AbsolutePath logPath, ILogger logger) { ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName) { RetryCount = cacheConfig.ConnectionRetryCount, RetryIntervalSeconds = cacheConfig.ConnectionRetryIntervalSeconds, TraceOperationStarted = cacheConfig.GrpcTraceOperationStarted, }; MemoizationStore.Interfaces.Caches.ICache localCache; if (cacheConfig.EnableMetadataServer) { localCache = LocalCache.CreateRpcCache(logger, serviceClientConfiguration); } else { var metadataRootPath = new AbsolutePath(cacheConfig.MetadataRootPath); localCache = LocalCache.CreateRpcContentStoreInProcMemoizationStoreCache(logger, metadataRootPath, serviceClientConfiguration, CreateInProcMemoizationStoreConfiguration(cacheConfig, metadataRootPath)); } var statsFilePath = new AbsolutePath(logPath.Path + ".stats"); if (!string.IsNullOrEmpty(cacheConfig.VfsCasRoot)) { logger.Debug($"Creating virtualized cache"); localCache = new VirtualizedContentCache(localCache, new ContentStore.Vfs.VfsCasConfiguration.Builder() { RootPath = new AbsolutePath(cacheConfig.VfsCasRoot), UseSymlinks = cacheConfig.UseVfsSymlinks }.Build()); } var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile); return(cache); }
/// <inheritdoc /> public TestInProcessServiceClientCache( ILogger logger, IAbsFileSystem fileSystem, Func <AbsolutePath, ICache> contentStoreFactory, LocalServerConfiguration contentServerConfiguration, ServiceClientContentStoreConfiguration clientConfiguration) { _server = new LocalCacheServer(fileSystem, logger, clientConfiguration.Scenario, contentStoreFactory, contentServerConfiguration, Capabilities.All); _client = new ServiceClientCache(logger, fileSystem, clientConfiguration); SetThreadPoolSizes(); }
/// <summary> /// Initializes a new instance of the <see cref="ServiceClientContentSession"/> class. /// </summary> public ServiceClientContentSession( string name, ImplicitPin implicitPin, ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration, Func <IRpcClient> rpcClientFactory = null) : base(name, implicitPin, logger, fileSystem, sessionTracer, configuration, rpcClientFactory) { }
private static IRpcClient CreateRpcClient( IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration, PublishingCacheConfiguration publishingConfig, string pat) => new GrpcPublishingCacheClient( sessionTracer, fileSystem, configuration.RpcConfiguration, configuration.Scenario, publishingConfig, pat);
/// <nodoc /> public ServiceClientCacheSession( string name, ImplicitPin implicitPin, ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration) : base(name, implicitPin, logger, fileSystem, sessionTracer, configuration, () => GetRpcClient(fileSystem, sessionTracer, configuration)) { // RpcClient is created by the base class constructor, but we know that this is the result of GetPrcClient call // that actually returns GrpcCacheClient instance. _rpcCacheClient = (GrpcCacheClient)RpcClient; }
/// <inheritdoc /> public TestInProcessServiceClientCache( ILogger logger, IAbsFileSystem fileSystem, Func <AbsolutePath, ICache> contentStoreFactory, LocalServerConfiguration contentServerConfiguration, ServiceClientContentStoreConfiguration clientConfiguration) { // Initialize with fewer threads for tests GrpcEnvironment.InitializeIfNeeded(3); _server = new LocalCacheService(logger, fileSystem, clientConfiguration.Scenario, contentStoreFactory, contentServerConfiguration); _client = new ServiceClientCache(logger, fileSystem, clientConfiguration); SetThreadPoolSizes(); }
/// <nodoc /> public ServiceClientPublishingCache( ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentStoreConfiguration configuration, PublishingCacheConfiguration publishingConfig, string pat) : base(logger, fileSystem, configuration) { Contract.Assert(publishingConfig != null, "Publishing configuration should not be null"); _publishingConfig = publishingConfig; _pat = pat; }
private static ServiceClientContentStoreConfiguration CreateGrpcServiceConfiguration(Config cacheConfig, ILogger logger) { var rpcConfiguration = CreateGrpcClientConfiguration(cacheConfig, logger); var serviceClientContentStoreConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName) { RetryCount = cacheConfig.ConnectionRetryCount, RetryIntervalSeconds = cacheConfig.ConnectionRetryIntervalSeconds, TraceOperationStarted = cacheConfig.GrpcTraceOperationStarted, }; serviceClientContentStoreConfiguration.GrpcEnvironmentOptions = cacheConfig.GrpcEnvironmentOptions; return(serviceClientContentStoreConfiguration); }
private LocalCache( ILogger logger, AbsolutePath rootPath, IAbsFileSystem fileSystem, IClock clock, ServiceClientContentStoreConfiguration configuration, MemoizationStoreConfiguration memoConfig) : base( () => new ServiceClientContentStore( logger, fileSystem, configuration), CreateMemoizationStoreFactory(logger, clock, memoConfig), PersistentId.Load(fileSystem, rootPath / IdFileName)) { _fileSystem = fileSystem; }
/// <summary> /// Content Stores: /// - <see cref="ServiceClientContentStore"/> /// Memoization Stores: /// - <see cref="CreateInProcessLocalMemoizationStoreFactory(ILogger, IClock, MemoizationStoreConfiguration)"/> /// </summary> public static LocalCache CreateRpcContentStoreInProcMemoizationStoreCache(ILogger logger, AbsolutePath rootPath, ServiceClientContentStoreConfiguration serviceClientContentStoreConfiguration, MemoizationStoreConfiguration memoizationStoreConfiguration, IClock clock = null) { var fileSystem = new PassThroughFileSystem(logger); clock = clock ?? SystemClock.Instance; Func <IContentStore> remoteContentStoreFactory = () => new ServiceClientContentStore(logger, fileSystem, serviceClientContentStoreConfiguration); var localMemoizationStoreFactory = CreateInProcessLocalMemoizationStoreFactory(logger, clock, memoizationStoreConfiguration); return(new LocalCache(fileSystem, remoteContentStoreFactory, localMemoizationStoreFactory, LoadPersistentCacheGuid(rootPath, fileSystem))); }
public TestServiceClientContentStore( ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentStoreConfiguration configuration, TimeSpan?heartbeatInterval, ServiceConfiguration serviceConfiguration, LocalServerConfiguration localContentServerConfiguration = null) : base(logger, fileSystem, configuration) { _logger = logger; _localContentServerConfiguration = localContentServerConfiguration; _serviceProcess = new ServiceProcess(_configuration, localContentServerConfiguration, configuration.Scenario, WaitForServerReadyTimeoutMs, WaitForExitTimeoutMs); _configuration = serviceConfiguration; _heartbeatInterval = heartbeatInterval; }
private static MemoizationStore.Interfaces.Caches.ICache CreateGrpcCache(Config config, DisposeLogger logger) { Contract.Requires(config.RetryIntervalSeconds >= 0); Contract.Requires(config.RetryCount >= 0); var serviceClientRpcConfiguration = new ServiceClientRpcConfiguration() { GrpcCoreClientOptions = config.GrpcCoreClientOptions, }; if (config.GrpcPort > 0) { serviceClientRpcConfiguration.GrpcPort = config.GrpcPort; } ServiceClientContentStoreConfiguration serviceClientContentStoreConfiguration = null; if (config.EnableContentServer) { new ServiceClientContentStoreConfiguration(config.CacheName, serviceClientRpcConfiguration, config.ScenarioName) { RetryIntervalSeconds = (uint)config.RetryIntervalSeconds, RetryCount = (uint)config.RetryCount, GrpcEnvironmentOptions = config.GrpcEnvironmentOptions, }; } if (config.EnableContentServer && config.EnableMetadataServer) { return(LocalCache.CreateRpcCache(logger, serviceClientContentStoreConfiguration)); } else { Contract.Assert(!config.EnableMetadataServer, "It is not supported to use a Metadata server without a Content server"); var memoizationStoreConfiguration = GetInProcMemoizationStoreConfiguration(new AbsolutePath(config.CacheRootPath), config, GetCasConfig(config)); return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(logger, new AbsolutePath(config.CacheRootPath), memoizationStoreConfiguration, new LocalCacheConfiguration(serviceClientContentStoreConfiguration), configurationModel: CreateConfigurationModel(GetCasConfig(config)), clock: null, checkLocalFiles: config.CheckLocalFiles)); } }
private static MemoizationStore.Interfaces.Caches.ICache CreateLocalCacheWithSingleCas(Config config, DisposeLogger logger) { if (config.EnableContentServer && config.EnableMetadataServer) { Contract.Assert(config.RetryIntervalSeconds >= 0); Contract.Assert(config.RetryCount >= 0); var rpcConfiguration = new ServiceClientRpcConfiguration(config.GrpcPort); var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(config.CacheName, rpcConfiguration, config.ScenarioName) { RetryIntervalSeconds = (uint)config.RetryIntervalSeconds, RetryCount = (uint)config.RetryCount, }; return(LocalCache.CreateRpcCache(logger, serviceClientConfiguration)); } else { Contract.Assert(!config.EnableMetadataServer, "It is not supported to use a Metadata server without a Content server"); LocalCacheConfiguration localCacheConfiguration; if (config.EnableContentServer) { localCacheConfiguration = LocalCacheConfiguration.CreateServerEnabled( config.GrpcPort, config.CacheName, config.ScenarioName, config.RetryIntervalSeconds, config.RetryCount); } else { localCacheConfiguration = LocalCacheConfiguration.CreateServerDisabled(); } return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(logger, new AbsolutePath(config.CacheRootPath), GetInProcMemoizationStoreConfiguration(new AbsolutePath(config.CacheRootPath), config, GetCasConfig(config)), localCacheConfiguration, configurationModel: CreateConfigurationModel(GetCasConfig(config)), clock: null, checkLocalFiles: config.CheckLocalFiles, emptyFileHashShortcutEnabled: config.EmptyFileHashShortcutEnabled)); } }
/// <nodoc /> public ServiceClientPublishingCacheSession( string name, ImplicitPin implicitPin, ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration, PublishingCacheConfiguration publishingConfig, string pat) : base( name, implicitPin, logger, fileSystem, sessionTracer, configuration, () => CreateRpcClient(fileSystem, sessionTracer, configuration, publishingConfig, pat)) { }
private static MemoizationStoreAdapterCache CreateCache(Config cacheConfig, AbsolutePath logPath, ILogger logger) { ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName) { RetryCount = cacheConfig.ConnectionRetryCount, RetryIntervalSeconds = cacheConfig.ConnectionRetryIntervalSeconds, }; MemoizationStore.Interfaces.Caches.ICache localCache; if (cacheConfig.EnableMetadataServer) { localCache = LocalCache.CreateRpcCache(logger, serviceClientConfiguration); } else { var metadataRootPath = new AbsolutePath(cacheConfig.MetadataRootPath); localCache = LocalCache.CreateRpcContentStoreInProcMemoizationStoreCache(logger, metadataRootPath, serviceClientConfiguration, CreateInProcMemoizationStoreConfiguration(cacheConfig, metadataRootPath)); } var statsFilePath = new AbsolutePath(logPath.Path + ".stats"); var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile); return(cache); }
/// <inheritdoc /> protected override ICache CreateCache(DisposableDirectory testDirectory) { var backendCacheDirectory = testDirectory.Path / "Backend"; FileSystem.CreateDirectory(backendCacheDirectory); var namedCacheRoots = new Dictionary <string, AbsolutePath> { [CacheName] = backendCacheDirectory / "Root" }; var grpcPort = PortExtensions.GetNextAvailablePort(); var serverConfiguration = new LocalServerConfiguration(backendCacheDirectory, namedCacheRoots, grpcPort, FileSystem) { GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it. }; var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(CacheName, new ServiceClientRpcConfiguration(serverConfiguration.GrpcPort), "Scenario-" + Guid.NewGuid()); Func <AbsolutePath, ICache> contentStoreFactory = CreateBackendCache; var serviceClient = new TestInProcessServiceClientCache(Logger, FileSystem, contentStoreFactory, serverConfiguration, serviceClientConfiguration); return(serviceClient); }
/// <summary> /// Initializes a new instance of the <see cref="ReadOnlyServiceClientContentSession"/> class. /// </summary> public ReadOnlyServiceClientContentSession( string name, ImplicitPin implicitPin, ILogger logger, IAbsFileSystem fileSystem, ServiceClientContentSessionTracer sessionTracer, ServiceClientContentStoreConfiguration configuration, Func <IRpcClient>?rpcClientFactory = null) : base(name) { Contract.Requires(name != null); Contract.Requires(logger != null); Contract.Requires(fileSystem != null); _implicitPin = implicitPin; SessionTracer = sessionTracer; Logger = logger; FileSystem = fileSystem; Configuration = configuration; TempFileStreamFactory = new TempFileStreamFactory(FileSystem); RpcClient = (rpcClientFactory ?? GetRpcClient)(); RetryPolicy = configuration.RetryPolicy; }
public async Task ServiceTestAsync() { using var fileSystem = new PassThroughFileSystem(Logger); using var dir = new DisposableDirectory(fileSystem); var cacheDir = dir.Path / "cache"; var dataPath = dir.Path / "data"; var args = new Dictionary <string, string> { ["paths"] = cacheDir.Path, ["names"] = "Default", ["grpcPort"] = "7090", ["LogSeverity"] = "Diagnostic", ["dataRootPath"] = dataPath.Path, ["Scenario"] = "AppTests", ["grpcPortFileName"] = "AppTestsMMF" }; var serviceProcess = RunService("Service", args, Logger); try { await RunAppAsync("ServiceRunning", new Dictionary <string, string> { { "waitSeconds", "5" }, { "Scenario", "AppTests" } }, Logger); var context = new Context(Logger); var config = new ServiceClientContentStoreConfiguration("Default", new ServiceClientRpcConfiguration { GrpcPort = 7090 }, scenario: "AppTests"); using var store = new ServiceClientContentStore(Logger, fileSystem, config); await store.StartupAsync(context).ShouldBeSuccess(); var sessionResult = store.CreateSession(context, "Default", ImplicitPin.None).ShouldBeSuccess(); using var session = sessionResult.Session; await session.StartupAsync(context).ShouldBeSuccess(); var source = dir.Path / "source.txt"; var contents = new byte[1024]; Random.NextBytes(contents); fileSystem.WriteAllBytes(source, contents); var putResult = await session.PutFileAsync(context, HashType.MD5, source, FileRealizationMode.Any, CancellationToken.None).ShouldBeSuccess(); var hash = putResult.ContentHash; await session.PinAsync(context, hash, CancellationToken.None).ShouldBeSuccess(); var destination = dir.Path / "destination.txt"; await session.PlaceFileAsync( context, hash, destination, FileAccessMode.ReadOnly, FileReplacementMode.FailIfExists, FileRealizationMode.Any, CancellationToken.None).ShouldBeSuccess(); fileSystem.ReadAllBytes(destination).Should().BeEquivalentTo(contents); } finally { if (!serviceProcess.HasExited) { serviceProcess.Kill(); #pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead serviceProcess.WaitForExit(); #pragma warning restore AsyncFixer02 } } }
private ServiceClientPublishingCache CreateClientCache(PublishingCacheConfiguration publishingConfig, string pat, string cacheName, int grpcPort, string scenario) { var config = new ServiceClientContentStoreConfiguration(cacheName, new ContentStore.Sessions.ServiceClientRpcConfiguration(grpcPort), scenario); return(new ServiceClientPublishingCache(Logger, FileSystem, config, publishingConfig, pat)); }