internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig { // TODO: Remove check when all clients are updated with unified Dedup flag if ((ContentHashingUtilities.HashInfo.HashType.IsValidDedup()) ^ cacheConfig.UseDedupStore) { var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore"; throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}"); } var credentialProviderHelper = new CredentialProviderHelper(m => logger.Debug(m)); if (credentialProviderHelper.IsCredentialProviderSpecified()) { logger.Debug($"Credential providers path specified: '{credentialProviderHelper.CredentialHelperPath}'"); } else { logger.Debug("Using current user's credentials for obtaining AAD token"); } var credentialsFactory = new VssCredentialsFactory(pat, credentialProviderHelper, m => logger.Debug(m)); logger.Diagnostic("Creating BuildCacheCache factory"); var fileSystem = new PassThroughFileSystem(logger); // TODO: Once write-behind is implemented send a contentstorefunc down to the create. Func <IContentStore> writeThroughStore = null; if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName)) { ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } writeThroughStore = () => new ServiceClientContentStore( logger, fileSystem, cacheConfig.CacheName, rpcConfiguration, cacheConfig.ConnectionRetryIntervalSeconds, cacheConfig.ConnectionRetryCount, scenario: cacheConfig.ScenarioName); } BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile(); return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore)); }
/// <nodoc /> public GrpcContentClient( ServiceClientContentSessionTracer tracer, IAbsFileSystem fileSystem, ServiceClientRpcConfiguration configuration, string?scenario) : this(tracer, fileSystem, configuration, scenario, Capabilities.ContentOnly) { }
/// <nodoc /> public GrpcCacheClient( ServiceClientContentSessionTracer tracer, IAbsFileSystem fileSystem, ServiceClientRpcConfiguration configuration, string scenario) : base(tracer, fileSystem, configuration, scenario, Capabilities.All) { }
private static MemoizationStoreAdapterCache CreateCache(Config cacheConfig, AbsolutePath logPath, ILogger logger) { ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName) { RetryCount = cacheConfig.ConnectionRetryCount, RetryIntervalSeconds = cacheConfig.ConnectionRetryIntervalSeconds, TraceOperationStarted = cacheConfig.GrpcTraceOperationStarted, }; MemoizationStore.Interfaces.Caches.ICache localCache; if (cacheConfig.EnableMetadataServer) { localCache = LocalCache.CreateRpcCache(logger, serviceClientConfiguration); } else { var metadataRootPath = new AbsolutePath(cacheConfig.MetadataRootPath); localCache = LocalCache.CreateRpcContentStoreInProcMemoizationStoreCache(logger, metadataRootPath, serviceClientConfiguration, CreateInProcMemoizationStoreConfiguration(cacheConfig, metadataRootPath)); } var statsFilePath = new AbsolutePath(logPath.Path + ".stats"); if (!string.IsNullOrEmpty(cacheConfig.VfsCasRoot)) { logger.Debug($"Creating virtualized cache"); localCache = new VirtualizedContentCache(localCache, new ContentStore.Vfs.VfsCasConfiguration.Builder() { RootPath = new AbsolutePath(cacheConfig.VfsCasRoot), UseSymlinks = cacheConfig.UseVfsSymlinks }.Build()); } var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile); return(cache); }
private void RunServiceClientContentStore(string cacheName, ServiceClientRpcConfiguration rpcConfiguration, Func <Context, IContentSession, Task> funcAsync) { System.Func <IContentStore> createFunc = () => new ServiceClientContentStore( _logger, _fileSystem, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, _scenario) { RetryCount = _retryCount, RetryIntervalSeconds = _retryIntervalSeconds, }); RunContentStore(createFunc, funcAsync); }
/// <nodoc /> protected GrpcContentClient( ServiceClientContentSessionTracer tracer, IAbsFileSystem fileSystem, ServiceClientRpcConfiguration configuration, string?scenario, Capabilities capabilities = Capabilities.ContentOnly) : base(fileSystem, tracer, configuration, scenario, capabilities) { Client = new ContentServer.ContentServerClient(Channel); }
/// <nodoc /> public GrpcContentClient( ServiceClientContentSessionTracer tracer, IAbsFileSystem fileSystem, ServiceClientRpcConfiguration configuration, string scenario, Capabilities capabilities = Capabilities.ContentOnly) : base(fileSystem, tracer, configuration, scenario, capabilities) { GrpcEnvironment.InitializeIfNeeded(); Client = new ContentServer.ContentServerClient(Channel); }
/// <nodoc /> public GrpcPublishingCacheClient( ServiceClientContentSessionTracer tracer, IAbsFileSystem fileSystem, ServiceClientRpcConfiguration configuration, string scenario, PublishingCacheConfiguration publishingConfig, string pat) : base(tracer, fileSystem, configuration, scenario, Capabilities.All) { _serializedPublishingConfig = DynamicJson.Serialize(publishingConfig); _pat = pat; }
private async Task RunServerTestAsync(Context context, string scenario, Func <Context, ServiceConfiguration, IRpcClient, Task> funcAsync) { using (var directory = new DisposableDirectory(FileSystem)) { var storeConfig = new ContentStoreConfiguration(new MaxSizeQuota($"{1 * 1024 * 1024}")); await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false); var serviceConfig = new ServiceConfiguration( new Dictionary <string, AbsolutePath> { { CacheName, directory.Path } }, directory.Path, ServiceConfiguration.DefaultMaxConnections, ServiceConfiguration.DefaultGracefulShutdownSeconds, PortExtensions.GetNextAvailablePort(), Guid.NewGuid().ToString()) { TraceGrpcOperation = true }; using (var server = new LocalContentServer(FileSystem, Logger, scenario, path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path), new LocalServerConfiguration(serviceConfig))) { BoolResult r = await server.StartupAsync(context).ConfigureAwait(false); r.ShouldBeSuccess(); var configuration = new ServiceClientRpcConfiguration() { GrpcPort = (int)serviceConfig.GrpcPort, }; using (var rpcClient = new GrpcContentClient(new ServiceClientContentSessionTracer(scenario), FileSystem, configuration, scenario)) { try { var createSessionResult = await rpcClient.CreateSessionAsync(new OperationContext(context), SessionName, CacheName, ImplicitPin.None); createSessionResult.ShouldBeSuccess(); await funcAsync(context, serviceConfig, rpcClient); } finally { (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess(); } } r = await server.ShutdownAsync(context); r.ShouldBeSuccess(); } } }
/// <nodoc /> public ServiceClientContentStoreConfiguration( string cacheName, ServiceClientRpcConfiguration rpcConfiguration, string scenario, RetryPolicy retryPolicy) { Contract.Requires(cacheName != null); CacheName = cacheName; RpcConfiguration = rpcConfiguration; Scenario = scenario; _retryPolicy = new Lazy <RetryPolicy>(() => retryPolicy); }
/// <nodoc /> public ServiceClientContentStoreConfiguration( string cacheName, ServiceClientRpcConfiguration rpcConfiguration, string scenario = null) { Contract.Requires(cacheName != null); CacheName = cacheName; RpcConfiguration = rpcConfiguration; Scenario = scenario; _retryPolicy = new Lazy <RetryPolicy>( () => new RetryPolicy( new TransientErrorDetectionStrategy(), new FixedInterval("RetryInterval", (int)RetryCount, TimeSpan.FromSeconds(RetryIntervalSeconds), false))); }
private void RunContentStore(string cacheName, string cachePath, ServiceClientRpcConfiguration rpcConfiguration, Func <Context, IContentSession, Task> funcAsync) { VerifyCachePathOrNameProvided(cacheName, cachePath); if (cacheName != null) { RunServiceClientContentStore(cacheName, rpcConfiguration, funcAsync); } else { RunFileSystemContentStore(new AbsolutePath(cachePath), funcAsync); } PauseUntilKeyboardHit(); }
private static ServiceClientContentStoreConfiguration CreateConfiguration( string cacheName, string scenario, ServiceConfiguration serviceConfiguration, uint retryIntervalSeconds = DefaultRetryIntervalSeconds, uint retryCount = DefaultRetryCount) { return(new ServiceClientContentStoreConfiguration( cacheName, ServiceClientRpcConfiguration.CreateGrpc((int)serviceConfiguration.GrpcPort), scenario) { RetryCount = retryCount, RetryIntervalSeconds = retryIntervalSeconds, }); }
protected GrpcClientBase( IAbsFileSystem fileSystem, ServiceClientContentSessionTracer tracer, ServiceClientRpcConfiguration configuration, string?scenario, Capabilities clientCapabilities) { FileSystem = fileSystem; ServiceClientTracer = tracer; Configuration = configuration; Scenario = scenario; _clientCapabilities = clientCapabilities; GrpcEnvironment.WaitUntilInitialized(); Channel = new Channel(configuration.GrpcHost, configuration.GrpcPort, ChannelCredentials.Insecure, GrpcEnvironment.GetClientOptions(configuration.GrpcCoreClientOptions)); }
private static MemoizationStore.Interfaces.Caches.ICache CreateGrpcCache(Config config, DisposeLogger logger) { Contract.Requires(config.RetryIntervalSeconds >= 0); Contract.Requires(config.RetryCount >= 0); var serviceClientRpcConfiguration = new ServiceClientRpcConfiguration() { GrpcCoreClientOptions = config.GrpcCoreClientOptions, }; if (config.GrpcPort > 0) { serviceClientRpcConfiguration.GrpcPort = config.GrpcPort; } ServiceClientContentStoreConfiguration serviceClientContentStoreConfiguration = null; if (config.EnableContentServer) { new ServiceClientContentStoreConfiguration(config.CacheName, serviceClientRpcConfiguration, config.ScenarioName) { RetryIntervalSeconds = (uint)config.RetryIntervalSeconds, RetryCount = (uint)config.RetryCount, GrpcEnvironmentOptions = config.GrpcEnvironmentOptions, }; } if (config.EnableContentServer && config.EnableMetadataServer) { return(LocalCache.CreateRpcCache(logger, serviceClientContentStoreConfiguration)); } else { Contract.Assert(!config.EnableMetadataServer, "It is not supported to use a Metadata server without a Content server"); var memoizationStoreConfiguration = GetInProcMemoizationStoreConfiguration(new AbsolutePath(config.CacheRootPath), config, GetCasConfig(config)); return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(logger, new AbsolutePath(config.CacheRootPath), memoizationStoreConfiguration, new LocalCacheConfiguration(serviceClientContentStoreConfiguration), configurationModel: CreateConfigurationModel(GetCasConfig(config)), clock: null, checkLocalFiles: config.CheckLocalFiles)); } }
public TestServiceClientContentSession( string name, ImplicitPin implicitPin, RetryPolicy retryPolicy, AbsolutePath rootPath, string cacheName, ILogger logger, IAbsFileSystem fileSystem, string scenario, ITestServiceClientContentStore store, ServiceClientContentSessionTracer sessionTracer, ServiceClientRpcConfiguration rpcConfiguration) : base(name, implicitPin, logger, fileSystem, sessionTracer, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenario, retryPolicy)) { _rootPath = rootPath; Store = store; }
private static ServiceClientRpcConfiguration CreateGrpcClientConfiguration(Config cacheConfig, ILogger logger) { ServiceClientRpcConfiguration rpcConfiguration = new ServiceClientRpcConfiguration(); if (cacheConfig.GrpcPort > 0) { rpcConfiguration.GrpcPort = (int)cacheConfig.GrpcPort; } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); rpcConfiguration.GrpcPort = portReader.ReadPort(); } rpcConfiguration.GrpcCoreClientOptions = cacheConfig.GrpcCoreClientOptions; return(rpcConfiguration); }
internal void OpenStream ( [Description("Cache root directory path (using in-process cache)")] string cachePath, [Description("Cache name (using cache service)")] string cacheName, [Required, Description("Content hash value of referenced content to place")] string hash, [Description(HashTypeDescription)] string hashType, [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName, [Description("The GRPC port."), DefaultValue(0)] int grpcPort ) { var ht = GetHashTypeByNameOrDefault(hashType); var contentHash = new ContentHash(ht, HexUtilities.HexToBytes(hash)); ServiceClientRpcConfiguration rpcConfig = null; if (cacheName != null) { if (grpcPort == 0) { grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName); } rpcConfig = new ServiceClientRpcConfiguration(grpcPort); } RunContentStore(cacheName, cachePath, rpcConfig, async(context, session) => { var r = await session.OpenStreamAsync(context, contentHash, CancellationToken.None).ConfigureAwait(false); if (r.Succeeded) { using (r.Stream) { var path = _fileSystem.GetTempPath() / $"{contentHash.ToHex()}.dat"; using (Stream fileStream = await _fileSystem.OpenSafeAsync(path, FileAccess.Write, FileMode.Create, FileShare.None)) { await r.Stream.CopyToAsync(fileStream); context.Always($"Content streamed to file path=[{path}]"); } } } else { context.Error(r.ToString()); } }); }
private static MemoizationStore.Interfaces.Caches.ICache CreateLocalCacheWithSingleCas(Config config, DisposeLogger logger) { if (config.EnableContentServer && config.EnableMetadataServer) { Contract.Assert(config.RetryIntervalSeconds >= 0); Contract.Assert(config.RetryCount >= 0); var rpcConfiguration = new ServiceClientRpcConfiguration(config.GrpcPort); var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(config.CacheName, rpcConfiguration, config.ScenarioName) { RetryIntervalSeconds = (uint)config.RetryIntervalSeconds, RetryCount = (uint)config.RetryCount, }; return(LocalCache.CreateRpcCache(logger, serviceClientConfiguration)); } else { Contract.Assert(!config.EnableMetadataServer, "It is not supported to use a Metadata server without a Content server"); LocalCacheConfiguration localCacheConfiguration; if (config.EnableContentServer) { localCacheConfiguration = LocalCacheConfiguration.CreateServerEnabled( config.GrpcPort, config.CacheName, config.ScenarioName, config.RetryIntervalSeconds, config.RetryCount); } else { localCacheConfiguration = LocalCacheConfiguration.CreateServerDisabled(); } return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(logger, new AbsolutePath(config.CacheRootPath), GetInProcMemoizationStoreConfiguration(new AbsolutePath(config.CacheRootPath), config, GetCasConfig(config)), localCacheConfiguration, configurationModel: CreateConfigurationModel(GetCasConfig(config)), clock: null, checkLocalFiles: config.CheckLocalFiles, emptyFileHashShortcutEnabled: config.EmptyFileHashShortcutEnabled)); } }
/// <nodoc /> protected GrpcClientBase( IAbsFileSystem fileSystem, ServiceClientContentSessionTracer tracer, ServiceClientRpcConfiguration configuration, string scenario, Capabilities clientCapabilities, TimeSpan?heartbeatInterval = null) { FileSystem = fileSystem; ServiceClientTracer = tracer; _configuration = configuration; Scenario = scenario; GrpcEnvironment.InitializeIfNeeded(); Channel = new Channel(configuration.GrpcHost ?? GrpcEnvironment.Localhost, configuration.GrpcPort, ChannelCredentials.Insecure, GrpcEnvironment.DefaultConfiguration); _clientCapabilities = clientCapabilities; _heartbeatInterval = _configuration.HeartbeatInterval ?? TimeSpan.FromMinutes(DefaultHeartbeatIntervalMinutes); }
/// <nodoc /> public ServiceClientContentStore( ILogger logger, IAbsFileSystem fileSystem, string cacheName, ServiceClientRpcConfiguration rpcConfiguration, uint retryIntervalSeconds, uint retryCount, string scenario = null) : this( logger, fileSystem, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenario) { RetryCount = retryCount, RetryIntervalSeconds = retryIntervalSeconds }) { }
private static MemoizationStoreAdapterCache CreateCache(Config cacheConfig, AbsolutePath logPath, ILogger logger) { ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName) { RetryCount = cacheConfig.ConnectionRetryCount, RetryIntervalSeconds = cacheConfig.ConnectionRetryIntervalSeconds, }; MemoizationStore.Interfaces.Caches.ICache localCache; if (cacheConfig.EnableMetadataServer) { localCache = LocalCache.CreateRpcCache(logger, serviceClientConfiguration); } else { var metadataRootPath = new AbsolutePath(cacheConfig.MetadataRootPath); localCache = LocalCache.CreateRpcContentStoreInProcMemoizationStoreCache(logger, metadataRootPath, serviceClientConfiguration, CreateInProcMemoizationStoreConfiguration(cacheConfig, metadataRootPath)); } var statsFilePath = new AbsolutePath(logPath.Path + ".stats"); var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile); return(cache); }
/// <summary> /// Backward compat constructor. /// </summary> public ServiceClientContentStore( ILogger logger, IAbsFileSystem fileSystem, string cacheName, ServiceClientRpcConfiguration rpcConfiguration, uint retryIntervalSeconds, uint retryCount, Sensitivity sensitivity, // Not used. Left for backward compatibility. string scenario = null) : this( logger, fileSystem, cacheName, rpcConfiguration, retryIntervalSeconds, retryCount, scenario) { }
/// <nodoc /> public ServiceClientContentStoreConfiguration( string cacheName, ServiceClientRpcConfiguration rpcConfiguration, string?scenario = null) { Contract.RequiresNotNullOrEmpty(cacheName); CacheName = cacheName; RpcConfiguration = rpcConfiguration; Scenario = scenario; _retryPolicy = new Lazy <IRetryPolicy>( () => { var strategy = new TransientErrorDetectionStrategy(); return(RetryPolicyFactory.GetLinearPolicy( shouldRetry: e => strategy.IsTransient(e), (int)RetryCount, TimeSpan.FromSeconds(RetryIntervalSeconds))); }); }
internal void PutStream ( [Description("Cache root directory path (using in-process cache)")] string cachePath, [Description("Cache name (using cache service)")] string cacheName, [DefaultValue(100), Description("Content size in bytes")] int size, [Description(HashTypeDescription)] string hashType, [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName, [Description("The GRPC port."), DefaultValue(0)] int grpcPort ) { var ht = GetHashTypeByNameOrDefault(hashType); ServiceClientRpcConfiguration rpcConfig = null; if (cacheName != null) { if (grpcPort == 0) { grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName); } rpcConfig = new ServiceClientRpcConfiguration(grpcPort); } RunContentStore(cacheName, cachePath, rpcConfig, async(context, session) => { using (var stream = new MemoryStream(ThreadSafeRandom.GetBytes(size))) { PutResult result = await session.PutStreamAsync( context, ht, stream, CancellationToken.None).ConfigureAwait(false); if (!result.Succeeded) { context.Error(result.ToString()); } else { context.Always($"Content added with hash=[{result.ContentHash}], size=[{result.ContentSize}]"); } } }); }
/// <summary> /// Initializes a new instance of the <see cref="LocalCache" /> class backed by <see cref="ServiceClientContentStore"/> /// </summary> public LocalCache( ILogger logger, string cacheName, AbsolutePath rootPath, ServiceClientRpcConfiguration rpcConfiguration, uint retryIntervalSeconds, uint retryCount, SQLiteMemoizationStoreConfiguration memoConfig, IClock clock = null, string scenarioName = null) : this( logger, rootPath, new PassThroughFileSystem(logger), clock ?? SystemClock.Instance, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenarioName) { RetryCount = retryCount, RetryIntervalSeconds = retryIntervalSeconds, }, memoConfig) { }
public async Task MultipleCaches() { const string CacheName1 = "test1"; const string CacheName2 = "test2"; using (var testDirectory0 = new DisposableDirectory(FileSystem)) using (var testDirectory1 = new DisposableDirectory(FileSystem)) using (var testDirectory2 = new DisposableDirectory(FileSystem)) { var config = CreateStoreConfiguration(); var rootPath1 = testDirectory1.Path; config.Write(FileSystem, rootPath1); var rootPath2 = testDirectory2.Path; config.Write(FileSystem, rootPath2); var grpcPort = PortExtensions.GetNextAvailablePort(); var grpcPortFileName = Guid.NewGuid().ToString(); var serviceConfiguration = new ServiceConfiguration( new Dictionary <string, AbsolutePath> { { CacheName1, rootPath1 }, { CacheName2, rootPath2 } }, testDirectory0.Path, ServiceConfiguration.DefaultGracefulShutdownSeconds, grpcPort, grpcPortFileName); using (var server = CreateServer(serviceConfiguration)) { var factory = new MemoryMappedFileGrpcPortSharingFactory(Logger, grpcPortFileName); var reader = factory.GetPortReader(); var port = reader.ReadPort(); var rpcConfig = new ServiceClientRpcConfiguration(port); using (var store1 = new ServiceClientContentStore( Logger, FileSystem, new ServiceClientContentStoreConfiguration(CacheName1, rpcConfig, Scenario))) using (var store2 = new ServiceClientContentStore( Logger, FileSystem, new ServiceClientContentStoreConfiguration(CacheName2, rpcConfig, Scenario))) { try { var rs = await server.StartupAsync(_context); rs.ShouldBeSuccess(); var storeBoolResult1 = await store1.StartupAsync(_context); storeBoolResult1.ShouldBeSuccess(); var storeBoolResult2 = await store2.StartupAsync(_context); storeBoolResult2.ShouldBeSuccess(); IContentSession session1 = null; IContentSession session2 = null; try { var createSessionResult1 = store1.CreateSession(_context, "session1", ImplicitPin.None); createSessionResult1.ShouldBeSuccess(); var createSessionResult2 = store2.CreateSession(_context, "session2", ImplicitPin.None); createSessionResult2.ShouldBeSuccess(); using (createSessionResult1.Session) using (createSessionResult2.Session) { var r1 = await createSessionResult1.Session.StartupAsync(_context); r1.ShouldBeSuccess(); session1 = createSessionResult1.Session; var r2 = await createSessionResult2.Session.StartupAsync(_context); r2.ShouldBeSuccess(); session2 = createSessionResult2.Session; var r3 = await session1.PutRandomAsync( _context, ContentHashType, false, RandomContentByteCount, Token); var pinResult = await session1.PinAsync(_context, r3.ContentHash, Token); pinResult.ShouldBeSuccess(); r3 = await session2.PutRandomAsync( _context, ContentHashType, false, RandomContentByteCount, Token); pinResult = await session2.PinAsync(_context, r3.ContentHash, Token); pinResult.ShouldBeSuccess(); } } finally { if (session2 != null) { await session2.ShutdownAsync(_context).ShouldBeSuccess(); } if (session1 != null) { await session1.ShutdownAsync(_context).ShouldBeSuccess(); } } } finally { BoolResult r1 = null; BoolResult r2 = null; if (store1.StartupCompleted) { r1 = await store1.ShutdownAsync(_context); } if (store2.StartupCompleted) { r2 = await store2.ShutdownAsync(_context); } var r3 = await server.ShutdownAsync(_context); r1?.ShouldBeSuccess(); r2?.ShouldBeSuccess(); r3?.ShouldBeSuccess(); } } } } }
/// <inheritdoc /> public async Task <Possible <ICache, Failure> > InitializeCacheAsync(ICacheConfigData cacheData, Guid activityId) { Contract.Requires(cacheData != null); var possibleCacheConfig = cacheData.Create <Config>(); if (!possibleCacheConfig.Succeeded) { return(possibleCacheConfig.Failure); } Config cacheConfig = possibleCacheConfig.Result; try { Contract.Assert(cacheConfig.CacheName != null); var logPath = new AbsolutePath(cacheConfig.MetadataLogPath); var logger = new DisposeLogger(() => new EtwFileLog(logPath.Path, cacheConfig.CacheId), cacheConfig.LogFlushIntervalSeconds); logger.Debug($"Creating CASaaS backed LocalCache using cache name: {cacheConfig.CacheName}"); ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } var rootPath = new AbsolutePath(cacheConfig.MetadataRootPath); var memoConfig = new SQLiteMemoizationStoreConfiguration(rootPath) { MaxRowCount = cacheConfig.MaxStrongFingerprints, BackupDatabase = cacheConfig.BackupLKGCache, VerifyIntegrityOnStartup = cacheConfig.CheckCacheIntegrityOnStartup, SingleInstanceTimeoutSeconds = (int)cacheConfig.SingleInstanceTimeoutInSeconds }; if (!string.IsNullOrEmpty(cacheConfig.SynchronizationMode)) { memoConfig.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), cacheConfig.SynchronizationMode, ignoreCase: true); } var localCache = new LocalCache( logger, cacheConfig.CacheName, rootPath, rpcConfiguration, cacheConfig.ConnectionRetryIntervalSeconds, cacheConfig.ConnectionRetryCount, memoConfig, scenarioName: cacheConfig.ScenarioName); var statsFilePath = new AbsolutePath(logPath.Path + ".stats"); var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile); var startupResult = await cache.StartupAsync(); if (!startupResult.Succeeded) { logger.Error($"Error while initializing the cache [{cacheConfig.CacheId}]. Failure: {startupResult.Failure}"); return(startupResult.Failure); } logger.Debug("Successfully started CloudStoreLocalCacheService client."); return(cache); } catch (Exception e) { return(new CacheConstructionFailure(cacheConfig.CacheId, e)); } }
internal void PlaceFile ( [Description("Cache root directory path (using in-process cache)")] string cachePath, [Description("Cache name (using cache service)")] string cacheName, [Required, Description("Content hash value of referenced content to place")] string hash, [Required, Description("Path to destination file")] string path, [Description(HashTypeDescription)] string hashType, [DefaultValue(FileAccessMode.ReadOnly)] FileAccessMode accessMode, [DefaultValue(FileReplacementMode.ReplaceExisting)] FileReplacementMode replacementMode, [DefaultValue(FileRealizationMode.HardLink)] FileRealizationMode realizationMode, [DefaultValue(false), Description("Stream bytes if true")] bool useStream, [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName, [Description("The GRPC port."), DefaultValue(0)] int grpcPort ) { var ht = GetHashTypeByNameOrDefault(hashType); var contentHash = new ContentHash(ht, HexUtilities.HexToBytes(hash)); var filePath = new AbsolutePath(path); ServiceClientRpcConfiguration rpcConfig = null; if (cacheName != null) { if (grpcPort == 0) { grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName); } rpcConfig = new ServiceClientRpcConfiguration(grpcPort); } RunContentStore(cacheName, cachePath, rpcConfig, async(context, session) => { if (useStream) { var r = await session.OpenStreamAsync(context, contentHash, CancellationToken.None).ConfigureAwait(false); if (r.Succeeded) { using (r.Stream) { using (var fileStream = File.OpenWrite(filePath.Path)) { await r.Stream.CopyToAsync(fileStream); context.Always("Success"); } } } else { context.Error(r.ToString()); } } else { var r = await session.PlaceFileAsync( context, contentHash, filePath, accessMode, replacementMode, realizationMode, CancellationToken.None).ConfigureAwait(false); if (!r.Succeeded) { context.Error(r.ToString()); } else { context.Always("Success"); } } }); }
internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig { // TODO: Remove check when all clients are updated with unified Dedup flag if (ContentHashingUtilities.HashInfo.HashType == HashType.DedupNodeOrChunk ^ cacheConfig.UseDedupStore) { var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore"; throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}"); } string credentialProviderPath = Environment.GetEnvironmentVariable(CredentialProvidersPathEnvVariable); if (!string.IsNullOrWhiteSpace(credentialProviderPath)) { logger.Debug($"Credential providers path specified: {credentialProviderPath}"); } else { logger.Debug("Using current user's credentials for obtaining AAD token"); } VssCredentialsFactory credentialsFactory; #if !PLATFORM_OSX string userName = null; // when running on .NET Framework, user name doesn't have to be explicitly provided #if FEATURE_CORECLR userName = GetAadUserNameUpn(); #endif credentialsFactory = new VssCredentialsFactory(new VsoCredentialHelper(s => logger.Debug(s)), userName); #else var secPat = new SecureString(); if (!string.IsNullOrWhiteSpace(pat)) { foreach (char c in pat) { secPat.AppendChar(c); } } else { throw new ArgumentException("PAT must be supplied when running with CoreCLR"); } credentialsFactory = new VssCredentialsFactory(new VssBasicCredential(new NetworkCredential(string.Empty, secPat))); #endif logger.Diagnostic("Creating BuildCacheCache factory"); var fileSystem = new PassThroughFileSystem(logger); // TODO: Once write-behind is implemented send a contentstorefunc down to the create. Func <IContentStore> writeThroughStore = null; if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName)) { ServiceClientRpcConfiguration rpcConfiguration; if (cacheConfig.GrpcPort != 0) { rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort); } else { var factory = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName); var portReader = factory.GetPortReader(); var port = portReader.ReadPort(); rpcConfiguration = new ServiceClientRpcConfiguration(port); } writeThroughStore = () => new ServiceClientContentStore( logger, fileSystem, cacheConfig.CacheName, rpcConfiguration, cacheConfig.ConnectionRetryIntervalSeconds, cacheConfig.ConnectionRetryCount, scenario: cacheConfig.ScenarioName); } BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile(); return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore)); }