Пример #1
0
        internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
        {
            // TODO: Remove check when all clients are updated with unified Dedup flag
            if ((ContentHashingUtilities.HashInfo.HashType.IsValidDedup()) ^ cacheConfig.UseDedupStore)
            {
                var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
                throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
            }

            var credentialProviderHelper = new CredentialProviderHelper(m => logger.Debug(m));

            if (credentialProviderHelper.IsCredentialProviderSpecified())
            {
                logger.Debug($"Credential providers path specified: '{credentialProviderHelper.CredentialHelperPath}'");
            }
            else
            {
                logger.Debug("Using current user's credentials for obtaining AAD token");
            }

            var credentialsFactory = new VssCredentialsFactory(pat, credentialProviderHelper, m => logger.Debug(m));

            logger.Diagnostic("Creating BuildCacheCache factory");
            var fileSystem = new PassThroughFileSystem(logger);

            // TODO: Once write-behind is implemented send a contentstorefunc down to the create.
            Func <IContentStore> writeThroughStore = null;

            if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
            {
                ServiceClientRpcConfiguration rpcConfiguration;
                if (cacheConfig.GrpcPort != 0)
                {
                    rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
                }
                else
                {
                    var factory    = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
                    var portReader = factory.GetPortReader();
                    var port       = portReader.ReadPort();

                    rpcConfiguration = new ServiceClientRpcConfiguration(port);
                }

                writeThroughStore = () =>
                                    new ServiceClientContentStore(
                    logger,
                    fileSystem,
                    cacheConfig.CacheName,
                    rpcConfiguration,
                    cacheConfig.ConnectionRetryIntervalSeconds,
                    cacheConfig.ConnectionRetryCount,
                    scenario: cacheConfig.ScenarioName);
            }

            BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();

            return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
        }
Пример #2
0
 /// <nodoc />
 public GrpcContentClient(
     ServiceClientContentSessionTracer tracer,
     IAbsFileSystem fileSystem,
     ServiceClientRpcConfiguration configuration,
     string?scenario)
     : this(tracer, fileSystem, configuration, scenario, Capabilities.ContentOnly)
 {
 }
Пример #3
0
 /// <nodoc />
 public GrpcCacheClient(
     ServiceClientContentSessionTracer tracer,
     IAbsFileSystem fileSystem,
     ServiceClientRpcConfiguration configuration,
     string scenario)
     : base(tracer, fileSystem, configuration, scenario, Capabilities.All)
 {
 }
Пример #4
0
        private static MemoizationStoreAdapterCache CreateCache(Config cacheConfig, AbsolutePath logPath, ILogger logger)
        {
            ServiceClientRpcConfiguration rpcConfiguration;

            if (cacheConfig.GrpcPort != 0)
            {
                rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
            }
            else
            {
                var factory    = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
                var portReader = factory.GetPortReader();
                var port       = portReader.ReadPort();

                rpcConfiguration = new ServiceClientRpcConfiguration(port);
            }

            var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName)
            {
                RetryCount            = cacheConfig.ConnectionRetryCount,
                RetryIntervalSeconds  = cacheConfig.ConnectionRetryIntervalSeconds,
                TraceOperationStarted = cacheConfig.GrpcTraceOperationStarted,
            };

            MemoizationStore.Interfaces.Caches.ICache localCache;
            if (cacheConfig.EnableMetadataServer)
            {
                localCache = LocalCache.CreateRpcCache(logger, serviceClientConfiguration);
            }
            else
            {
                var metadataRootPath = new AbsolutePath(cacheConfig.MetadataRootPath);

                localCache = LocalCache.CreateRpcContentStoreInProcMemoizationStoreCache(logger,
                                                                                         metadataRootPath,
                                                                                         serviceClientConfiguration,
                                                                                         CreateInProcMemoizationStoreConfiguration(cacheConfig, metadataRootPath));
            }

            var statsFilePath = new AbsolutePath(logPath.Path + ".stats");

            if (!string.IsNullOrEmpty(cacheConfig.VfsCasRoot))
            {
                logger.Debug($"Creating virtualized cache");

                localCache = new VirtualizedContentCache(localCache, new ContentStore.Vfs.VfsCasConfiguration.Builder()
                {
                    RootPath    = new AbsolutePath(cacheConfig.VfsCasRoot),
                    UseSymlinks = cacheConfig.UseVfsSymlinks
                }.Build());
            }

            var cache = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile);

            return(cache);
        }
Пример #5
0
 private void RunServiceClientContentStore(string cacheName, ServiceClientRpcConfiguration rpcConfiguration, Func <Context, IContentSession, Task> funcAsync)
 {
     System.Func <IContentStore> createFunc = () => new ServiceClientContentStore(
         _logger, _fileSystem, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, _scenario)
     {
         RetryCount           = _retryCount,
         RetryIntervalSeconds = _retryIntervalSeconds,
     });
     RunContentStore(createFunc, funcAsync);
 }
Пример #6
0
 /// <nodoc />
 protected GrpcContentClient(
     ServiceClientContentSessionTracer tracer,
     IAbsFileSystem fileSystem,
     ServiceClientRpcConfiguration configuration,
     string?scenario,
     Capabilities capabilities = Capabilities.ContentOnly)
     : base(fileSystem, tracer, configuration, scenario, capabilities)
 {
     Client = new ContentServer.ContentServerClient(Channel);
 }
 /// <nodoc />
 public GrpcContentClient(
     ServiceClientContentSessionTracer tracer,
     IAbsFileSystem fileSystem,
     ServiceClientRpcConfiguration configuration,
     string scenario,
     Capabilities capabilities = Capabilities.ContentOnly)
     : base(fileSystem, tracer, configuration, scenario, capabilities)
 {
     GrpcEnvironment.InitializeIfNeeded();
     Client = new ContentServer.ContentServerClient(Channel);
 }
 /// <nodoc />
 public GrpcPublishingCacheClient(
     ServiceClientContentSessionTracer tracer,
     IAbsFileSystem fileSystem,
     ServiceClientRpcConfiguration configuration,
     string scenario,
     PublishingCacheConfiguration publishingConfig,
     string pat)
     : base(tracer, fileSystem, configuration, scenario, Capabilities.All)
 {
     _serializedPublishingConfig = DynamicJson.Serialize(publishingConfig);
     _pat = pat;
 }
Пример #9
0
        private async Task RunServerTestAsync(Context context, string scenario, Func <Context, ServiceConfiguration, IRpcClient, Task> funcAsync)
        {
            using (var directory = new DisposableDirectory(FileSystem))
            {
                var storeConfig = new ContentStoreConfiguration(new MaxSizeQuota($"{1 * 1024 * 1024}"));
                await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);

                var serviceConfig =
                    new ServiceConfiguration(
                        new Dictionary <string, AbsolutePath> {
                    { CacheName, directory.Path }
                },
                        directory.Path,
                        ServiceConfiguration.DefaultMaxConnections,
                        ServiceConfiguration.DefaultGracefulShutdownSeconds,
                        PortExtensions.GetNextAvailablePort(),
                        Guid.NewGuid().ToString())
                {
                    TraceGrpcOperation = true
                };

                using (var server = new LocalContentServer(FileSystem, Logger, scenario, path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path), new LocalServerConfiguration(serviceConfig)))
                {
                    BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);

                    r.ShouldBeSuccess();
                    var configuration = new ServiceClientRpcConfiguration()
                    {
                        GrpcPort = (int)serviceConfig.GrpcPort,
                    };

                    using (var rpcClient = new GrpcContentClient(new ServiceClientContentSessionTracer(scenario), FileSystem, configuration, scenario))
                    {
                        try
                        {
                            var createSessionResult = await rpcClient.CreateSessionAsync(new OperationContext(context), SessionName, CacheName, ImplicitPin.None);

                            createSessionResult.ShouldBeSuccess();

                            await funcAsync(context, serviceConfig, rpcClient);
                        }
                        finally
                        {
                            (await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
                        }
                    }

                    r = await server.ShutdownAsync(context);

                    r.ShouldBeSuccess();
                }
            }
        }
        /// <nodoc />
        public ServiceClientContentStoreConfiguration(
            string cacheName,
            ServiceClientRpcConfiguration rpcConfiguration,
            string scenario,
            RetryPolicy retryPolicy)
        {
            Contract.Requires(cacheName != null);

            CacheName        = cacheName;
            RpcConfiguration = rpcConfiguration;
            Scenario         = scenario;

            _retryPolicy = new Lazy <RetryPolicy>(() => retryPolicy);
        }
        /// <nodoc />
        public ServiceClientContentStoreConfiguration(
            string cacheName,
            ServiceClientRpcConfiguration rpcConfiguration,
            string scenario = null)
        {
            Contract.Requires(cacheName != null);
            CacheName        = cacheName;
            RpcConfiguration = rpcConfiguration;
            Scenario         = scenario;

            _retryPolicy = new Lazy <RetryPolicy>(
                () => new RetryPolicy(
                    new TransientErrorDetectionStrategy(),
                    new FixedInterval("RetryInterval", (int)RetryCount, TimeSpan.FromSeconds(RetryIntervalSeconds), false)));
        }
Пример #12
0
        private void RunContentStore(string cacheName, string cachePath, ServiceClientRpcConfiguration rpcConfiguration, Func <Context, IContentSession, Task> funcAsync)
        {
            VerifyCachePathOrNameProvided(cacheName, cachePath);

            if (cacheName != null)
            {
                RunServiceClientContentStore(cacheName, rpcConfiguration, funcAsync);
            }
            else
            {
                RunFileSystemContentStore(new AbsolutePath(cachePath), funcAsync);
            }

            PauseUntilKeyboardHit();
        }
 private static ServiceClientContentStoreConfiguration CreateConfiguration(
     string cacheName,
     string scenario,
     ServiceConfiguration serviceConfiguration,
     uint retryIntervalSeconds = DefaultRetryIntervalSeconds,
     uint retryCount           = DefaultRetryCount)
 {
     return(new ServiceClientContentStoreConfiguration(
                cacheName,
                ServiceClientRpcConfiguration.CreateGrpc((int)serviceConfiguration.GrpcPort),
                scenario)
     {
         RetryCount = retryCount,
         RetryIntervalSeconds = retryIntervalSeconds,
     });
 }
Пример #14
0
        protected GrpcClientBase(
            IAbsFileSystem fileSystem,
            ServiceClientContentSessionTracer tracer,
            ServiceClientRpcConfiguration configuration,
            string?scenario,
            Capabilities clientCapabilities)
        {
            FileSystem          = fileSystem;
            ServiceClientTracer = tracer;
            Configuration       = configuration;
            Scenario            = scenario;
            _clientCapabilities = clientCapabilities;

            GrpcEnvironment.WaitUntilInitialized();
            Channel = new Channel(configuration.GrpcHost, configuration.GrpcPort, ChannelCredentials.Insecure, GrpcEnvironment.GetClientOptions(configuration.GrpcCoreClientOptions));
        }
        private static MemoizationStore.Interfaces.Caches.ICache CreateGrpcCache(Config config, DisposeLogger logger)
        {
            Contract.Requires(config.RetryIntervalSeconds >= 0);
            Contract.Requires(config.RetryCount >= 0);

            var serviceClientRpcConfiguration = new ServiceClientRpcConfiguration()
            {
                GrpcCoreClientOptions = config.GrpcCoreClientOptions,
            };

            if (config.GrpcPort > 0)
            {
                serviceClientRpcConfiguration.GrpcPort = config.GrpcPort;
            }

            ServiceClientContentStoreConfiguration serviceClientContentStoreConfiguration = null;

            if (config.EnableContentServer)
            {
                new ServiceClientContentStoreConfiguration(config.CacheName, serviceClientRpcConfiguration, config.ScenarioName)
                {
                    RetryIntervalSeconds   = (uint)config.RetryIntervalSeconds,
                    RetryCount             = (uint)config.RetryCount,
                    GrpcEnvironmentOptions = config.GrpcEnvironmentOptions,
                };
            }

            if (config.EnableContentServer && config.EnableMetadataServer)
            {
                return(LocalCache.CreateRpcCache(logger, serviceClientContentStoreConfiguration));
            }
            else
            {
                Contract.Assert(!config.EnableMetadataServer, "It is not supported to use a Metadata server without a Content server");

                var memoizationStoreConfiguration = GetInProcMemoizationStoreConfiguration(new AbsolutePath(config.CacheRootPath), config, GetCasConfig(config));

                return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(logger,
                                                                                       new AbsolutePath(config.CacheRootPath),
                                                                                       memoizationStoreConfiguration,
                                                                                       new LocalCacheConfiguration(serviceClientContentStoreConfiguration),
                                                                                       configurationModel: CreateConfigurationModel(GetCasConfig(config)),
                                                                                       clock: null,
                                                                                       checkLocalFiles: config.CheckLocalFiles));
            }
        }
Пример #16
0
 public TestServiceClientContentSession(
     string name,
     ImplicitPin implicitPin,
     RetryPolicy retryPolicy,
     AbsolutePath rootPath,
     string cacheName,
     ILogger logger,
     IAbsFileSystem fileSystem,
     string scenario,
     ITestServiceClientContentStore store,
     ServiceClientContentSessionTracer sessionTracer,
     ServiceClientRpcConfiguration rpcConfiguration)
     : base(name, implicitPin, logger, fileSystem, sessionTracer, new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenario, retryPolicy))
 {
     _rootPath = rootPath;
     Store     = store;
 }
        private static ServiceClientRpcConfiguration CreateGrpcClientConfiguration(Config cacheConfig, ILogger logger)
        {
            ServiceClientRpcConfiguration rpcConfiguration = new ServiceClientRpcConfiguration();

            if (cacheConfig.GrpcPort > 0)
            {
                rpcConfiguration.GrpcPort = (int)cacheConfig.GrpcPort;
            }
            else
            {
                var factory    = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
                var portReader = factory.GetPortReader();
                rpcConfiguration.GrpcPort = portReader.ReadPort();
            }
            rpcConfiguration.GrpcCoreClientOptions = cacheConfig.GrpcCoreClientOptions;
            return(rpcConfiguration);
        }
Пример #18
0
        internal void OpenStream
        (
            [Description("Cache root directory path (using in-process cache)")] string cachePath,
            [Description("Cache name (using cache service)")] string cacheName,
            [Required, Description("Content hash value of referenced content to place")] string hash,
            [Description(HashTypeDescription)] string hashType,
            [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName,
            [Description("The GRPC port."), DefaultValue(0)] int grpcPort
        )
        {
            var ht          = GetHashTypeByNameOrDefault(hashType);
            var contentHash = new ContentHash(ht, HexUtilities.HexToBytes(hash));

            ServiceClientRpcConfiguration rpcConfig = null;

            if (cacheName != null)
            {
                if (grpcPort == 0)
                {
                    grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
                }
                rpcConfig = new ServiceClientRpcConfiguration(grpcPort);
            }

            RunContentStore(cacheName, cachePath, rpcConfig, async(context, session) =>
            {
                var r = await session.OpenStreamAsync(context, contentHash, CancellationToken.None).ConfigureAwait(false);

                if (r.Succeeded)
                {
                    using (r.Stream)
                    {
                        var path = _fileSystem.GetTempPath() / $"{contentHash.ToHex()}.dat";
                        using (Stream fileStream = await _fileSystem.OpenSafeAsync(path, FileAccess.Write, FileMode.Create, FileShare.None))
                        {
                            await r.Stream.CopyToAsync(fileStream);
                            context.Always($"Content streamed to file path=[{path}]");
                        }
                    }
                }
                else
                {
                    context.Error(r.ToString());
                }
            });
        }
Пример #19
0
        private static MemoizationStore.Interfaces.Caches.ICache CreateLocalCacheWithSingleCas(Config config, DisposeLogger logger)
        {
            if (config.EnableContentServer && config.EnableMetadataServer)
            {
                Contract.Assert(config.RetryIntervalSeconds >= 0);
                Contract.Assert(config.RetryCount >= 0);

                var rpcConfiguration           = new ServiceClientRpcConfiguration(config.GrpcPort);
                var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(config.CacheName, rpcConfiguration, config.ScenarioName)
                {
                    RetryIntervalSeconds = (uint)config.RetryIntervalSeconds,
                    RetryCount           = (uint)config.RetryCount,
                };

                return(LocalCache.CreateRpcCache(logger, serviceClientConfiguration));
            }
            else
            {
                Contract.Assert(!config.EnableMetadataServer, "It is not supported to use a Metadata server without a Content server");

                LocalCacheConfiguration localCacheConfiguration;
                if (config.EnableContentServer)
                {
                    localCacheConfiguration = LocalCacheConfiguration.CreateServerEnabled(
                        config.GrpcPort,
                        config.CacheName,
                        config.ScenarioName,
                        config.RetryIntervalSeconds,
                        config.RetryCount);
                }
                else
                {
                    localCacheConfiguration = LocalCacheConfiguration.CreateServerDisabled();
                }

                return(LocalCache.CreateUnknownContentStoreInProcMemoizationStoreCache(logger,
                                                                                       new AbsolutePath(config.CacheRootPath),
                                                                                       GetInProcMemoizationStoreConfiguration(new AbsolutePath(config.CacheRootPath), config, GetCasConfig(config)),
                                                                                       localCacheConfiguration,
                                                                                       configurationModel: CreateConfigurationModel(GetCasConfig(config)),
                                                                                       clock: null,
                                                                                       checkLocalFiles: config.CheckLocalFiles,
                                                                                       emptyFileHashShortcutEnabled: config.EmptyFileHashShortcutEnabled));
            }
        }
Пример #20
0
        /// <nodoc />
        protected GrpcClientBase(
            IAbsFileSystem fileSystem,
            ServiceClientContentSessionTracer tracer,
            ServiceClientRpcConfiguration configuration,
            string scenario,
            Capabilities clientCapabilities,
            TimeSpan?heartbeatInterval = null)
        {
            FileSystem          = fileSystem;
            ServiceClientTracer = tracer;
            _configuration      = configuration;
            Scenario            = scenario;

            GrpcEnvironment.InitializeIfNeeded();
            Channel             = new Channel(configuration.GrpcHost ?? GrpcEnvironment.Localhost, configuration.GrpcPort, ChannelCredentials.Insecure, GrpcEnvironment.DefaultConfiguration);
            _clientCapabilities = clientCapabilities;
            _heartbeatInterval  = _configuration.HeartbeatInterval ?? TimeSpan.FromMinutes(DefaultHeartbeatIntervalMinutes);
        }
Пример #21
0
 /// <nodoc />
 public ServiceClientContentStore(
     ILogger logger,
     IAbsFileSystem fileSystem,
     string cacheName,
     ServiceClientRpcConfiguration rpcConfiguration,
     uint retryIntervalSeconds,
     uint retryCount,
     string scenario = null)
     : this(
         logger,
         fileSystem,
         new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenario)
 {
     RetryCount = retryCount,
     RetryIntervalSeconds = retryIntervalSeconds
 })
 {
 }
        private static MemoizationStoreAdapterCache CreateCache(Config cacheConfig, AbsolutePath logPath, ILogger logger)
        {
            ServiceClientRpcConfiguration rpcConfiguration;

            if (cacheConfig.GrpcPort != 0)
            {
                rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
            }
            else
            {
                var factory    = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
                var portReader = factory.GetPortReader();
                var port       = portReader.ReadPort();

                rpcConfiguration = new ServiceClientRpcConfiguration(port);
            }

            var serviceClientConfiguration = new ServiceClientContentStoreConfiguration(cacheConfig.CacheName, rpcConfiguration, cacheConfig.ScenarioName)
            {
                RetryCount           = cacheConfig.ConnectionRetryCount,
                RetryIntervalSeconds = cacheConfig.ConnectionRetryIntervalSeconds,
            };

            MemoizationStore.Interfaces.Caches.ICache localCache;
            if (cacheConfig.EnableMetadataServer)
            {
                localCache = LocalCache.CreateRpcCache(logger, serviceClientConfiguration);
            }
            else
            {
                var metadataRootPath = new AbsolutePath(cacheConfig.MetadataRootPath);

                localCache = LocalCache.CreateRpcContentStoreInProcMemoizationStoreCache(logger,
                                                                                         metadataRootPath,
                                                                                         serviceClientConfiguration,
                                                                                         CreateInProcMemoizationStoreConfiguration(cacheConfig, metadataRootPath));
            }

            var statsFilePath = new AbsolutePath(logPath.Path + ".stats");
            var cache         = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile);

            return(cache);
        }
Пример #23
0
 /// <summary>
 /// Backward compat constructor.
 /// </summary>
 public ServiceClientContentStore(
     ILogger logger,
     IAbsFileSystem fileSystem,
     string cacheName,
     ServiceClientRpcConfiguration rpcConfiguration,
     uint retryIntervalSeconds,
     uint retryCount,
     Sensitivity sensitivity, // Not used. Left for backward compatibility.
     string scenario = null)
     : this(
         logger,
         fileSystem,
         cacheName,
         rpcConfiguration,
         retryIntervalSeconds,
         retryCount,
         scenario)
 {
 }
Пример #24
0
        /// <nodoc />
        public ServiceClientContentStoreConfiguration(
            string cacheName,
            ServiceClientRpcConfiguration rpcConfiguration,
            string?scenario = null)
        {
            Contract.RequiresNotNullOrEmpty(cacheName);
            CacheName        = cacheName;
            RpcConfiguration = rpcConfiguration;
            Scenario         = scenario;

            _retryPolicy = new Lazy <IRetryPolicy>(
                () =>
            {
                var strategy = new TransientErrorDetectionStrategy();
                return(RetryPolicyFactory.GetLinearPolicy(
                           shouldRetry: e => strategy.IsTransient(e),
                           (int)RetryCount,
                           TimeSpan.FromSeconds(RetryIntervalSeconds)));
            });
        }
Пример #25
0
        internal void PutStream
        (
            [Description("Cache root directory path (using in-process cache)")] string cachePath,
            [Description("Cache name (using cache service)")] string cacheName,
            [DefaultValue(100), Description("Content size in bytes")] int size,
            [Description(HashTypeDescription)] string hashType,
            [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName,
            [Description("The GRPC port."), DefaultValue(0)] int grpcPort
        )
        {
            var ht = GetHashTypeByNameOrDefault(hashType);

            ServiceClientRpcConfiguration rpcConfig = null;

            if (cacheName != null)
            {
                if (grpcPort == 0)
                {
                    grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
                }
                rpcConfig = new ServiceClientRpcConfiguration(grpcPort);
            }

            RunContentStore(cacheName, cachePath, rpcConfig, async(context, session) =>
            {
                using (var stream = new MemoryStream(ThreadSafeRandom.GetBytes(size)))
                {
                    PutResult result = await session.PutStreamAsync(
                        context, ht, stream, CancellationToken.None).ConfigureAwait(false);

                    if (!result.Succeeded)
                    {
                        context.Error(result.ToString());
                    }
                    else
                    {
                        context.Always($"Content added with hash=[{result.ContentHash}], size=[{result.ContentSize}]");
                    }
                }
            });
        }
Пример #26
0
 /// <summary>
 ///     Initializes a new instance of the <see cref="LocalCache" /> class backed by <see cref="ServiceClientContentStore"/>
 /// </summary>
 public LocalCache(
     ILogger logger,
     string cacheName,
     AbsolutePath rootPath,
     ServiceClientRpcConfiguration rpcConfiguration,
     uint retryIntervalSeconds,
     uint retryCount,
     SQLiteMemoizationStoreConfiguration memoConfig,
     IClock clock        = null,
     string scenarioName = null)
     : this(
         logger,
         rootPath,
         new PassThroughFileSystem(logger),
         clock ?? SystemClock.Instance,
         new ServiceClientContentStoreConfiguration(cacheName, rpcConfiguration, scenarioName)
 {
     RetryCount = retryCount,
     RetryIntervalSeconds = retryIntervalSeconds,
 },
         memoConfig)
 {
 }
        public async Task MultipleCaches()
        {
            const string CacheName1 = "test1";
            const string CacheName2 = "test2";

            using (var testDirectory0 = new DisposableDirectory(FileSystem))
                using (var testDirectory1 = new DisposableDirectory(FileSystem))
                    using (var testDirectory2 = new DisposableDirectory(FileSystem))
                    {
                        var config = CreateStoreConfiguration();

                        var rootPath1 = testDirectory1.Path;
                        config.Write(FileSystem, rootPath1);

                        var rootPath2 = testDirectory2.Path;
                        config.Write(FileSystem, rootPath2);

                        var grpcPort         = PortExtensions.GetNextAvailablePort();
                        var grpcPortFileName = Guid.NewGuid().ToString();

                        var serviceConfiguration = new ServiceConfiguration(
                            new Dictionary <string, AbsolutePath> {
                            { CacheName1, rootPath1 }, { CacheName2, rootPath2 }
                        },
                            testDirectory0.Path,
                            ServiceConfiguration.DefaultGracefulShutdownSeconds,
                            grpcPort,
                            grpcPortFileName);

                        using (var server = CreateServer(serviceConfiguration))
                        {
                            var factory   = new MemoryMappedFileGrpcPortSharingFactory(Logger, grpcPortFileName);
                            var reader    = factory.GetPortReader();
                            var port      = reader.ReadPort();
                            var rpcConfig = new ServiceClientRpcConfiguration(port);

                            using (var store1 = new ServiceClientContentStore(
                                       Logger,
                                       FileSystem,
                                       new ServiceClientContentStoreConfiguration(CacheName1, rpcConfig, Scenario)))
                                using (var store2 = new ServiceClientContentStore(
                                           Logger,
                                           FileSystem,
                                           new ServiceClientContentStoreConfiguration(CacheName2, rpcConfig, Scenario)))
                                {
                                    try
                                    {
                                        var rs = await server.StartupAsync(_context);

                                        rs.ShouldBeSuccess();

                                        var storeBoolResult1 = await store1.StartupAsync(_context);

                                        storeBoolResult1.ShouldBeSuccess();

                                        var storeBoolResult2 = await store2.StartupAsync(_context);

                                        storeBoolResult2.ShouldBeSuccess();

                                        IContentSession session1 = null;
                                        IContentSession session2 = null;

                                        try
                                        {
                                            var createSessionResult1 = store1.CreateSession(_context, "session1", ImplicitPin.None);
                                            createSessionResult1.ShouldBeSuccess();

                                            var createSessionResult2 = store2.CreateSession(_context, "session2", ImplicitPin.None);
                                            createSessionResult2.ShouldBeSuccess();

                                            using (createSessionResult1.Session)
                                                using (createSessionResult2.Session)
                                                {
                                                    var r1 = await createSessionResult1.Session.StartupAsync(_context);

                                                    r1.ShouldBeSuccess();
                                                    session1 = createSessionResult1.Session;

                                                    var r2 = await createSessionResult2.Session.StartupAsync(_context);

                                                    r2.ShouldBeSuccess();
                                                    session2 = createSessionResult2.Session;

                                                    var r3 = await session1.PutRandomAsync(
                                                        _context,
                                                        ContentHashType,
                                                        false,
                                                        RandomContentByteCount,
                                                        Token);

                                                    var pinResult = await session1.PinAsync(_context, r3.ContentHash, Token);

                                                    pinResult.ShouldBeSuccess();

                                                    r3 = await session2.PutRandomAsync(
                                                        _context,
                                                        ContentHashType,
                                                        false,
                                                        RandomContentByteCount,
                                                        Token);

                                                    pinResult = await session2.PinAsync(_context, r3.ContentHash, Token);

                                                    pinResult.ShouldBeSuccess();
                                                }
                                        }
                                        finally
                                        {
                                            if (session2 != null)
                                            {
                                                await session2.ShutdownAsync(_context).ShouldBeSuccess();
                                            }

                                            if (session1 != null)
                                            {
                                                await session1.ShutdownAsync(_context).ShouldBeSuccess();
                                            }
                                        }
                                    }
                                    finally
                                    {
                                        BoolResult r1 = null;
                                        BoolResult r2 = null;

                                        if (store1.StartupCompleted)
                                        {
                                            r1 = await store1.ShutdownAsync(_context);
                                        }

                                        if (store2.StartupCompleted)
                                        {
                                            r2 = await store2.ShutdownAsync(_context);
                                        }

                                        var r3 = await server.ShutdownAsync(_context);

                                        r1?.ShouldBeSuccess();
                                        r2?.ShouldBeSuccess();
                                        r3?.ShouldBeSuccess();
                                    }
                                }
                        }
                    }
        }
Пример #28
0
        /// <inheritdoc />
        public async Task <Possible <ICache, Failure> > InitializeCacheAsync(ICacheConfigData cacheData, Guid activityId)
        {
            Contract.Requires(cacheData != null);

            var possibleCacheConfig = cacheData.Create <Config>();

            if (!possibleCacheConfig.Succeeded)
            {
                return(possibleCacheConfig.Failure);
            }

            Config cacheConfig = possibleCacheConfig.Result;

            try
            {
                Contract.Assert(cacheConfig.CacheName != null);
                var logPath = new AbsolutePath(cacheConfig.MetadataLogPath);
                var logger  = new DisposeLogger(() => new EtwFileLog(logPath.Path, cacheConfig.CacheId), cacheConfig.LogFlushIntervalSeconds);

                logger.Debug($"Creating CASaaS backed LocalCache using cache name: {cacheConfig.CacheName}");

                ServiceClientRpcConfiguration rpcConfiguration;
                if (cacheConfig.GrpcPort != 0)
                {
                    rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
                }
                else
                {
                    var factory    = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
                    var portReader = factory.GetPortReader();
                    var port       = portReader.ReadPort();

                    rpcConfiguration = new ServiceClientRpcConfiguration(port);
                }

                var rootPath   = new AbsolutePath(cacheConfig.MetadataRootPath);
                var memoConfig = new SQLiteMemoizationStoreConfiguration(rootPath)
                {
                    MaxRowCount                  = cacheConfig.MaxStrongFingerprints,
                    BackupDatabase               = cacheConfig.BackupLKGCache,
                    VerifyIntegrityOnStartup     = cacheConfig.CheckCacheIntegrityOnStartup,
                    SingleInstanceTimeoutSeconds = (int)cacheConfig.SingleInstanceTimeoutInSeconds
                };

                if (!string.IsNullOrEmpty(cacheConfig.SynchronizationMode))
                {
                    memoConfig.SyncMode = (SynchronizationMode)Enum.Parse(typeof(SynchronizationMode), cacheConfig.SynchronizationMode, ignoreCase: true);
                }

                var localCache = new LocalCache(
                    logger,
                    cacheConfig.CacheName,
                    rootPath,
                    rpcConfiguration,
                    cacheConfig.ConnectionRetryIntervalSeconds,
                    cacheConfig.ConnectionRetryCount,
                    memoConfig,
                    scenarioName: cacheConfig.ScenarioName);

                var statsFilePath = new AbsolutePath(logPath.Path + ".stats");
                var cache         = new MemoizationStoreAdapterCache(cacheConfig.CacheId, localCache, logger, statsFilePath, cacheConfig.ReplaceExistingOnPlaceFile);

                var startupResult = await cache.StartupAsync();

                if (!startupResult.Succeeded)
                {
                    logger.Error($"Error while initializing the cache [{cacheConfig.CacheId}]. Failure: {startupResult.Failure}");
                    return(startupResult.Failure);
                }

                logger.Debug("Successfully started CloudStoreLocalCacheService client.");
                return(cache);
            }
            catch (Exception e)
            {
                return(new CacheConstructionFailure(cacheConfig.CacheId, e));
            }
        }
Пример #29
0
        internal void PlaceFile
        (
            [Description("Cache root directory path (using in-process cache)")] string cachePath,
            [Description("Cache name (using cache service)")] string cacheName,
            [Required, Description("Content hash value of referenced content to place")] string hash,
            [Required, Description("Path to destination file")] string path,
            [Description(HashTypeDescription)] string hashType,
            [DefaultValue(FileAccessMode.ReadOnly)] FileAccessMode accessMode,
            [DefaultValue(FileReplacementMode.ReplaceExisting)] FileReplacementMode replacementMode,
            [DefaultValue(FileRealizationMode.HardLink)] FileRealizationMode realizationMode,
            [DefaultValue(false), Description("Stream bytes if true")] bool useStream,
            [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified.")] string grpcPortFileName,
            [Description("The GRPC port."), DefaultValue(0)] int grpcPort
        )
        {
            var ht          = GetHashTypeByNameOrDefault(hashType);
            var contentHash = new ContentHash(ht, HexUtilities.HexToBytes(hash));
            var filePath    = new AbsolutePath(path);

            ServiceClientRpcConfiguration rpcConfig = null;

            if (cacheName != null)
            {
                if (grpcPort == 0)
                {
                    grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
                }
                rpcConfig = new ServiceClientRpcConfiguration(grpcPort);
            }

            RunContentStore(cacheName, cachePath, rpcConfig, async(context, session) =>
            {
                if (useStream)
                {
                    var r = await session.OpenStreamAsync(context, contentHash, CancellationToken.None).ConfigureAwait(false);

                    if (r.Succeeded)
                    {
                        using (r.Stream)
                        {
                            using (var fileStream = File.OpenWrite(filePath.Path))
                            {
                                await r.Stream.CopyToAsync(fileStream);
                                context.Always("Success");
                            }
                        }
                    }
                    else
                    {
                        context.Error(r.ToString());
                    }
                }
                else
                {
                    var r = await session.PlaceFileAsync(
                        context,
                        contentHash,
                        filePath,
                        accessMode,
                        replacementMode,
                        realizationMode,
                        CancellationToken.None).ConfigureAwait(false);

                    if (!r.Succeeded)
                    {
                        context.Error(r.ToString());
                    }
                    else
                    {
                        context.Always("Success");
                    }
                }
            });
        }
Пример #30
0
        internal static BuildXL.Cache.MemoizationStore.Interfaces.Caches.ICache CreateBuildCacheCache <T>(T cacheConfig, ILogger logger, string pat = null) where T : BuildCacheCacheConfig
        {
            // TODO: Remove check when all clients are updated with unified Dedup flag
            if (ContentHashingUtilities.HashInfo.HashType == HashType.DedupNodeOrChunk ^ cacheConfig.UseDedupStore)
            {
                var store = cacheConfig.UseDedupStore ? "DedupStore" : "BlobStore";
                throw new ArgumentException($"HashType {ContentHashingUtilities.HashInfo.HashType} cannot be used with {store}");
            }

            string credentialProviderPath = Environment.GetEnvironmentVariable(CredentialProvidersPathEnvVariable);

            if (!string.IsNullOrWhiteSpace(credentialProviderPath))
            {
                logger.Debug($"Credential providers path specified: {credentialProviderPath}");
            }
            else
            {
                logger.Debug("Using current user's credentials for obtaining AAD token");
            }

            VssCredentialsFactory credentialsFactory;

#if !PLATFORM_OSX
            string userName = null; // when running on .NET Framework, user name doesn't have to be explicitly provided
#if FEATURE_CORECLR
            userName = GetAadUserNameUpn();
#endif
            credentialsFactory = new VssCredentialsFactory(new VsoCredentialHelper(s => logger.Debug(s)), userName);
#else
            var secPat = new SecureString();
            if (!string.IsNullOrWhiteSpace(pat))
            {
                foreach (char c in pat)
                {
                    secPat.AppendChar(c);
                }
            }
            else
            {
                throw new ArgumentException("PAT must be supplied when running with CoreCLR");
            }

            credentialsFactory = new VssCredentialsFactory(new VssBasicCredential(new NetworkCredential(string.Empty, secPat)));
#endif

            logger.Diagnostic("Creating BuildCacheCache factory");
            var fileSystem = new PassThroughFileSystem(logger);

            // TODO: Once write-behind is implemented send a contentstorefunc down to the create.
            Func <IContentStore> writeThroughStore = null;
            if (!string.IsNullOrWhiteSpace(cacheConfig.CacheName))
            {
                ServiceClientRpcConfiguration rpcConfiguration;
                if (cacheConfig.GrpcPort != 0)
                {
                    rpcConfiguration = new ServiceClientRpcConfiguration((int)cacheConfig.GrpcPort);
                }
                else
                {
                    var factory    = new MemoryMappedFileGrpcPortSharingFactory(logger, cacheConfig.GrpcPortFileName);
                    var portReader = factory.GetPortReader();
                    var port       = portReader.ReadPort();

                    rpcConfiguration = new ServiceClientRpcConfiguration(port);
                }

                writeThroughStore = () =>
                                    new ServiceClientContentStore(
                    logger,
                    fileSystem,
                    cacheConfig.CacheName,
                    rpcConfiguration,
                    cacheConfig.ConnectionRetryIntervalSeconds,
                    cacheConfig.ConnectionRetryCount,
                    scenario: cacheConfig.ScenarioName);
            }

            BuildCacheServiceConfiguration buildCacheServiceConfiguration = cacheConfig.AsBuildCacheServiceConfigurationFile();
            return(BuildCacheCacheFactory.Create(fileSystem, logger, credentialsFactory, buildCacheServiceConfiguration, writeThroughStore));
        }