internal void CopyFile( [Required, Description("Machine to copy from")] string host, [Required, Description("Expected content hash")] string hashString, [Required, Description("Path to destination file")] string destinationPath, [Description("Whether or not GZip is enabled"), DefaultValue(false)] bool useCompressionForCopies, [Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified")] string grpcPortFileName, [Description("The GRPC port"), DefaultValue(0)] int grpcPort) { Initialize(); var context = new Context(_logger); var retryPolicy = new RetryPolicy( new TransientErrorDetectionStrategy(), new FixedInterval("RetryInterval", (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds), false)); if (grpcPort == 0) { grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName); } if (!ContentHash.TryParse(hashString, out ContentHash hash)) { throw new CacheException($"Invalid content hash string provided: {hashString}"); } try { using (var rpcClient = GrpcCopyClient.Create(host, grpcPort, useCompressionForCopies)) { var finalPath = new AbsolutePath(destinationPath); // This action is synchronous to make sure the calling application doesn't exit before the method returns. var copyFileResult = retryPolicy.ExecuteAsync(() => rpcClient.CopyFileAsync(context, hash, finalPath, CancellationToken.None)).Result; if (!copyFileResult.Succeeded) { throw new CacheException(copyFileResult.ErrorMessage); } else { _logger.Debug($"Copy of {hashString} to {finalPath} was successful"); } var shutdownResult = rpcClient.ShutdownAsync(context).Result; if (!shutdownResult.Succeeded) { throw new CacheException(shutdownResult.ErrorMessage); } } } catch (Exception ex) { throw new CacheException(ex.ToString()); } }
private async Task RunTestCase(string testName, Func <AbsolutePath, IContentSession, GrpcCopyClient, Task> testAct) { var cacheName = testName + "_cache"; using (var directory = new DisposableDirectory(FileSystem)) { var rootPath = directory.Path; var namedCacheRoots = new Dictionary <string, AbsolutePath> { { cacheName, rootPath } }; var grpcPort = PortExtensions.GetNextAvailablePort(); var grpcPortFileName = Guid.NewGuid().ToString(); var configuration = new ServiceConfiguration( namedCacheRoots, rootPath, 42, ServiceConfiguration.DefaultGracefulShutdownSeconds, grpcPort, grpcPortFileName); var storeConfig = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1); Func <AbsolutePath, IContentStore> contentStoreFactory = (path) => new FileSystemContentStore( FileSystem, SystemClock.Instance, directory.Path, new ConfigurationModel(storeConfig)); var server = new LocalContentServer(FileSystem, Logger, testName, contentStoreFactory, new LocalServerConfiguration(configuration)); await server.StartupAsync(_context).ShouldBeSuccess(); var createSessionResult = await server.CreateSessionAsync(new OperationContext(_context), testName, cacheName, ImplicitPin.PutAndGet, Capabilities.ContentOnly); createSessionResult.ShouldBeSuccess(); (int sessionId, AbsolutePath tempDir) = createSessionResult.Value; var session = server.GetSession(sessionId); // Create a GRPC client to connect to the server var port = new MemoryMappedFilePortReader(grpcPortFileName, Logger).ReadPort(); using (var client = GrpcCopyClient.Create("localhost", port)) { // Run validation await testAct(rootPath, session, client); } await server.ShutdownAsync(_context).ShouldBeSuccess(); } }
/// <inheritdoc /> public async Task <FileExistenceResult> CheckFileExistsAsync(AbsolutePath path, TimeSpan timeout, CancellationToken cancellationToken) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(path); FileExistenceResult fileExistenceResult = null; using (var client = GrpcCopyClient.Create(host, _grpcPort)) { fileExistenceResult = await client.CheckFileExistsAsync(_context, contentHash); } return(fileExistenceResult); }
public async Task WrongPort() { await RunTestCase(nameof(WrongPort), async (rootPath, session, client) => { // Copy fake file out via GRPC var host = "localhost"; var bogusPort = PortExtensions.GetNextAvailablePort(); using (client = GrpcCopyClient.Create(host, bogusPort)) { var copyFileResult = await client.CopyFileAsync(_context, ContentHash.Random(), rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None); Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code); } }); }
/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath); CopyFileResult copyFileResult = null; // Contact hard-coded port on source using (var client = GrpcCopyClient.Create(System.Net.IPAddress.Any.ToString(), DefaultGrpcPort)) { copyFileResult = await client.CopyToAsync(_context, contentHash, destinationStream, cancellationToken, expectedContentSize); } return(copyFileResult); }
/// <inheritdoc /> public async Task <CopyFileResult> CopyFileAsync(AbsolutePath sourcePath, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath); CopyFileResult copyFileResult = null; // Contact hard-coded port on source using (var client = GrpcCopyClient.Create(host, _grpcPort, _useCompression)) { copyFileResult = await client.CopyFileAsync(_context, contentHash, destinationPath, cancellationToken); } return(copyFileResult); }
/// <summary> /// Use an existing <see cref="GrpcCopyClient"/> if possible, else create a new one. /// </summary> public async Task <TResult> UseWithInvalidationAsync <TResult>(OperationContext context, string host, int grpcPort, Func <OperationContext, IResourceWrapperAdapter <GrpcCopyClient>, Task <TResult> > operation) { var key = new GrpcCopyClientKey(host, grpcPort); switch (_configuration.ResourcePoolVersion) { case GrpcCopyClientCacheConfiguration.PoolVersion.Disabled: { var client = new GrpcCopyClient(key, _configuration.GrpcCopyClientConfiguration, sharedBufferPool: _grpcCopyClientBufferPool); await client.StartupAsync(context).ThrowIfFailure(); var result = await operation(context, new DefaultResourceWrapperAdapter <GrpcCopyClient>(client)); await client.ShutdownAsync(context).ThrowIfFailure(); return(result); } case GrpcCopyClientCacheConfiguration.PoolVersion.V1: { Contract.AssertNotNull(_resourcePool); using var resourceWrapper = await _resourcePool.CreateAsync(key); return(await operation(context, new ResourceWrapperV1Adapter <GrpcCopyClient>(resourceWrapper))); } case GrpcCopyClientCacheConfiguration.PoolVersion.V2: { Contract.AssertNotNull(_resourcePoolV2); return(await _resourcePoolV2.UseAsync(context, key, async resourceWrapper => { // This ensures that the operation we want to perform conforms to the cancellation. When the // resource needs to be removed, the token will be cancelled. Once the operation completes, we // will be able to proceed with shutdown. using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.Token, resourceWrapper.ShutdownToken); var nestedContext = new OperationContext(context, cancellationTokenSource.Token); return await operation(nestedContext, new ResourceWrapperV2Adapter <GrpcCopyClient>(resourceWrapper)); })); } } throw new NotImplementedException($"Unhandled resource pool version `{_configuration.ResourcePoolVersion}`"); }
public async Task WrongPort() { await RunTestCase(nameof(WrongPort), async (rootPath, session, client) => { // Write a random file to put into the cache var content = ThreadSafeRandom.GetBytes(FileSize); var contentHash = HashingExtensions.CalculateHash(content, HashType.Vso0); // Copy the file out via GRPC var host = "localhost"; var bogusPort = PortExtensions.GetNextAvailablePort(); using (client = GrpcCopyClient.Create(host, bogusPort)) { var copyFileResult = await client.CopyFileAsync(_context, contentHash, rootPath / ThreadSafeRandom.Generator.Next().ToString(), CancellationToken.None); Assert.Equal(CopyFileResult.ResultCode.SourcePathError, copyFileResult.Code); } }); }
/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync( OperationContext context, ContentLocation sourceLocation, Stream destinationStream, CopyOptions options) { // Extract host and port from machine location (string host, int port) = ExtractHostInfo(sourceLocation.Machine); // Contact hard-coded port on source try { // ResourcePoolV2 may throw TimeoutException if the connection fails. // Wrapping this error and converting it to an "error code". return(await _clientCache.UseWithInvalidationAsync(context, host, port, async (nestedContext, clientWrapper) => { var result = await clientWrapper.Value.CopyToAsync(nestedContext, sourceLocation.Hash, destinationStream, options); InvalidateResourceIfNeeded(nestedContext, options, result, clientWrapper); return result; })); } catch (ResultPropagationException e) { if (e.Result.Exception != null) { return(GrpcCopyClient.CreateResultFromException(e.Result.Exception)); } return(new CopyFileResult(CopyResultCode.Unknown, e.Result)); } catch (Exception e) { return(new CopyFileResult(CopyResultCode.Unknown, e)); } }