/// <summary> /// Checks if file exists on remote machine. /// </summary> public async Task <FileExistenceResult> CheckFileExistsAsync(Context context, ContentHash hash) { try { var request = new ExistenceRequest() { TraceId = context.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString() }; ExistenceResponse response = await _client.CheckFileExistsAsync(request); if (response.Header.Succeeded) { return(new FileExistenceResult()); } else { return(new FileExistenceResult(FileExistenceResult.ResultCode.FileNotFound, response.Header.ErrorMessage)); } } catch (RpcException r) { if (r.StatusCode == StatusCode.Unavailable) { return(new FileExistenceResult(FileExistenceResult.ResultCode.SourceError, r)); } else { return(new FileExistenceResult(FileExistenceResult.ResultCode.Error, r)); } } }
private async Task <PutResult> PutFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode, int sessionId) { DateTime startTime = DateTime.UtcNow; PutFileResponse response = await RunClientActionAndThrowIfFailedAsync(context, async() => await _client.PutFileAsync( new PutFileRequest { Header = new RequestHeader(context.Id, sessionId), ContentHash = contentHash.ToByteString(), HashType = (int)contentHash.HashType, FileRealizationMode = (int)realizationMode, Path = path.Path })); long ticksWaited = response.Header.ServerReceiptTimeUtcTicks - startTime.Ticks; _tracer.TrackClientWaitForServerTicks(ticksWaited); if (!response.Header.Succeeded) { await ResetOnUnknownSessionAsync(context, response.Header, sessionId); return(new PutResult(contentHash, response.Header.ErrorMessage, response.Header.Diagnostics)); } else { return(new PutResult(response.ContentHash.ToContentHash((HashType)response.HashType), response.ContentSize)); } }
/// <inheritdoc /> public async Task <PinResult> PinAsync(Context context, ContentHash contentHash) { StructResult <int> sessionResult = await _sessionState.GetIdAsync(); if (!sessionResult.Succeeded) { return(new PinResult(sessionResult)); } int sessionId = sessionResult.Data; DateTime startTime = DateTime.UtcNow; PinResponse response = await RunClientActionAndThrowIfFailedAsync( context, async() => await _client.PinAsync( new PinRequest { HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Header = new RequestHeader(context.Id, sessionId) })); long ticksWaited = response.Header.ServerReceiptTimeUtcTicks - startTime.Ticks; _tracer.TrackClientWaitForServerTicks(ticksWaited); await ResetOnUnknownSessionAsync(context, response.Header, sessionId); return(UnpackPinResult(response.Header)); }
/// <inheritdoc /> public async Task <DeleteResult> DeleteContentAsync(Context context, ContentHash hash) { try { DeleteContentRequest request = new DeleteContentRequest() { TraceId = context.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString() }; DeleteContentResponse response = await _client.DeleteAsync(request); if (response.Header.Succeeded) { return(new DeleteResult((DeleteResult.ResultCode)response.Result, hash, response.EvictedSize, response.PinnedSize)); } else { return(new DeleteResult((DeleteResult.ResultCode)response.Result, response.Header.ErrorMessage, response.Header.Diagnostics)); } } catch (RpcException r) { if (r.StatusCode == StatusCode.Unavailable) { return(new DeleteResult(DeleteResult.ResultCode.ServerError, r)); } else { return(new DeleteResult(DeleteResult.ResultCode.Error, r)); } } }
private Task <PutResult> PutFileAsync( SessionContext context, ContentHash contentHash, AbsolutePath path, FileRealizationMode realizationMode) { return(PerformOperationAsync( context, sessionContext => _client.PutFileAsync( new PutFileRequest { Header = sessionContext.CreateHeader(), ContentHash = contentHash.ToByteString(), HashType = (int)contentHash.HashType, FileRealizationMode = (int)realizationMode, Path = path.Path }), response => { if (!response.Header.Succeeded) { return new PutResult(contentHash, response.Header.ErrorMessage, response.Header.Diagnostics); } else { return new PutResult(response.ContentHash.ToContentHash((HashType)response.HashType), response.ContentSize); } })); }
public void RoundtripContentHash() { ContentHash hash = ContentHash.Random(HashType.Vso0); ByteString byteString = hash.ToByteString(); ContentHash roundtripContentHash = byteString.ToContentHash(HashType.Vso0); roundtripContentHash.Should().Be(hash); }
/// <nodoc /> public static ContentHashAndHashTypeData ToGrpc(this ContentHash input) { return(new ContentHashAndHashTypeData() { ContentHash = input.ToByteString(), HashType = (int)input.HashType, }); }
/// <inheritdoc /> public async Task <DeleteResult> DeleteContentAsync(OperationContext context, ContentHash hash, bool deleteLocalOnly) { try { DeleteContentRequest request = new DeleteContentRequest() { TraceId = context.TracingContext.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString(), DeleteLocalOnly = deleteLocalOnly }; DeleteContentResponse response = await Client.DeleteAsync(request, options : GetCallOptions(Configuration.Deadline, context.Token)); if (!deleteLocalOnly) { var deleteResultsMapping = new Dictionary <string, DeleteResult>(); foreach (var kvp in response.DeleteResults) { var header = kvp.Value; var deleteResult = string.IsNullOrEmpty(header.ErrorMessage) ? new DeleteResult( (DeleteResult.ResultCode)header.Result, hash, response.ContentSize) : new DeleteResult((DeleteResult.ResultCode)header.Result, header.ErrorMessage, header.Diagnostics); deleteResultsMapping.Add(kvp.Key, deleteResult); } return(new DistributedDeleteResult(hash, response.ContentSize, deleteResultsMapping)); } if (response.Header.Succeeded) { return(new DeleteResult((DeleteResult.ResultCode)response.Result, hash, response.ContentSize)); } else { return(new DeleteResult((DeleteResult.ResultCode)response.Result, response.Header.ErrorMessage, response.Header.Diagnostics)); } } catch (RpcException r) { if (r.StatusCode == StatusCode.Unavailable) { return(new DeleteResult(DeleteResult.ResultCode.ServerError, r)); } else { return(new DeleteResult(DeleteResult.ResultCode.Error, r)); } } }
/// <inheritdoc /> public Task <PinResult> PinAsync(Context context, ContentHash contentHash) { return(PerformOperationAsync( new OperationContext(context), sessionContext => _client.PinAsync( new PinRequest { HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Header = sessionContext.CreateHeader(), }), response => UnpackPinResult(response.Header) )); }
private async Task <PlaceFileResult> PlaceFileAsync( Context context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, int sessionId) { var startTime = DateTime.UtcNow; PlaceFileResponse response = await RunClientActionAndThrowIfFailedAsync(context, async() => await _client.PlaceFileAsync( new PlaceFileRequest { Header = new RequestHeader(context.Id, sessionId), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Path = path.Path, FileAccessMode = (int)accessMode, FileRealizationMode = (int)realizationMode, FileReplacementMode = (int)replacementMode })); long ticksWaited = response.Header.ServerReceiptTimeUtcTicks - startTime.Ticks; _tracer.TrackClientWaitForServerTicks(ticksWaited); // Workaround: Handle the service returning negative result codes in error cases PlaceFileResult.ResultCode resultCode = response.Header.Result < 0 ? PlaceFileResult.ResultCode.Error : (PlaceFileResult.ResultCode)response.Header.Result; if (!response.Header.Succeeded) { await ResetOnUnknownSessionAsync(context, response.Header, sessionId); var message = string.IsNullOrEmpty(response.Header.ErrorMessage) ? resultCode.ToString() : response.Header.ErrorMessage; return(new PlaceFileResult(resultCode, message, response.Header.Diagnostics)); } else { return(new PlaceFileResult(resultCode, response.ContentSize)); } }
private Task <PlaceFileResult> PlaceFileAsync( OperationContext operationContext, SessionContext context, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode) { return(PerformOperationAsync( context, _ => Client.PlaceFileAsync( new PlaceFileRequest { Header = context.CreateHeader(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Path = path.Path, FileAccessMode = (int)accessMode, FileRealizationMode = (int)realizationMode, FileReplacementMode = (int)replacementMode }, options: GetCallOptions(Configuration.PlaceDeadline, operationContext.Token)), response => { // Workaround: Handle the service returning negative result codes in error cases PlaceFileResult.ResultCode resultCode = response.Header.Result < 0 ? PlaceFileResult.ResultCode.Error : (PlaceFileResult.ResultCode)response.Header.Result; if (!response.Header.Succeeded) { var message = string.IsNullOrEmpty(response.Header.ErrorMessage) ? resultCode.ToString() : response.Header.ErrorMessage; return new PlaceFileResult(resultCode, message, response.Header.Diagnostics); } else { return new PlaceFileResult(resultCode, response.ContentSize); } })); }
/// <summary> /// Requests host to copy a file from another source machine. /// </summary> public async Task <BoolResult> RequestCopyFileAsync(OperationContext context, ContentHash hash) { try { var request = new RequestCopyFileRequest { TraceId = context.TracingContext.Id.ToString(), ContentHash = hash.ToByteString(), HashType = (int)hash.HashType }; var response = await _client.RequestCopyFileAsync(request, cancellationToken : context.Token); return(response.Header.Succeeded ? BoolResult.Success : new BoolResult(response.Header.ErrorMessage)); } catch (RpcException r) { return(new BoolResult(r)); } }
/// <summary> /// Copies file to stream /// </summary> public async Task <CopyFileResult> CopyToAsync(Context context, ContentHash contentHash, Stream stream, CancellationToken cancellationToken, long fileSize = -1, bool enableCompression = false) { // TODO: Pipe through flag for compression type CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None; long bytesReceived = 0L; try { AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest { TraceId = context.Id.ToString(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), // TODO: If `Drive` is expected to be the drive of the file on the source machine, then this should have nothing to do with the destination's drive Drive = "B", Offset = 0, Compression = compression }); bytesReceived = await StreamContentAsync(stream, response.ResponseStream); } catch (RpcException r) when(r.StatusCode == StatusCode.Unavailable && r.Message.Contains("Connect Failed")) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r, $"Failed to connect to server {_host} at port {_grpcPort}")); } if (bytesReceived == 0) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Received {bytesReceived} bytes for {contentHash}. Source file does not exist")); } else if (fileSize >= 0 && bytesReceived != fileSize) { return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}")); } return(CopyFileResult.SuccessWithSize(bytesReceived)); }
/// <summary> /// Copies file to stream /// </summary> public async Task <CopyFileResult> CopyFileAsync(Context context, ContentHash contentHash, AbsolutePath destinationPath, long fileSize = -1, bool enableCompression = false) { // TODO: Pipe through flag for compression type CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None; AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest() { TraceId = context.Id.ToString(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), Drive = destinationPath.DriveLetter.ToString(), Offset = 0, Compression = compression }); IAsyncStreamReader <CopyFileResponse> replyStream = response.ResponseStream; long bytesReceived = 0L; using (var stream = new FileStream(destinationPath.Path, FileMode.Create, FileAccess.Write, FileShare.None, ContentStore.Grpc.Utils.DefaultBufferSize, FileOptions.Asynchronous | FileOptions.SequentialScan)) { bytesReceived = await StreamContentAsync(stream, response.ResponseStream); while (await replyStream.MoveNext(CancellationToken.None)) { CopyFileResponse oneOfManyReply = replyStream.Current; bytesReceived += oneOfManyReply.Content.Length; oneOfManyReply.Content.WriteTo(stream); } } if (fileSize >= 0 && bytesReceived != fileSize) { return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}")); } return(CopyFileResult.Success); }
/// <summary> /// Copies content from the server to the stream returned by the factory. /// </summary> private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, Func <Stream> streamFactory) { try { CopyFileRequest request = new CopyFileRequest() { TraceId = context.TracingContext.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString(), Offset = 0, Compression = Key.UseCompression ? CopyCompression.Gzip : CopyCompression.None }; AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request); Metadata headers = await response.ResponseHeadersAsync; // If the remote machine couldn't be contacted, GRPC returns an empty // header collection. GRPC would throw an RpcException when we tried // to stream response, but by that time we would have created target // stream. To avoid that, exit early instead. if (headers.Count == 0) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}.")); } // Parse header collection. string exception = null; string message = null; CopyCompression compression = CopyCompression.None; foreach (Metadata.Entry header in headers) { switch (header.Key) { case "exception": exception = header.Value; break; case "message": message = header.Value; break; case "compression": Enum.TryParse(header.Value, out compression); break; } } // Process reported server-side errors. if (exception != null) { Contract.Assert(message != null); switch (exception) { case "ContentNotFound": return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, message)); default: return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, message)); } } // We got headers back with no errors, so create the target stream. Stream targetStream; try { targetStream = streamFactory(); } catch (Exception targetException) { return(new CopyFileResult(CopyFileResult.ResultCode.DestinationPathError, targetException)); } // Copy the content to the target stream. using (targetStream) { switch (compression) { case CopyCompression.None: await StreamContentAsync(targetStream, response.ResponseStream, context.Token); break; case CopyCompression.Gzip: await StreamContentWithCompressionAsync(targetStream, response.ResponseStream, context.Token); break; default: throw new NotSupportedException($"CopyCompression {compression} is not supported."); } } return(CopyFileResult.Success); } catch (RpcException r) { if (r.StatusCode == StatusCode.Unavailable) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r)); } else { return(new CopyFileResult(CopyFileResult.ResultCode.Unknown, r)); } } }
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, CopyOptions options, Func <Stream> streamFactory, bool closeStream) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token); var token = cts.Token; bool exceptionThrown = false; TimeSpan? headerResponseTime = null; CopyFileResult?result = null; try { CopyFileRequest request = new CopyFileRequest() { TraceId = context.TracingContext.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString(), Offset = 0, Compression = _configuration.UseGzipCompression ? CopyCompression.Gzip : CopyCompression.None, FailFastIfBusy = options.BandwidthConfiguration?.FailFastIfServerIsBusy ?? false, }; using AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request, options: GetDefaultGrpcOptions(token)); Metadata headers; var stopwatch = StopwatchSlim.Start(); try { var connectionTimeout = GetResponseHeadersTimeout(options); headers = await response.ResponseHeadersAsync.WithTimeoutAsync(connectionTimeout, token); headerResponseTime = stopwatch.Elapsed; } catch (TimeoutException t) { // Trying to cancel the back end operation as well. cts.Cancel(); result = new CopyFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t); return(result); } // If the remote machine couldn't be contacted, GRPC returns an empty // header collection. GRPC would throw an RpcException when we tried // to stream response, but by that time we would have created target // stream. To avoid that, exit early instead. if (headers.Count == 0) { result = new CopyFileResult( CopyResultCode.ServerUnavailable, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}."); return(result); } // Parse header collection. string? exception = null; string? message = null; CopyCompression compression = CopyCompression.None; foreach (Metadata.Entry header in headers) { switch (header.Key) { case "exception": exception = header.Value; break; case "message": message = header.Value; break; case "compression": Enum.TryParse(header.Value, out compression); break; } } // Process reported server-side errors. if (exception != null) { Contract.Assert(message != null); switch (exception) { case "ContentNotFound": result = new CopyFileResult(CopyResultCode.FileNotFoundError, message); return(result); default: result = new CopyFileResult(CopyResultCode.UnknownServerError, message); return(result); } } // We got headers back with no errors, so create the target stream. Stream targetStream; try { targetStream = streamFactory(); } catch (Exception targetException) { result = new CopyFileResult(CopyResultCode.DestinationPathError, targetException); return(result); } result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync( context, innerToken => copyToCoreImplementation(response, compression, targetStream, innerToken), options, getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics)); return(result); } catch (RpcException r) { result = CreateResultFromException(r); return(result); } catch (Exception) { exceptionThrown = true; throw; } finally { // Even though we don't expect exceptions in this method, we can't assume they won't happen. // So asserting that the result is not null only when the method completes successfully or with a known errors. Contract.Assert(exceptionThrown || result != null); if (result != null) { result.HeaderResponseTime = headerResponseTime; } } async Task <CopyFileResult> copyToCoreImplementation(AsyncServerStreamingCall <CopyFileResponse> response, CopyCompression compression, Stream targetStream, CancellationToken token) { // Copy the content to the target stream. try { switch (compression) { case CopyCompression.None: await StreamContentAsync(response.ResponseStream, targetStream, options, token); break; case CopyCompression.Gzip: await StreamContentWithCompressionAsync(response.ResponseStream, targetStream, options, token); break; default: throw new NotSupportedException($"CopyCompression {compression} is not supported."); } } finally { if (closeStream) { #pragma warning disable AsyncFixer02 // A disposable object used in a fire & forget async call targetStream.Dispose(); #pragma warning restore AsyncFixer02 // A disposable object used in a fire & forget async call } } return(CopyFileResult.Success); } }