/// <summary> /// Handles a request to copy content to this machine. /// </summary> public async Task HandlePushFileAsync(IAsyncStreamReader <PushFileRequest> requestStream, IServerStreamWriter <PushFileResponse> responseStream, ServerCallContext callContext) { OperationStarted(); var startTime = DateTime.UtcNow; var pushRequest = PushRequest.FromMetadata(callContext.RequestHeaders); var hash = pushRequest.Hash; var cacheContext = new Context(pushRequest.TraceId, Logger); var token = callContext.CancellationToken; var store = _contentStoreByCacheName.Values.OfType <IPushFileHandler>().FirstOrDefault(); if (store == null) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because no stores implement {nameof(IPushFileHandler)}."); await callContext.WriteResponseHeadersAsync(PushResponse.DontCopy.Metadata); return; } if (store.HasContentLocally(cacheContext, hash)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because content is already local."); await callContext.WriteResponseHeadersAsync(PushResponse.DontCopy.Metadata); return; } if (!_ongoingPushes.Add(hash)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because another request to push it is already being handled."); await callContext.WriteResponseHeadersAsync(PushResponse.DontCopy.Metadata); return; } try { await callContext.WriteResponseHeadersAsync(PushResponse.Copy.Metadata); var tempFilePath = AbsolutePath.CreateRandomFileName(_tempDirectory); using (var tempFile = File.OpenWrite(tempFilePath.Path)) { while (await requestStream.MoveNext()) { if (token.IsCancellationRequested) { return; } var request = requestStream.Current; var bytes = request.Content.ToByteArray(); await tempFile.WriteAsync(bytes, 0, bytes.Length); } } var result = await store.HandlePushFileAsync(cacheContext, hash, tempFilePath, token); File.Delete(tempFilePath.Path); var response = result ? new PushFileResponse { Header = ResponseHeader.Success(startTime) } : new PushFileResponse { Header = ResponseHeader.Failure(startTime, result.ErrorMessage) }; await responseStream.WriteAsync(response); } finally { _ongoingPushes.Remove(hash); } }
/// <summary> /// Handles a request to copy content to this machine. /// </summary> public async Task HandlePushFileAsync(IAsyncStreamReader <PushFileRequest> requestStream, IServerStreamWriter <PushFileResponse> responseStream, ServerCallContext callContext) { OperationStarted(); var startTime = DateTime.UtcNow; var pushRequest = PushRequest.FromMetadata(callContext.RequestHeaders); var hash = pushRequest.Hash; var cacheContext = new Context(pushRequest.TraceId, Logger); var token = callContext.CancellationToken; var store = _contentStoreByCacheName.Values.OfType <IPushFileHandler>().FirstOrDefault(); if (store == null) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because no stores implement {nameof(IPushFileHandler)}."); await callContext.WriteResponseHeadersAsync(PushResponse.DontCopy.Metadata); return; } if (store.HasContentLocally(cacheContext, hash)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because content is already local."); await callContext.WriteResponseHeadersAsync(PushResponse.DontCopy.Metadata); return; } if (!_ongoingPushes.Add(hash)) { Tracer.Debug(cacheContext, $"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} skipped because another request to push it is already being handled."); await callContext.WriteResponseHeadersAsync(PushResponse.DontCopy.Metadata); return; } try { await callContext.WriteResponseHeadersAsync(PushResponse.Copy.Metadata); PutResult result = null; using (var disposableFile = new DisposableFile(cacheContext, _fileSystem, _temporaryDirectory.CreateRandomFileName())) { // NOTE(jubayard): DeleteOnClose not used here because the file needs to be placed into the CAS. // Opening a file for read/write and then doing pretty much anything to it leads to weird behavior // that needs to be tested on a case by case basis. Since we don't know what the underlying store // plans to do with the file, it is more robust to just use the DisposableFile construct. using (var tempFile = await _fileSystem.OpenSafeAsync(disposableFile.Path, FileAccess.Write, FileMode.CreateNew, FileShare.None)) { while (await requestStream.MoveNext()) { if (token.IsCancellationRequested) { return; } var request = requestStream.Current; var bytes = request.Content.ToByteArray(); await tempFile.WriteAsync(bytes, 0, bytes.Length); } } result = await store.HandlePushFileAsync(cacheContext, hash, disposableFile.Path, token); } var response = result ? new PushFileResponse { Header = ResponseHeader.Success(startTime) } : new PushFileResponse { Header = ResponseHeader.Failure(startTime, result.ErrorMessage) }; await responseStream.WriteAsync(response); } finally { _ongoingPushes.Remove(hash); } }
public async Task HandlePushFileAsync(IAsyncStreamReader <PushFileRequest> requestStream, IServerStreamWriter <PushFileResponse> responseStream, ServerCallContext callContext) { // Detaching from the calling thread to (potentially) avoid IO Completion port thread exhaustion await Task.Yield(); var startTime = DateTime.UtcNow; var pushRequest = PushRequest.FromMetadata(callContext.RequestHeaders); var hash = pushRequest.Hash; var cacheContext = new Context(pushRequest.TraceId, Logger); // Cancelling the operation when the instance is shut down. using var shutdownTracker = TrackShutdown(cacheContext, callContext.CancellationToken); var token = shutdownTracker.Context.Token; var store = _contentStoreByCacheName.Values.OfType <IPushFileHandler>().FirstOrDefault(); var rejection = CanHandlePushRequest(cacheContext, hash, store); if (rejection != RejectionReason.Accepted) { await callContext.WriteResponseHeadersAsync(PushResponse.DoNotCopy(rejection).Metadata); return; } try { // Running the logic inside try/finally block to remove the hash being processed regardless of the result of this method. await callContext.WriteResponseHeadersAsync(PushResponse.Copy.Metadata); PutResult?result = null; using (var disposableFile = new DisposableFile(cacheContext, _fileSystem, _temporaryDirectory !.CreateRandomFileName())) { // NOTE(jubayard): DeleteOnClose not used here because the file needs to be placed into the CAS. // Opening a file for read/write and then doing pretty much anything to it leads to weird behavior // that needs to be tested on a case by case basis. Since we don't know what the underlying store // plans to do with the file, it is more robust to just use the DisposableFile construct. using (var tempFile = await _fileSystem.OpenSafeAsync(disposableFile.Path, FileAccess.Write, FileMode.CreateNew, FileShare.None)) { // From the docs: On the server side, MoveNext() does not throw exceptions. // In case of a failure, the request stream will appear to be finished (MoveNext will return false) // and the CancellationToken associated with the call will be cancelled to signal the failure. await GrpcExtensions.CopyChunksToStreamAsync(requestStream, tempFile.Stream, request => request.Content, cancellationToken : token); } if (token.IsCancellationRequested) { if (!callContext.CancellationToken.IsCancellationRequested) { await responseStream.WriteAsync(new PushFileResponse() { Header = ResponseHeader.Failure(startTime, "Operation cancelled by handler.") }); } var cancellationSource = callContext.CancellationToken.IsCancellationRequested ? "caller" : "handler"; cacheContext.Debug($"{nameof(HandlePushFileAsync)}: Copy of {hash.ToShortString()} cancelled by {cancellationSource}."); return; } result = await store.HandlePushFileAsync(cacheContext, hash, disposableFile.Path, token); } var response = result ? new PushFileResponse { Header = ResponseHeader.Success(startTime) } : new PushFileResponse { Header = ResponseHeader.Failure(startTime, result.ErrorMessage) }; await responseStream.WriteAsync(response); } finally { lock (_pushesLock) { _ongoingPushes.Remove(hash); } } }
public async Task HandlePushFileAsync(IAsyncStreamReader <PushFileRequest> requestStream, IServerStreamWriter <PushFileResponse> responseStream, ServerCallContext callContext) { // Detaching from the calling thread to (potentially) avoid IO Completion port thread exhaustion await Task.Yield(); var startTime = DateTime.UtcNow; var pushRequest = PushRequest.FromMetadata(callContext.RequestHeaders); var hash = pushRequest.Hash; var cacheContext = new Context(pushRequest.TraceId, Logger); var token = callContext.CancellationToken; var store = _contentStoreByCacheName.Values.OfType <IPushFileHandler>().FirstOrDefault(); if (!CanHandlePushRequest(cacheContext, hash, store)) { await callContext.WriteResponseHeadersAsync(PushResponse.DoNotCopy.Metadata); return; } try { // Running the logic inside try/finally block to remove the hash being processed regardless of the result of this method. await callContext.WriteResponseHeadersAsync(PushResponse.Copy.Metadata); PutResult?result = null; using (var disposableFile = new DisposableFile(cacheContext, _fileSystem, _temporaryDirectory !.CreateRandomFileName())) { // NOTE(jubayard): DeleteOnClose not used here because the file needs to be placed into the CAS. // Opening a file for read/write and then doing pretty much anything to it leads to weird behavior // that needs to be tested on a case by case basis. Since we don't know what the underlying store // plans to do with the file, it is more robust to just use the DisposableFile construct. using (var tempFile = await _fileSystem.OpenSafeAsync(disposableFile.Path, FileAccess.Write, FileMode.CreateNew, FileShare.None)) { while (await requestStream.MoveNext()) { if (token.IsCancellationRequested) { return; } var request = requestStream.Current; var bytes = request.Content.ToByteArray(); await tempFile.WriteAsync(bytes, 0, bytes.Length, token); } } result = await store.HandlePushFileAsync(cacheContext, hash, disposableFile.Path, token); } var response = result ? new PushFileResponse { Header = ResponseHeader.Success(startTime) } : new PushFileResponse { Header = ResponseHeader.Failure(startTime, result.ErrorMessage) }; await responseStream.WriteAsync(response); } finally { _ongoingPushes.Remove(hash); } }