public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { FilesCopied.AddOrUpdate(destinationPath, p => path, (dest, prevPath) => overwrite ? path : prevPath); if (!File.Exists(path.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {path} doesn't exist.")); } if (File.Exists(destinationPath.Path)) { if (!overwrite) { return(new CopyFileResult( CopyFileResult.ResultCode.DestinationPathError, $"Destination file {destinationPath} exists but overwrite not specified.")); } } if (FilesToCorrupt.ContainsKey(path)) { TestGlobal.Logger.Debug($"Corrupting file {path}"); #pragma warning disable AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes await Task.Run( () => File.WriteAllBytes(destinationPath.Path, ThreadSafeRandom.GetBytes(150))); #pragma warning restore AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes } else { await Task.Run(() => File.Copy(path.Path, destinationPath.Path), cancellationToken); } return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length)); }
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!File.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } Stream s; if (FilesToCorrupt.ContainsKey(sourcePath)) { TestGlobal.Logger.Debug($"Corrupting file {sourcePath}"); s = new MemoryStream(ThreadSafeRandom.GetBytes((int)expectedContentSize)); } else { s = File.OpenRead(sourcePath.Path); } return(await s.CopyToAsync(destinationStream).ContinueWith((_) => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); }
private async Task <CopyFileResult> CopyToAsyncCore(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, CopyOptions options) { try { if (CopyDelay != null) { await Task.Delay(CopyDelay.Value); } long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!_fileSystem.FileExists(sourcePath)) { return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist.")); } using Stream s = await GetStreamAsync(sourcePath); await s.CopyToAsync(destinationStream); return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)); } catch (Exception e) { return(new CopyFileResult(CopyResultCode.DestinationPathError, e)); } }
public async Task CopyFailsForWrongHash() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero); var hash = ContentHash.Random(); var wrongHash = VsoHashInfo.Instance.EmptyHash; var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(42); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(wrongHash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain(hash.ToShortString()); result.ErrorMessage.Should().Contain(wrongHash.ToShortString()); } }
public async Task CopyFailsForWrongCopySize() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path); var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyFileAsyncResult = CopyFileResult.SuccessWithSize(41); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain("size"); result.ErrorMessage.Should().Contain("mismatch"); } }
public async Task CopyWithDesignatedLocations() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var machineLocations = new MachineLocation[] { new MachineLocation("Non-designated"), new MachineLocation("Designated") }; var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), designatedLocations: new MachineLocation[] { machineLocations[1] }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(99); var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); var destinationResult = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); destinationResult.ShouldBeSuccess(); mockFileCopier.CopyAttempts.Should().Be(1); distributedCopier.PathTransformer.LastContentLocation.Should().BeEquivalentTo(machineLocations[1].Data); } }
public async Task CopyFailsForWrongCopySize() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var hash = VsoHashInfo.Instance.EmptyHash; var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(41); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain("size"); result.ErrorMessage.Should().Contain("mismatch"); } }
/// <inheritdoc /> public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { // NOTE: Assumes both source and destination are local Contract.Assert(path.IsLocal); Contract.Assert(destinationPath.IsLocal); if (!FileUtilities.Exists(path.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, $"Source file {path} doesn't exist.")); } if (FileUtilities.Exists(destinationPath.Path)) { if (!overwrite) { return(new CopyFileResult( CopyFileResult.ResultCode.DestinationPathError, $"Destination file {destinationPath} exists but overwrite not specified.")); } } if (!await FileUtilities.CopyFileAsync(path.Path, destinationPath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to copy {destinationPath} from {path}")); } return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length)); }
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!File.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } using Stream s = GetStream(sourcePath, expectedContentSize); await s.CopyToAsync(destinationStream); return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)); }
/// <summary> /// Copies file to stream /// </summary> public async Task <CopyFileResult> CopyToAsync(Context context, ContentHash contentHash, Stream stream, CancellationToken cancellationToken, long fileSize = -1, bool enableCompression = false) { // TODO: Pipe through flag for compression type CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None; long bytesReceived = 0L; try { AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest { TraceId = context.Id.ToString(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), // TODO: If `Drive` is expected to be the drive of the file on the source machine, then this should have nothing to do with the destination's drive Drive = "B", Offset = 0, Compression = compression }); bytesReceived = await StreamContentAsync(stream, response.ResponseStream); } catch (RpcException r) when(r.StatusCode == StatusCode.Unavailable && r.Message.Contains("Connect Failed")) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r, $"Failed to connect to server {_host} at port {_grpcPort}")); } if (bytesReceived == 0) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Received {bytesReceived} bytes for {contentHash}. Source file does not exist")); } else if (fileSize >= 0 && bytesReceived != fileSize) { return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}")); } return(CopyFileResult.SuccessWithSize(bytesReceived)); }
/// <inheritdoc /> public virtual async Task <CopyFileResult> CopyToAsync(OperationContext context, ContentLocation sourceLocation, Stream destinationStream, CopyOptions options) { var sourcePath = new AbsolutePath(sourceLocation.Machine.Path) / FileSystemContentStoreInternal.GetPrimaryRelativePath(sourceLocation.Hash, includeSharedFolder: false); if (!FileUtilities.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist.")); } long startPosition = destinationStream.Position; using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan)) { return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); } }
/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { // NOTE: Assumes source is local Contract.Assert(sourcePath.IsLocal); if (!FileUtilities.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } long startPosition = destinationStream.Position; using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan)) { return(await s.CopyToAsync(destinationStream, 81920, cancellationToken).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); } }