public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
FilesCopied.AddOrUpdate(destinationPath, p => path, (dest, prevPath) => overwrite ? path : prevPath);
if (!File.Exists(path.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {path} doesn't exist."));
}
if (File.Exists(destinationPath.Path))
{
if (!overwrite)
{
return(new CopyFileResult(
CopyFileResult.ResultCode.DestinationPathError,
$"Destination file {destinationPath} exists but overwrite not specified."));
}
}
if (FilesToCorrupt.ContainsKey(path))
{
TestGlobal.Logger.Debug($"Corrupting file {path}");
#pragma warning disable AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes
await Task.Run(
() => File.WriteAllBytes(destinationPath.Path, ThreadSafeRandom.GetBytes(150)));
#pragma warning restore AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes
}
else
{
await Task.Run(() => File.Copy(path.Path, destinationPath.Path), cancellationToken);
}
return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length));
}
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!File.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
Stream s;
if (FilesToCorrupt.ContainsKey(sourcePath))
{
TestGlobal.Logger.Debug($"Corrupting file {sourcePath}");
s = new MemoryStream(ThreadSafeRandom.GetBytes((int)expectedContentSize));
}
else
{
s = File.OpenRead(sourcePath.Path);
}
return(await s.CopyToAsync(destinationStream).ContinueWith((_) => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
private async Task <CopyFileResult> CopyToAsyncCore(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, CopyOptions options)
{
try
{
if (CopyDelay != null)
{
await Task.Delay(CopyDelay.Value);
}
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!_fileSystem.FileExists(sourcePath))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
using Stream s = await GetStreamAsync(sourcePath);
await s.CopyToAsync(destinationStream);
return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition));
}
catch (Exception e)
{
return(new CopyFileResult(CopyResultCode.DestinationPathError, e));
}
}
public async Task CopyFailsForWrongHash()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero);
var hash = ContentHash.Random();
var wrongHash = VsoHashInfo.Instance.EmptyHash;
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(42);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(wrongHash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain(hash.ToShortString());
result.ErrorMessage.Should().Contain(wrongHash.ToShortString());
}
}
public async Task CopyFailsForWrongCopySize()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path);
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyFileAsyncResult = CopyFileResult.SuccessWithSize(41);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain("size");
result.ErrorMessage.Should().Contain("mismatch");
}
}
public async Task CopyWithDesignatedLocations()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var machineLocations = new MachineLocation[] { new MachineLocation("Non-designated"), new MachineLocation("Designated") };
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), designatedLocations: new MachineLocation[] { machineLocations[1] });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(99);
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
var destinationResult = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
destinationResult.ShouldBeSuccess();
mockFileCopier.CopyAttempts.Should().Be(1);
distributedCopier.PathTransformer.LastContentLocation.Should().BeEquivalentTo(machineLocations[1].Data);
}
}
public async Task CopyFailsForWrongCopySize()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var hash = VsoHashInfo.Instance.EmptyHash;
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(41);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain("size");
result.ErrorMessage.Should().Contain("mismatch");
}
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
// NOTE: Assumes both source and destination are local
Contract.Assert(path.IsLocal);
Contract.Assert(destinationPath.IsLocal);
if (!FileUtilities.Exists(path.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, $"Source file {path} doesn't exist."));
}
if (FileUtilities.Exists(destinationPath.Path))
{
if (!overwrite)
{
return(new CopyFileResult(
CopyFileResult.ResultCode.DestinationPathError,
$"Destination file {destinationPath} exists but overwrite not specified."));
}
}
if (!await FileUtilities.CopyFileAsync(path.Path, destinationPath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to copy {destinationPath} from {path}"));
}
return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length));
}
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!File.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
using Stream s = GetStream(sourcePath, expectedContentSize);
await s.CopyToAsync(destinationStream);
return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition));
}
/// <summary>
/// Copies file to stream
/// </summary>
public async Task <CopyFileResult> CopyToAsync(Context context, ContentHash contentHash, Stream stream, CancellationToken cancellationToken, long fileSize = -1, bool enableCompression = false)
{
// TODO: Pipe through flag for compression type
CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None;
long bytesReceived = 0L;
try
{
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest
{
TraceId = context.Id.ToString(),
HashType = (int)contentHash.HashType,
ContentHash = contentHash.ToByteString(),
// TODO: If `Drive` is expected to be the drive of the file on the source machine, then this should have nothing to do with the destination's drive
Drive = "B",
Offset = 0,
Compression = compression
});
bytesReceived = await StreamContentAsync(stream, response.ResponseStream);
}
catch (RpcException r) when(r.StatusCode == StatusCode.Unavailable && r.Message.Contains("Connect Failed"))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r, $"Failed to connect to server {_host} at port {_grpcPort}"));
}
if (bytesReceived == 0)
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Received {bytesReceived} bytes for {contentHash}. Source file does not exist"));
}
else if (fileSize >= 0 && bytesReceived != fileSize)
{
return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}"));
}
return(CopyFileResult.SuccessWithSize(bytesReceived));
}
/// <inheritdoc />
public virtual async Task <CopyFileResult> CopyToAsync(OperationContext context, ContentLocation sourceLocation, Stream destinationStream, CopyOptions options)
{
var sourcePath = new AbsolutePath(sourceLocation.Machine.Path) / FileSystemContentStoreInternal.GetPrimaryRelativePath(sourceLocation.Hash, includeSharedFolder: false);
if (!FileUtilities.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
long startPosition = destinationStream.Position;
using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
// NOTE: Assumes source is local
Contract.Assert(sourcePath.IsLocal);
if (!FileUtilities.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
long startPosition = destinationStream.Position;
using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
return(await s.CopyToAsync(destinationStream, 81920, cancellationToken).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
}