/// <inheritdoc />
public async Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize,
CopyToOptions options)
{
// Extract host and contentHash from sourcePath
(string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath);
// Contact hard-coded port on source
using var clientWrapper = await _clientCache.CreateAsync(host, _grpcPort, _useCompression);
return(await clientWrapper.Value.CopyToAsync(context, contentHash, destinationStream, options, context.Token));
}
protected override async Task <CopyFileResult> CopyFileAsync(
OperationContext context,
IRemoteFileCopier <AbsolutePath> copier,
AbsolutePath sourcePath,
AbsolutePath destinationPath,
long expectedContentSize,
bool overwrite,
CancellationToken cancellationToken,
CopyToOptions options)
{
// TODO: why the destination str
using var destinationStream = await FileSystem.OpenSafeAsync(destinationPath, FileAccess.Write, FileMode.Create, FileShare.None, FileOptions.None, 1024);
return(await copier.CopyToAsync(context, sourcePath, destinationStream, expectedContentSize));
}
public void WorkWithCopyOptions()
{
// ExStart:ProjectCopyingWithOptions
// ExFor: CopyToOptions
// ExFor: CopyToOptions.#ctor
// ExFor: CopyToOptions.CopyViewData
// ExSummary: Shows how to use project copy options.
var project = new Project(DataDir + "CopyToProjectEmpty.xml");
File.Copy(DataDir + "CopyToProjectEmpty.mpp", OutDir + "ProjectCopying_out.mpp", true);
var mppProject = new Project(OutDir + "ProjectCopying_out.mpp");
// skip copying of view data while copying common project data.
var copyToOptions = new CopyToOptions();
copyToOptions.CopyViewData = false;
project.CopyTo(mppProject, copyToOptions);
// ExEnd:ProjectCopyingWithOptions
}
public async Task BandwidthCheckTimesOutOnSlowCopy()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec * 2); // Lower limit is twice actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null);
var checker = new BandwidthChecker(checkerConfig);
using (var stream = new MemoryStream())
{
var options = new CopyToOptions();
var result = await checker.CheckBandwidthAtIntervalAsync(
_context,
token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options),
options);
Assert.Equal(CopyResultCode.CopyBandwidthTimeoutError, result.Code);
}
}
public async Task BandwidthCheckDoesNotAffectGoodCopies()
{
var checkInterval = TimeSpan.FromSeconds(1);
var actualBandwidthBytesPerSec = 1024;
var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec);
var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Lower limit is half actual bandwidth
var totalBytes = actualBandwidthBytesPerSec * 2;
var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null);
var checker = new BandwidthChecker(checkerConfig);
using (var stream = new MemoryStream())
{
var options = new CopyToOptions();
var result = await checker.CheckBandwidthAtIntervalAsync(
_context,
token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options),
options);
Assert.True(result.Succeeded);
}
}
/// <summary>
/// Copies object to another object using reflection.
/// </summary>
/// <param name="source">The base class instance.</param>
/// <param name="target">The target.</param>
/// <param name="options"></param>
/// <returns></returns>
public static void CopyTo(this object source, object target, CopyToOptions options)
{
var sourceType = source.GetType();
var ignorePrivate = !options.CopyPrivates;
var sourceProperties = options.PropertiesToInclude != null && options.PropertiesToInclude.Any()
? options.PropertiesToInclude
: TypeRepository.GetProperty(sourceType, ignorePrivate);
var targetType = target.GetType();
var targetProperties = options.PropertiesToInclude != null && options.PropertiesToInclude.Any()
? options.PropertiesToInclude.ToDictionary(i => i.Name)
: TypeRepository.GetProperty(targetType, ignorePrivate).ToDictionary(i => i.Name);
foreach (var propertyInfo in sourceProperties.Except(options.PropertiesToIgnore ?? Enumerable.Empty <PropertyInfo>())
)
{
if (!targetProperties.ContainsKey(propertyInfo.Name))
{
continue;
}
var copyContext = new CopyContext(propertyInfo, source, target);
var rawValue = options.Resolver.Invoke(copyContext);
var value = options.DeepCloneValue ? DeepCloneObject(rawValue) : rawValue;
if (options.IgnoreNulls && value == null)
{
continue;
}
var targetProperty = targetProperties[propertyInfo.Name];
if (options.UseNullableBaseType)
{
var trueSourceType = Nullable.GetUnderlyingType(propertyInfo.PropertyType) ??
propertyInfo.PropertyType;
var trueTargetType =
Nullable.GetUnderlyingType(targetProperty.PropertyType) ??
targetProperty.PropertyType;
if (!targetProperty.CanWrite)
{
continue;
}
if (trueSourceType == trueTargetType)
{
targetProperty.SetValue(target, value);
}
else if (options.TryToConvert && CanConvert(value, propertyInfo, targetProperty, out var x))
{
targetProperty.SetValue(target, x);
}
}
else
{
if (propertyInfo.PropertyType == targetProperty.PropertyType)
{
targetProperty.SetValue(target, value);
}
else if (options.TryToConvert && CanConvert(value, propertyInfo, targetProperty, out var x))
{
targetProperty.SetValue(target, x);
}
}
}
}
public async Task <CopyFileResult> CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task <CopyFileResult> > copyTaskFactory, CopyToOptions options)
{
if (_historicalBandwidthLimitSource != null)
{
var timer = Stopwatch.StartNew();
var(result, bytesCopied) = await impl();
timer.Stop();
// Bandwidth checker expects speed in MiB/s, so convert it.
var speed = bytesCopied / timer.Elapsed.TotalSeconds / BytesInMb;
_historicalBandwidthLimitSource.AddBandwidthRecord(speed);
return(result);
}
else
{
return((await impl()).result);
}
async Task <(CopyFileResult result, long bytesCopied)> impl()
{
// This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash
// (given that the app is configured to fail on unobserved task exceptions).
var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec() * _config.BandwidthLimitMultiplier;
minimumSpeedInMbPerSec = Math.Min(minimumSpeedInMbPerSec, _config.MaxBandwidthLimit);
long startPosition = options.TotalBytesCopied;
long previousPosition = startPosition;
var copyCompleted = false;
using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
Task <CopyFileResult> copyTask = copyTaskFactory(copyCancellation.Token);
// Subscribing for potential task failure here to avoid unobserved task exceptions.
traceCopyTaskFailures();
while (!copyCompleted)
{
// Wait some time for bytes to be copied
var firstCompletedTask = await Task.WhenAny(copyTask,
Task.Delay(_config.BandwidthCheckInterval, context.Token));
copyCompleted = firstCompletedTask == copyTask;
if (copyCompleted)
{
var result = await copyTask;
result.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec;
var bytesCopied = result.Size ?? options.TotalBytesCopied;
return(result, bytesCopied);
}
else if (context.Token.IsCancellationRequested)
{
context.Token.ThrowIfCancellationRequested();
}
// Copy is not completed and operation has not been canceled, perform
// bandwidth check
var position = options.TotalBytesCopied;
var receivedMiB = (position - previousPosition) / BytesInMb;
var currentSpeed = receivedMiB / _config.BandwidthCheckInterval.TotalSeconds;
if (currentSpeed == 0 || currentSpeed < minimumSpeedInMbPerSec)
{
copyCancellation.Cancel();
var totalBytesCopied = position - startPosition;
var result = new CopyFileResult(
CopyResultCode.CopyBandwidthTimeoutError,
$"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {totalBytesCopied} bytes copied (received {position - previousPosition} bytes in {_config.BandwidthCheckInterval.TotalSeconds} seconds).");
return(result, totalBytesCopied);
}
previousPosition = position;
}
var copyFileResult = await copyTask;
copyFileResult.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec;
return(copyFileResult, previousPosition - startPosition);
void traceCopyTaskFailures()
{
// When the operation is cancelled, it is possible for the copy operation to fail.
// In this case we still want to trace the failure (but just with the debug severity and not with the error),
// but we should exclude ObjectDisposedException completely.
// That's why we don't use task.FireAndForget but tracing inside the task's continuation.
copyTask.ContinueWith(t =>
{
if (t.IsFaulted)
{
if (!(t.Exception?.InnerException is ObjectDisposedException))
{
context.TraceDebug($"Checked copy failed. {t.Exception}");
}
}
});
}
}
}
private async Task <CopyFileResult> CopyToAsyncCore(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options)
{
try
{
if (CopyDelay != null)
{
await Task.Delay(CopyDelay.Value);
}
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!File.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
using Stream s = GetStream(sourcePath, expectedContentSize);
await s.CopyToAsync(destinationStream);
return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition));
}
catch (Exception e)
{
return(new CopyFileResult(CopyResultCode.DestinationPathError, e));
}
}
public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options)
{
var result = CopyToAsyncCore(context, sourcePath, destinationStream, expectedContentSize, options);
CopyToAsyncTask = result;
return(result);
}
/// <inheritdoc />
public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options)
{
// The bandwidth checker needs to have an options instance, because it is used for tracking the copy progress as well.
options ??= new CopyToOptions();
return(_checker.CheckBandwidthAtIntervalAsync(
context,
// NOTE: We need to pass through the token from bandwidth checker to ensure copy cancellation for insufficient bandwidth gets triggered.
token => _inner.CopyToAsync(context.WithCancellationToken(token), sourcePath, destinationStream, expectedContentSize, options),
options));
}
public static void CopyTo(this Stream source, Stream destination, CopyToOptions options, CompressionMode mode, DecompressionMethods method = DecompressionMethods.Deflate, int bufferSize = 4096, long maxCount = -1, Action <CopyToEventArgs> onProgress = null)
{
if (mode == CompressionMode.Compress)
{
switch (method)
{
case DecompressionMethods.Deflate:
{
using (DeflateStream deflaterStream = new DeflateStream(destination, CompressionMode.Compress, true))
{
source.CopyTo(deflaterStream, options, bufferSize, maxCount, onProgress);
deflaterStream.Close();
}
}
break;
case DecompressionMethods.GZip:
{
using (GZipStream gzipStream = new GZipStream(destination, CompressionMode.Compress, true))
{
source.CopyTo(gzipStream, options, bufferSize, maxCount, onProgress);
gzipStream.Close();
}
}
break;
default:
{
source.CopyTo(destination, options, bufferSize, maxCount, onProgress);
}
break;
}
}
else
{
switch (method)
{
case DecompressionMethods.Deflate:
{
using (DeflateStream deflaterStream = new DeflateStream(source, CompressionMode.Decompress, true))
{
deflaterStream.CopyTo(destination, options, bufferSize, maxCount, onProgress);
deflaterStream.Close();
}
}
break;
case DecompressionMethods.GZip:
{
using (GZipStream gzipStream = new GZipStream(source, CompressionMode.Decompress, true))
{
gzipStream.CopyTo(destination, options, bufferSize, maxCount, onProgress);
gzipStream.Close();
}
}
break;
default:
{
source.CopyTo(destination, options, bufferSize, maxCount, onProgress);
}
break;
}
}
}
//
public static void CopyTo(this Stream source, Stream destination, CopyToOptions options, int bufferSize = 4096, long maxCount = -1, Action <CopyToEventArgs> onProgress = null)
{
byte[] buffer = new byte[bufferSize];
//
long totalBytesWritten = 0;
//
while (true)
{
int count = buffer.Length;
//
if (maxCount > 0)
{
if (totalBytesWritten > maxCount - count)
{
count = (int)(maxCount - totalBytesWritten);
if (count <= 0)
{
break;
}
}
}
//
int read = source.Read(buffer, 0, count);
if (read <= 0)
{
break;
}
//
destination.Write(buffer, 0, read);
//
if (options == CopyToOptions.Flush)
{
try
{
destination.Flush();
}
// Analysis disable once EmptyGeneralCatchClause
catch
{
// Do Nothing
}
}
//
totalBytesWritten += read;
//
if (onProgress != null)
{
onProgress(new CopyToEventArgs(totalBytesWritten));
}
}
//
if (options == CopyToOptions.FlushFinal)
{
try
{
destination.Flush();
}
// Analysis disable once EmptyGeneralCatchClause
catch
{
// Do Nothing
}
}
}
private static async Task <CopyFileResult> CopyRandomToStreamAtSpeed(CancellationToken token, Stream stream, long totalBytes, double mbPerSec, CopyToOptions options)
{
var interval = TimeSpan.FromSeconds(0.1);
var copied = 0;
var bytesPerInterval = (int)BytesPerInterval(mbPerSec, interval);
Assert.True(bytesPerInterval > 0);
var buffer = new byte[bytesPerInterval];
while (!token.IsCancellationRequested)
{
var intervalTask = Task.Delay(interval);
Random.NextBytes(buffer);
await stream.WriteAsync(buffer, 0, bytesPerInterval);
copied += bytesPerInterval;
options.UpdateTotalBytesCopied(copied);
if (copied >= totalBytes)
{
break;
}
await intervalTask;
}
return(new CopyFileResult());
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options)
{
// NOTE: Assumes source is local
Contract.Assert(sourcePath.IsLocal);
if (!FileUtilities.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
long startPosition = destinationStream.Position;
using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
}
/// <inheritdoc />
public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options)
{
CopyAttempts++;
if (CustomResults != null)
{
return(Task.FromResult(CustomResults[CopyAttempts - 1]));
}
return(Task.FromResult(CopyToAsyncResult));
}