/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath); // Contact hard-coded port on source using var clientWrapper = await _clientCache.CreateAsync(host, _grpcPort, _useCompression); return(await clientWrapper.Value.CopyToAsync(context, contentHash, destinationStream, options, context.Token)); }
protected override async Task <CopyFileResult> CopyFileAsync( OperationContext context, IRemoteFileCopier <AbsolutePath> copier, AbsolutePath sourcePath, AbsolutePath destinationPath, long expectedContentSize, bool overwrite, CancellationToken cancellationToken, CopyToOptions options) { // TODO: why the destination str using var destinationStream = await FileSystem.OpenSafeAsync(destinationPath, FileAccess.Write, FileMode.Create, FileShare.None, FileOptions.None, 1024); return(await copier.CopyToAsync(context, sourcePath, destinationStream, expectedContentSize)); }
public void WorkWithCopyOptions() { // ExStart:ProjectCopyingWithOptions // ExFor: CopyToOptions // ExFor: CopyToOptions.#ctor // ExFor: CopyToOptions.CopyViewData // ExSummary: Shows how to use project copy options. var project = new Project(DataDir + "CopyToProjectEmpty.xml"); File.Copy(DataDir + "CopyToProjectEmpty.mpp", OutDir + "ProjectCopying_out.mpp", true); var mppProject = new Project(OutDir + "ProjectCopying_out.mpp"); // skip copying of view data while copying common project data. var copyToOptions = new CopyToOptions(); copyToOptions.CopyViewData = false; project.CopyTo(mppProject, copyToOptions); // ExEnd:ProjectCopyingWithOptions }
public async Task BandwidthCheckTimesOutOnSlowCopy() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec * 2); // Lower limit is twice actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null); var checker = new BandwidthChecker(checkerConfig); using (var stream = new MemoryStream()) { var options = new CopyToOptions(); var result = await checker.CheckBandwidthAtIntervalAsync( _context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options), options); Assert.Equal(CopyResultCode.CopyBandwidthTimeoutError, result.Code); } }
public async Task BandwidthCheckDoesNotAffectGoodCopies() { var checkInterval = TimeSpan.FromSeconds(1); var actualBandwidthBytesPerSec = 1024; var actualBandwidth = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec); var bandwidthLimit = MbPerSec(bytesPerSec: actualBandwidthBytesPerSec / 2); // Lower limit is half actual bandwidth var totalBytes = actualBandwidthBytesPerSec * 2; var checkerConfig = new BandwidthChecker.Configuration(checkInterval, bandwidthLimit, maxBandwidthLimit: null, bandwidthLimitMultiplier: null, historicalBandwidthRecordsStored: null); var checker = new BandwidthChecker(checkerConfig); using (var stream = new MemoryStream()) { var options = new CopyToOptions(); var result = await checker.CheckBandwidthAtIntervalAsync( _context, token => CopyRandomToStreamAtSpeed(token, stream, totalBytes, actualBandwidth, options), options); Assert.True(result.Succeeded); } }
/// <summary> /// Copies object to another object using reflection. /// </summary> /// <param name="source">The base class instance.</param> /// <param name="target">The target.</param> /// <param name="options"></param> /// <returns></returns> public static void CopyTo(this object source, object target, CopyToOptions options) { var sourceType = source.GetType(); var ignorePrivate = !options.CopyPrivates; var sourceProperties = options.PropertiesToInclude != null && options.PropertiesToInclude.Any() ? options.PropertiesToInclude : TypeRepository.GetProperty(sourceType, ignorePrivate); var targetType = target.GetType(); var targetProperties = options.PropertiesToInclude != null && options.PropertiesToInclude.Any() ? options.PropertiesToInclude.ToDictionary(i => i.Name) : TypeRepository.GetProperty(targetType, ignorePrivate).ToDictionary(i => i.Name); foreach (var propertyInfo in sourceProperties.Except(options.PropertiesToIgnore ?? Enumerable.Empty <PropertyInfo>()) ) { if (!targetProperties.ContainsKey(propertyInfo.Name)) { continue; } var copyContext = new CopyContext(propertyInfo, source, target); var rawValue = options.Resolver.Invoke(copyContext); var value = options.DeepCloneValue ? DeepCloneObject(rawValue) : rawValue; if (options.IgnoreNulls && value == null) { continue; } var targetProperty = targetProperties[propertyInfo.Name]; if (options.UseNullableBaseType) { var trueSourceType = Nullable.GetUnderlyingType(propertyInfo.PropertyType) ?? propertyInfo.PropertyType; var trueTargetType = Nullable.GetUnderlyingType(targetProperty.PropertyType) ?? targetProperty.PropertyType; if (!targetProperty.CanWrite) { continue; } if (trueSourceType == trueTargetType) { targetProperty.SetValue(target, value); } else if (options.TryToConvert && CanConvert(value, propertyInfo, targetProperty, out var x)) { targetProperty.SetValue(target, x); } } else { if (propertyInfo.PropertyType == targetProperty.PropertyType) { targetProperty.SetValue(target, value); } else if (options.TryToConvert && CanConvert(value, propertyInfo, targetProperty, out var x)) { targetProperty.SetValue(target, x); } } } }
public async Task <CopyFileResult> CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task <CopyFileResult> > copyTaskFactory, CopyToOptions options) { if (_historicalBandwidthLimitSource != null) { var timer = Stopwatch.StartNew(); var(result, bytesCopied) = await impl(); timer.Stop(); // Bandwidth checker expects speed in MiB/s, so convert it. var speed = bytesCopied / timer.Elapsed.TotalSeconds / BytesInMb; _historicalBandwidthLimitSource.AddBandwidthRecord(speed); return(result); } else { return((await impl()).result); } async Task <(CopyFileResult result, long bytesCopied)> impl() { // This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash // (given that the app is configured to fail on unobserved task exceptions). var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec() * _config.BandwidthLimitMultiplier; minimumSpeedInMbPerSec = Math.Min(minimumSpeedInMbPerSec, _config.MaxBandwidthLimit); long startPosition = options.TotalBytesCopied; long previousPosition = startPosition; var copyCompleted = false; using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token); Task <CopyFileResult> copyTask = copyTaskFactory(copyCancellation.Token); // Subscribing for potential task failure here to avoid unobserved task exceptions. traceCopyTaskFailures(); while (!copyCompleted) { // Wait some time for bytes to be copied var firstCompletedTask = await Task.WhenAny(copyTask, Task.Delay(_config.BandwidthCheckInterval, context.Token)); copyCompleted = firstCompletedTask == copyTask; if (copyCompleted) { var result = await copyTask; result.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec; var bytesCopied = result.Size ?? options.TotalBytesCopied; return(result, bytesCopied); } else if (context.Token.IsCancellationRequested) { context.Token.ThrowIfCancellationRequested(); } // Copy is not completed and operation has not been canceled, perform // bandwidth check var position = options.TotalBytesCopied; var receivedMiB = (position - previousPosition) / BytesInMb; var currentSpeed = receivedMiB / _config.BandwidthCheckInterval.TotalSeconds; if (currentSpeed == 0 || currentSpeed < minimumSpeedInMbPerSec) { copyCancellation.Cancel(); var totalBytesCopied = position - startPosition; var result = new CopyFileResult( CopyResultCode.CopyBandwidthTimeoutError, $"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {totalBytesCopied} bytes copied (received {position - previousPosition} bytes in {_config.BandwidthCheckInterval.TotalSeconds} seconds)."); return(result, totalBytesCopied); } previousPosition = position; } var copyFileResult = await copyTask; copyFileResult.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec; return(copyFileResult, previousPosition - startPosition); void traceCopyTaskFailures() { // When the operation is cancelled, it is possible for the copy operation to fail. // In this case we still want to trace the failure (but just with the debug severity and not with the error), // but we should exclude ObjectDisposedException completely. // That's why we don't use task.FireAndForget but tracing inside the task's continuation. copyTask.ContinueWith(t => { if (t.IsFaulted) { if (!(t.Exception?.InnerException is ObjectDisposedException)) { context.TraceDebug($"Checked copy failed. {t.Exception}"); } } }); } } }
private async Task <CopyFileResult> CopyToAsyncCore(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { try { if (CopyDelay != null) { await Task.Delay(CopyDelay.Value); } long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!File.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist.")); } using Stream s = GetStream(sourcePath, expectedContentSize); await s.CopyToAsync(destinationStream); return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)); } catch (Exception e) { return(new CopyFileResult(CopyResultCode.DestinationPathError, e)); } }
public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { var result = CopyToAsyncCore(context, sourcePath, destinationStream, expectedContentSize, options); CopyToAsyncTask = result; return(result); }
/// <inheritdoc /> public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { // The bandwidth checker needs to have an options instance, because it is used for tracking the copy progress as well. options ??= new CopyToOptions(); return(_checker.CheckBandwidthAtIntervalAsync( context, // NOTE: We need to pass through the token from bandwidth checker to ensure copy cancellation for insufficient bandwidth gets triggered. token => _inner.CopyToAsync(context.WithCancellationToken(token), sourcePath, destinationStream, expectedContentSize, options), options)); }
public static void CopyTo(this Stream source, Stream destination, CopyToOptions options, CompressionMode mode, DecompressionMethods method = DecompressionMethods.Deflate, int bufferSize = 4096, long maxCount = -1, Action <CopyToEventArgs> onProgress = null) { if (mode == CompressionMode.Compress) { switch (method) { case DecompressionMethods.Deflate: { using (DeflateStream deflaterStream = new DeflateStream(destination, CompressionMode.Compress, true)) { source.CopyTo(deflaterStream, options, bufferSize, maxCount, onProgress); deflaterStream.Close(); } } break; case DecompressionMethods.GZip: { using (GZipStream gzipStream = new GZipStream(destination, CompressionMode.Compress, true)) { source.CopyTo(gzipStream, options, bufferSize, maxCount, onProgress); gzipStream.Close(); } } break; default: { source.CopyTo(destination, options, bufferSize, maxCount, onProgress); } break; } } else { switch (method) { case DecompressionMethods.Deflate: { using (DeflateStream deflaterStream = new DeflateStream(source, CompressionMode.Decompress, true)) { deflaterStream.CopyTo(destination, options, bufferSize, maxCount, onProgress); deflaterStream.Close(); } } break; case DecompressionMethods.GZip: { using (GZipStream gzipStream = new GZipStream(source, CompressionMode.Decompress, true)) { gzipStream.CopyTo(destination, options, bufferSize, maxCount, onProgress); gzipStream.Close(); } } break; default: { source.CopyTo(destination, options, bufferSize, maxCount, onProgress); } break; } } }
// public static void CopyTo(this Stream source, Stream destination, CopyToOptions options, int bufferSize = 4096, long maxCount = -1, Action <CopyToEventArgs> onProgress = null) { byte[] buffer = new byte[bufferSize]; // long totalBytesWritten = 0; // while (true) { int count = buffer.Length; // if (maxCount > 0) { if (totalBytesWritten > maxCount - count) { count = (int)(maxCount - totalBytesWritten); if (count <= 0) { break; } } } // int read = source.Read(buffer, 0, count); if (read <= 0) { break; } // destination.Write(buffer, 0, read); // if (options == CopyToOptions.Flush) { try { destination.Flush(); } // Analysis disable once EmptyGeneralCatchClause catch { // Do Nothing } } // totalBytesWritten += read; // if (onProgress != null) { onProgress(new CopyToEventArgs(totalBytesWritten)); } } // if (options == CopyToOptions.FlushFinal) { try { destination.Flush(); } // Analysis disable once EmptyGeneralCatchClause catch { // Do Nothing } } }
private static async Task <CopyFileResult> CopyRandomToStreamAtSpeed(CancellationToken token, Stream stream, long totalBytes, double mbPerSec, CopyToOptions options) { var interval = TimeSpan.FromSeconds(0.1); var copied = 0; var bytesPerInterval = (int)BytesPerInterval(mbPerSec, interval); Assert.True(bytesPerInterval > 0); var buffer = new byte[bytesPerInterval]; while (!token.IsCancellationRequested) { var intervalTask = Task.Delay(interval); Random.NextBytes(buffer); await stream.WriteAsync(buffer, 0, bytesPerInterval); copied += bytesPerInterval; options.UpdateTotalBytesCopied(copied); if (copied >= totalBytes) { break; } await intervalTask; } return(new CopyFileResult()); }
/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { // NOTE: Assumes source is local Contract.Assert(sourcePath.IsLocal); if (!FileUtilities.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist.")); } long startPosition = destinationStream.Position; using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan)) { return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); } }
/// <inheritdoc /> public Task <CopyFileResult> CopyToAsync(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CopyToOptions options) { CopyAttempts++; if (CustomResults != null) { return(Task.FromResult(CustomResults[CopyAttempts - 1])); } return(Task.FromResult(CopyToAsyncResult)); }