public async Task CopyFailsForWrongCopySize() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path); var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyFileAsyncResult = CopyFileResult.SuccessWithSize(41); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain("size"); result.ErrorMessage.Should().Contain("mismatch"); } }
public async Task CopyFailsForWrongCopySize() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var hash = VsoHashInfo.Instance.EmptyHash; var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(41); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain("size"); result.ErrorMessage.Should().Contain("mismatch"); } }
private async Task <CopyFileResult> CopyToAsyncCore(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, CopyOptions options) { try { if (CopyDelay != null) { await Task.Delay(CopyDelay.Value); } long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!_fileSystem.FileExists(sourcePath)) { return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist.")); } using Stream s = await GetStreamAsync(sourcePath); await s.CopyToAsync(destinationStream); return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)); } catch (Exception e) { return(new CopyFileResult(CopyResultCode.DestinationPathError, e)); } }
public async Task CopyFailsForWrongHash() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero); var hash = ContentHash.Random(); var wrongHash = VsoHashInfo.Instance.EmptyHash; var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 42, new MachineLocation[] { new MachineLocation("") }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(42); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(wrongHash, 42)) ); result.ShouldBeError(); result.ErrorMessage.Should().Contain(hash.ToShortString()); result.ErrorMessage.Should().Contain(wrongHash.ToShortString()); } }
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!File.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } Stream s; if (FilesToCorrupt.ContainsKey(sourcePath)) { TestGlobal.Logger.Debug($"Corrupting file {sourcePath}"); s = new MemoryStream(ThreadSafeRandom.GetBytes((int)expectedContentSize)); } else { s = File.OpenRead(sourcePath.Path); } return(await s.CopyToAsync(destinationStream).ContinueWith((_) => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); }
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { FilesCopied.AddOrUpdate(destinationPath, p => path, (dest, prevPath) => overwrite ? path : prevPath); if (!File.Exists(path.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {path} doesn't exist.")); } if (File.Exists(destinationPath.Path)) { if (!overwrite) { return(new CopyFileResult( CopyFileResult.ResultCode.DestinationPathError, $"Destination file {destinationPath} exists but overwrite not specified.")); } } if (FilesToCorrupt.ContainsKey(path)) { TestGlobal.Logger.Debug($"Corrupting file {path}"); #pragma warning disable AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes await Task.Run( () => File.WriteAllBytes(destinationPath.Path, ThreadSafeRandom.GetBytes(150))); #pragma warning restore AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes } else { await Task.Run(() => File.Copy(path.Path, destinationPath.Path), cancellationToken); } return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length)); }
public async Task CopyWithDestinationPathError(int retries) { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), retries); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyAttempts = 0; var totalCopyAttempts = (retries - 1) * machineLocations.Length + 1; mockFileCopier.CustomResults = new CopyFileResult[totalCopyAttempts]; mockFileCopier.CustomResults[0] = CopyFileResult.FromResultCode(CopyResultCode.DestinationPathError); for (int counter = 1; counter < totalCopyAttempts; counter++) { mockFileCopier.CustomResults[counter] = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError); } ; var destinationResult = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); destinationResult.ShouldBeError(); destinationResult.ErrorMessage.Should().Contain(hash.ToShortString()); mockFileCopier.CopyAttempts.Should().Be(totalCopyAttempts); } }
public async Task CopyRetries(int retries) { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero, retries); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var machineLocations = new MachineLocation[] { new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyToAsyncResult = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); result.ShouldBeError(); mockFileCopier.CopyAttempts.Should().Be(retries); } }
public async Task CopyWithDesignatedLocations() { var context = new Context(Logger); using (var directory = new DisposableDirectory(FileSystem)) { var machineLocations = new MachineLocation[] { new MachineLocation("Non-designated"), new MachineLocation("Designated") }; var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), designatedLocations: new MachineLocation[] { machineLocations[1] }); mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(99); var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); var destinationResult = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); destinationResult.ShouldBeSuccess(); mockFileCopier.CopyAttempts.Should().Be(1); distributedCopier.PathTransformer.LastContentLocation.Should().BeEquivalentTo(machineLocations[1].Data); } }
/// <inheritdoc /> public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { // NOTE: Assumes both source and destination are local Contract.Assert(path.IsLocal); Contract.Assert(destinationPath.IsLocal); if (!FileUtilities.Exists(path.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, $"Source file {path} doesn't exist.")); } if (FileUtilities.Exists(destinationPath.Path)) { if (!overwrite) { return(new CopyFileResult( CopyFileResult.ResultCode.DestinationPathError, $"Destination file {destinationPath} exists but overwrite not specified.")); } } if (!await FileUtilities.CopyFileAsync(path.Path, destinationPath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to copy {destinationPath} from {path}")); } return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length)); }
public async Task CopyRetriesWithRestrictions(int retries) { var context = new Context(Logger); var copyAttemptsWithRestrictedReplicas = 2; var restrictedCopyReplicaCount = 3; using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks( FileSystem, directory.Path, TimeSpan.Zero, retries, copyAttemptsWithRestrictedReplicas, restrictedCopyReplicaCount, maxRetryCount: retries + 1); await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context); var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyToAsyncResult = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); result.ShouldBeError(); int copyAttempts = 0; for (var attemptCount = 0; attemptCount < retries; attemptCount++) { var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas ? restrictedCopyReplicaCount : int.MaxValue; copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length); } if (copyAttempts < distributedCopier.Settings.MaxRetryCount) { mockFileCopier.CopyAttempts.Should().Be(copyAttempts); result.ErrorMessage.Should().NotContain("Maximum total retries"); } else { mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount); result.ErrorMessage.Should().Contain("Maximum total retries"); } } }
/// <inheritdoc /> public async Task <CopyFileResult> CopyFileAsync(AbsolutePath sourcePath, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath); CopyFileResult copyFileResult = null; // Contact hard-coded port on source using (var clientWrapper = await _clientCache.CreateAsync(host, _grpcPort, _useCompression)) { copyFileResult = await clientWrapper.Value.CopyFileAsync(_context, contentHash, destinationPath, cancellationToken); } return(copyFileResult); }
/// <inheritdoc /> public async Task <CopyFileResult> CopyFileAsync(AbsolutePath sourcePath, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath); CopyFileResult copyFileResult = null; // Contact hard-coded port on source using (var client = GrpcCopyClient.Create(host, DefaultGrpcPort)) { copyFileResult = await client.CopyFileAsync(_context, contentHash, destinationPath, cancellationToken); } return(copyFileResult); }
/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { // Extract host and contentHash from sourcePath (string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath); CopyFileResult copyFileResult = null; // Contact hard-coded port on source using (var client = GrpcCopyClient.Create(System.Net.IPAddress.Any.ToString(), DefaultGrpcPort)) { copyFileResult = await client.CopyToAsync(_context, contentHash, destinationStream, cancellationToken, expectedContentSize); } return(copyFileResult); }
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { long startPosition = destinationStream.Position; FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath); if (!File.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } using Stream s = GetStream(sourcePath, expectedContentSize); await s.CopyToAsync(destinationStream); return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)); }
/// <summary> /// Copies file to stream /// </summary> public async Task <CopyFileResult> CopyToAsync(Context context, ContentHash contentHash, Stream stream, CancellationToken cancellationToken, long fileSize = -1, bool enableCompression = false) { // TODO: Pipe through flag for compression type CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None; long bytesReceived = 0L; try { AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest { TraceId = context.Id.ToString(), HashType = (int)contentHash.HashType, ContentHash = contentHash.ToByteString(), // TODO: If `Drive` is expected to be the drive of the file on the source machine, then this should have nothing to do with the destination's drive Drive = "B", Offset = 0, Compression = compression }); bytesReceived = await StreamContentAsync(stream, response.ResponseStream); } catch (RpcException r) when(r.StatusCode == StatusCode.Unavailable && r.Message.Contains("Connect Failed")) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r, $"Failed to connect to server {_host} at port {_grpcPort}")); } if (bytesReceived == 0) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Received {bytesReceived} bytes for {contentHash}. Source file does not exist")); } else if (fileSize >= 0 && bytesReceived != fileSize) { return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}")); } return(CopyFileResult.SuccessWithSize(bytesReceived)); }
/// <inheritdoc /> public virtual async Task <CopyFileResult> CopyToAsync(OperationContext context, ContentLocation sourceLocation, Stream destinationStream, CopyOptions options) { var sourcePath = new AbsolutePath(sourceLocation.Machine.Path) / FileSystemContentStoreInternal.GetPrimaryRelativePath(sourceLocation.Hash, includeSharedFolder: false); if (!FileUtilities.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist.")); } long startPosition = destinationStream.Position; using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan)) { return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); } }
/// <inheritdoc /> public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken) { // NOTE: Assumes source is local Contract.Assert(sourcePath.IsLocal); if (!FileUtilities.Exists(sourcePath.Path)) { return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist.")); } long startPosition = destinationStream.Position; using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan)) { return(await s.CopyToAsync(destinationStream, 81920, cancellationToken).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition))); } }
public static void MakeInstalledPackageFileForDefPackage(string XmlPath, string _namespace, out LogDataList log, string rootDirectory = null, bool ignoreClash = false) { log = new LogDataList(); log.WriteInformationLog(MakeInstalledPackagesMessages.Info.CreateInstalledPack(XmlPath, rootDirectory), _namespace); LogDataList outLog = null; RMPackage newPack = null; Exception exResult; try { newPack = new RMPackage(XmlPath, _namespace, out outLog, rootDirectory); } catch (Exception ex) { log.AppendLogs(outLog); Logger.WriteErrorLog(MakeInstalledPackagesMessages.Error.InvalidXML(XmlPath, rootDirectory), _namespace, ex, BasicDebugLogger.DebugErrorType.Error); throw; } log.AppendLogs(outLog); string origDir = newPack.XMLDirectory; InstalledPackage installedPackage = GlobalPackages.FindByUID(newPack.UniqueID); string newPackDir = PMFileSystem.PackMan_ManDir + "\\" + newPack.UniqueIDInMD5; if (installedPackage != null) { if (ignoreClash) { Logger.WriteWarningLog(MakeInstalledPackagesMessages.Warning.InstalledPackExistsExit(newPack.Name), _namespace, null); return; } else { log.WriteWarningLog(MakeInstalledPackagesMessages.Warning.InstalledPackExistsReplace(newPack.Name), _namespace); if (Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, new DeleteFolderLogMessages(deleteFailed: MakeInstalledPackagesMessages.Error.UnableToDeleteExistingFolder)) == DeleteFolderResult.UserCancelled) { throw exResult; } } } string tempPackDir = PMFileSystem.PackMan_TempDir + "\\" + newPack.UniqueIDInMD5; if (Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG) == DeleteFolderResult.UserCancelled) { throw exResult; } if (Helper.CreateFolderSafely(newPackDir, _namespace, out exResult, new CreateFolderLogMessages(createFailed: MakeInstalledPackagesMessages.Error.UnableCreateRootDir)) == CreateFolderResult.UserCancelled) { throw exResult; } if (Helper.CreateFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_CREATE_TEMP_DIR_ARG) == CreateFolderResult.UserCancelled) { throw exResult; } string backupXMLInstallerPath = tempPackDir + "\\" + Vars.INSTALL_FILE_DEFAULT_FILENAME; makeInstPackageAnon onErrorBackup = delegate() { if (Helper.ShowMessageBox(MessageBoxStrings.PackageManagement.FailedMakeInstallPackageArch(newPack.Name), MessageBoxStrings.MESSAGEBOX_NAME, MessageBoxButtons.YesNo, MessageBoxIcon.Warning) == DialogResult.Yes) { return(true); } Logger.WriteErrorLog(MakeInstalledPackagesMessages.Error.UserRequiredOnFailedBackUpNo(newPack.Name), _namespace, null, BasicDebugLogger.DebugErrorType.Error); Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG); Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_UNUSED_DIR_ARG); return(false); }; string rootPathOfFiles; if (rootDirectory == null) { rootPathOfFiles = newPack.XMLDirectory; } else { rootPathOfFiles = rootDirectory; } PackageUtil.ExplicitCopyAssetsAndLicFileTo(newPack, rootPathOfFiles, tempPackDir, _namespace); try { newPack.SaveToFile(backupXMLInstallerPath, _namespace, RMPackage.SaveToFileMode.ImplicitAssetInfo, logMessage: new WriteAllTextLogMessages(writeFailed: MakeInstalledPackagesMessages.Error.UnableCreateInstXML, logGroup: log)); } catch { if (onErrorBackup()) { goto onerrorContinue; } throw; } string archDest = newPackDir + "\\" + Vars.INSTALLED_ARCH_FILENAME; try { ArchiveManagement.CreateNewZip(tempPackDir, archDest, _namespace); } catch (Exception ex) { log.WriteWarningLog(MakeInstalledPackagesMessages.Error.UnableMakeBackup(newPack.Name, tempPackDir, archDest), _namespace, ex); if (onErrorBackup()) { goto onerrorContinue; } throw; } onerrorContinue: //newPack.Installed = true; //newPack.SetInstalledPropertyAll(RMPackObject.InstallStatus.Installed); if (newPack.License != null && newPack.License.LicenseSource == RMPackLic.Source.File && RMPackLic.IsAValidLicenseSourceFile(newPack.License.Data, origDir)) { string licSourcePath = origDir + "\\" + newPack.License.Data; string fileName = Path.GetFileName(licSourcePath); string destPath = newPackDir + "\\" + fileName; CopyFileResult copyRes = Helper.CopyFileSafely(licSourcePath, destPath, true, _namespace, out exResult, new CopyFileLogMessages(copyFileFailed: MakeInstalledPackagesMessages.Error.UnableCopyLicenseFile)); if (copyRes == CopyFileResult.UserCancelled || copyRes == CopyFileResult.SourceFileNotFound) { Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG); Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_UNUSED_DIR_ARG); throw exResult; } newPack.License.Data = fileName; } string installedXMLDest = newPackDir + "\\" + Vars.INSTALLED_XML_FILENAME; newPack.Implicit = false; try { newPack.SaveToFile(installedXMLDest, _namespace, RMPackage.SaveToFileMode.ExplicitAssetInfo, logMessage: new WriteAllTextLogMessages(writeFailed: MakeInstalledPackagesMessages.Error.UnableMakeMainXML)); } catch { Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG); Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_UNUSED_DIR_ARG); throw; } try { GlobalPackages.AddSafely(new InstalledPackage(newPackDir, _namespace, out outLog)); } catch (Exception ex) { log.WriteWarningLog(LoggerMessages.PackageManagement.InstalledPackage.Error.ErrorTryLoad(newPackDir), _namespace, ex); if (ex is InvalidInstalledPackageFile castedEx) { log.WriteErrorLog(castedEx.Message, _namespace, castedEx.InnerException); } else { log.WriteErrorLog(ex.Message, _namespace, ex); } } log.AppendLogs(outLog); try { NewProject.CopyPackageInstallInfo(tempPackDir, _namespace, out outLog); } catch (Exception ex) { log.WriteWarningLog(MakeInstalledPackagesMessages.Warning.UnableCopyNewProj(newPack.Name, newPackDir), _namespace, ex); } log.AppendLogs(outLog); Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG); log.WriteInformationLog(MakeInstalledPackagesMessages.Info.SUCCESS_CREATE + newPack.Name + ".", _namespace); }
private void InvalidateResourceIfNeeded(Context context, CopyOptions options, CopyFileResult result, IResourceWrapperAdapter <GrpcCopyClient> clientWrapper) { if (!result) { switch (_configuration.GrpcCopyClientInvalidationPolicy) { case GrpcFileCopierConfiguration.ClientInvalidationPolicy.Disabled: break; case GrpcFileCopierConfiguration.ClientInvalidationPolicy.OnEveryError: clientWrapper.Invalidate(context); break; case GrpcFileCopierConfiguration.ClientInvalidationPolicy.OnConnectivityErrors: if ((result.Code == CopyResultCode.CopyBandwidthTimeoutError && options.TotalBytesCopied == 0) || result.Code == CopyResultCode.ConnectionTimeoutError) { if (options?.BandwidthConfiguration?.InvalidateOnTimeoutError ?? true) { clientWrapper.Invalidate(context); } } break; } } }
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, CopyOptions options, Func <Stream> streamFactory, bool closeStream) { using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token); var token = cts.Token; bool exceptionThrown = false; TimeSpan? headerResponseTime = null; CopyFileResult?result = null; try { CopyFileRequest request = new CopyFileRequest() { TraceId = context.TracingContext.Id.ToString(), HashType = (int)hash.HashType, ContentHash = hash.ToByteString(), Offset = 0, Compression = _configuration.UseGzipCompression ? CopyCompression.Gzip : CopyCompression.None, FailFastIfBusy = options.BandwidthConfiguration?.FailFastIfServerIsBusy ?? false, }; using AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request, options: GetDefaultGrpcOptions(token)); Metadata headers; var stopwatch = StopwatchSlim.Start(); try { var connectionTimeout = GetResponseHeadersTimeout(options); headers = await response.ResponseHeadersAsync.WithTimeoutAsync(connectionTimeout, token); headerResponseTime = stopwatch.Elapsed; } catch (TimeoutException t) { // Trying to cancel the back end operation as well. cts.Cancel(); result = new CopyFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t); return(result); } // If the remote machine couldn't be contacted, GRPC returns an empty // header collection. GRPC would throw an RpcException when we tried // to stream response, but by that time we would have created target // stream. To avoid that, exit early instead. if (headers.Count == 0) { result = new CopyFileResult( CopyResultCode.ServerUnavailable, $"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}."); return(result); } // Parse header collection. string? exception = null; string? message = null; CopyCompression compression = CopyCompression.None; foreach (Metadata.Entry header in headers) { switch (header.Key) { case "exception": exception = header.Value; break; case "message": message = header.Value; break; case "compression": Enum.TryParse(header.Value, out compression); break; } } // Process reported server-side errors. if (exception != null) { Contract.Assert(message != null); switch (exception) { case "ContentNotFound": result = new CopyFileResult(CopyResultCode.FileNotFoundError, message); return(result); default: result = new CopyFileResult(CopyResultCode.UnknownServerError, message); return(result); } } // We got headers back with no errors, so create the target stream. Stream targetStream; try { targetStream = streamFactory(); } catch (Exception targetException) { result = new CopyFileResult(CopyResultCode.DestinationPathError, targetException); return(result); } result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync( context, innerToken => copyToCoreImplementation(response, compression, targetStream, innerToken), options, getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics)); return(result); } catch (RpcException r) { result = CreateResultFromException(r); return(result); } catch (Exception) { exceptionThrown = true; throw; } finally { // Even though we don't expect exceptions in this method, we can't assume they won't happen. // So asserting that the result is not null only when the method completes successfully or with a known errors. Contract.Assert(exceptionThrown || result != null); if (result != null) { result.HeaderResponseTime = headerResponseTime; } } async Task <CopyFileResult> copyToCoreImplementation(AsyncServerStreamingCall <CopyFileResponse> response, CopyCompression compression, Stream targetStream, CancellationToken token) { // Copy the content to the target stream. try { switch (compression) { case CopyCompression.None: await StreamContentAsync(response.ResponseStream, targetStream, options, token); break; case CopyCompression.Gzip: await StreamContentWithCompressionAsync(response.ResponseStream, targetStream, options, token); break; default: throw new NotSupportedException($"CopyCompression {compression} is not supported."); } } finally { if (closeStream) { #pragma warning disable AsyncFixer02 // A disposable object used in a fire & forget async call targetStream.Dispose(); #pragma warning restore AsyncFixer02 // A disposable object used in a fire & forget async call } } return(CopyFileResult.Success); } }
public async Task <CopyFileResult> CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task <CopyFileResult> > copyTaskFactory, CopyToOptions options) { if (_historicalBandwidthLimitSource != null) { var timer = Stopwatch.StartNew(); var(result, bytesCopied) = await impl(); timer.Stop(); // Bandwidth checker expects speed in MiB/s, so convert it. var speed = bytesCopied / timer.Elapsed.TotalSeconds / BytesInMb; _historicalBandwidthLimitSource.AddBandwidthRecord(speed); return(result); } else { return((await impl()).result); } async Task <(CopyFileResult result, long bytesCopied)> impl() { // This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash // (given that the app is configured to fail on unobserved task exceptions). var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec() * _config.BandwidthLimitMultiplier; minimumSpeedInMbPerSec = Math.Min(minimumSpeedInMbPerSec, _config.MaxBandwidthLimit); long startPosition = options.TotalBytesCopied; long previousPosition = startPosition; var copyCompleted = false; using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token); Task <CopyFileResult> copyTask = copyTaskFactory(copyCancellation.Token); // Subscribing for potential task failure here to avoid unobserved task exceptions. traceCopyTaskFailures(); while (!copyCompleted) { // Wait some time for bytes to be copied var firstCompletedTask = await Task.WhenAny(copyTask, Task.Delay(_config.BandwidthCheckInterval, context.Token)); copyCompleted = firstCompletedTask == copyTask; if (copyCompleted) { var result = await copyTask; result.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec; var bytesCopied = result.Size ?? options.TotalBytesCopied; return(result, bytesCopied); } else if (context.Token.IsCancellationRequested) { context.Token.ThrowIfCancellationRequested(); } // Copy is not completed and operation has not been canceled, perform // bandwidth check var position = options.TotalBytesCopied; var receivedMiB = (position - previousPosition) / BytesInMb; var currentSpeed = receivedMiB / _config.BandwidthCheckInterval.TotalSeconds; if (currentSpeed == 0 || currentSpeed < minimumSpeedInMbPerSec) { copyCancellation.Cancel(); var totalBytesCopied = position - startPosition; var result = new CopyFileResult( CopyResultCode.CopyBandwidthTimeoutError, $"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {totalBytesCopied} bytes copied (received {position - previousPosition} bytes in {_config.BandwidthCheckInterval.TotalSeconds} seconds)."); return(result, totalBytesCopied); } previousPosition = position; } var copyFileResult = await copyTask; copyFileResult.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec; return(copyFileResult, previousPosition - startPosition); void traceCopyTaskFailures() { // When the operation is cancelled, it is possible for the copy operation to fail. // In this case we still want to trace the failure (but just with the debug severity and not with the error), // but we should exclude ObjectDisposedException completely. // That's why we don't use task.FireAndForget but tracing inside the task's continuation. copyTask.ContinueWith(t => { if (t.IsFaulted) { if (!(t.Exception?.InnerException is ObjectDisposedException)) { context.TraceDebug($"Checked copy failed. {t.Exception}"); } } }); } } }