public async Task CopyFailsForWrongCopySize()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path);
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyFileAsyncResult = CopyFileResult.SuccessWithSize(41);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain("size");
result.ErrorMessage.Should().Contain("mismatch");
}
}
public async Task CopyFailsForWrongCopySize()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var hash = VsoHashInfo.Instance.EmptyHash;
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(41);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain("size");
result.ErrorMessage.Should().Contain("mismatch");
}
}
private async Task <CopyFileResult> CopyToAsyncCore(OperationContext context, AbsolutePath sourcePath, Stream destinationStream, CopyOptions options)
{
try
{
if (CopyDelay != null)
{
await Task.Delay(CopyDelay.Value);
}
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!_fileSystem.FileExists(sourcePath))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
using Stream s = await GetStreamAsync(sourcePath);
await s.CopyToAsync(destinationStream);
return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition));
}
catch (Exception e)
{
return(new CopyFileResult(CopyResultCode.DestinationPathError, e));
}
}
public async Task CopyFailsForWrongHash()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = await CreateAsync(context, directory.Path, TimeSpan.Zero);
var hash = ContentHash.Random();
var wrongHash = VsoHashInfo.Instance.EmptyHash;
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 42,
new MachineLocation[] { new MachineLocation("") });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(42);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(wrongHash, 42))
);
result.ShouldBeError();
result.ErrorMessage.Should().Contain(hash.ToShortString());
result.ErrorMessage.Should().Contain(wrongHash.ToShortString());
}
}
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!File.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
Stream s;
if (FilesToCorrupt.ContainsKey(sourcePath))
{
TestGlobal.Logger.Debug($"Corrupting file {sourcePath}");
s = new MemoryStream(ThreadSafeRandom.GetBytes((int)expectedContentSize));
}
else
{
s = File.OpenRead(sourcePath.Path);
}
return(await s.CopyToAsync(destinationStream).ContinueWith((_) => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
FilesCopied.AddOrUpdate(destinationPath, p => path, (dest, prevPath) => overwrite ? path : prevPath);
if (!File.Exists(path.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {path} doesn't exist."));
}
if (File.Exists(destinationPath.Path))
{
if (!overwrite)
{
return(new CopyFileResult(
CopyFileResult.ResultCode.DestinationPathError,
$"Destination file {destinationPath} exists but overwrite not specified."));
}
}
if (FilesToCorrupt.ContainsKey(path))
{
TestGlobal.Logger.Debug($"Corrupting file {path}");
#pragma warning disable AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes
await Task.Run(
() => File.WriteAllBytes(destinationPath.Path, ThreadSafeRandom.GetBytes(150)));
#pragma warning restore AsyncFixer02 // WriteAllBytesAsync should be used instead of File.WriteAllBytes
}
else
{
await Task.Run(() => File.Copy(path.Path, destinationPath.Path), cancellationToken);
}
return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length));
}
public async Task CopyWithDestinationPathError(int retries)
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), retries);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyAttempts = 0;
var totalCopyAttempts = (retries - 1) * machineLocations.Length + 1;
mockFileCopier.CustomResults = new CopyFileResult[totalCopyAttempts];
mockFileCopier.CustomResults[0] = CopyFileResult.FromResultCode(CopyResultCode.DestinationPathError);
for (int counter = 1; counter < totalCopyAttempts; counter++)
{
mockFileCopier.CustomResults[counter] = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError);
}
;
var destinationResult = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
destinationResult.ShouldBeError();
destinationResult.ErrorMessage.Should().Contain(hash.ToShortString());
mockFileCopier.CopyAttempts.Should().Be(totalCopyAttempts);
}
}
public async Task CopyRetries(int retries)
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero, retries);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
mockFileCopier.CopyAttempts.Should().Be(retries);
}
}
public async Task CopyWithDesignatedLocations()
{
var context = new Context(Logger);
using (var directory = new DisposableDirectory(FileSystem))
{
var machineLocations = new MachineLocation[] { new MachineLocation("Non-designated"), new MachineLocation("Designated") };
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.FromMilliseconds((10)), designatedLocations: new MachineLocation[] { machineLocations[1] });
mockFileCopier.CopyToAsyncResult = CopyFileResult.SuccessWithSize(99);
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
var destinationResult = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
destinationResult.ShouldBeSuccess();
mockFileCopier.CopyAttempts.Should().Be(1);
distributedCopier.PathTransformer.LastContentLocation.Should().BeEquivalentTo(machineLocations[1].Data);
}
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath path, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
// NOTE: Assumes both source and destination are local
Contract.Assert(path.IsLocal);
Contract.Assert(destinationPath.IsLocal);
if (!FileUtilities.Exists(path.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.FileNotFoundError, $"Source file {path} doesn't exist."));
}
if (FileUtilities.Exists(destinationPath.Path))
{
if (!overwrite)
{
return(new CopyFileResult(
CopyFileResult.ResultCode.DestinationPathError,
$"Destination file {destinationPath} exists but overwrite not specified."));
}
}
if (!await FileUtilities.CopyFileAsync(path.Path, destinationPath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Failed to copy {destinationPath} from {path}"));
}
return(CopyFileResult.SuccessWithSize(new System.IO.FileInfo(destinationPath.Path).Length));
}
public async Task CopyRetriesWithRestrictions(int retries)
{
var context = new Context(Logger);
var copyAttemptsWithRestrictedReplicas = 2;
var restrictedCopyReplicaCount = 3;
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(
FileSystem,
directory.Path,
TimeSpan.Zero,
retries,
copyAttemptsWithRestrictedReplicas,
restrictedCopyReplicaCount,
maxRetryCount: retries + 1);
await using var _ = await distributedCopier.StartupWithAutoShutdownAsync(context);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = CopyFileResult.FromResultCode(CopyResultCode.UnknownServerError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
int copyAttempts = 0;
for (var attemptCount = 0; attemptCount < retries; attemptCount++)
{
var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas
? restrictedCopyReplicaCount
: int.MaxValue;
copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length);
}
if (copyAttempts < distributedCopier.Settings.MaxRetryCount)
{
mockFileCopier.CopyAttempts.Should().Be(copyAttempts);
result.ErrorMessage.Should().NotContain("Maximum total retries");
}
else
{
mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount);
result.ErrorMessage.Should().Contain("Maximum total retries");
}
}
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath sourcePath, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
// Extract host and contentHash from sourcePath
(string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath);
CopyFileResult copyFileResult = null;
// Contact hard-coded port on source
using (var clientWrapper = await _clientCache.CreateAsync(host, _grpcPort, _useCompression))
{
copyFileResult = await clientWrapper.Value.CopyFileAsync(_context, contentHash, destinationPath, cancellationToken);
}
return(copyFileResult);
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyFileAsync(AbsolutePath sourcePath, AbsolutePath destinationPath, long contentSize, bool overwrite, CancellationToken cancellationToken)
{
// Extract host and contentHash from sourcePath
(string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath);
CopyFileResult copyFileResult = null;
// Contact hard-coded port on source
using (var client = GrpcCopyClient.Create(host, DefaultGrpcPort))
{
copyFileResult = await client.CopyFileAsync(_context, contentHash, destinationPath, cancellationToken);
}
return(copyFileResult);
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
// Extract host and contentHash from sourcePath
(string host, ContentHash contentHash) = ExtractHostHashFromAbsolutePath(sourcePath);
CopyFileResult copyFileResult = null;
// Contact hard-coded port on source
using (var client = GrpcCopyClient.Create(System.Net.IPAddress.Any.ToString(), DefaultGrpcPort))
{
copyFileResult = await client.CopyToAsync(_context, contentHash, destinationStream, cancellationToken, expectedContentSize);
}
return(copyFileResult);
}
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
long startPosition = destinationStream.Position;
FilesCopied.AddOrUpdate(sourcePath, p => sourcePath, (dest, prevPath) => prevPath);
if (!File.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
using Stream s = GetStream(sourcePath, expectedContentSize);
await s.CopyToAsync(destinationStream);
return(CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition));
}
/// <summary>
/// Copies file to stream
/// </summary>
public async Task <CopyFileResult> CopyToAsync(Context context, ContentHash contentHash, Stream stream, CancellationToken cancellationToken, long fileSize = -1, bool enableCompression = false)
{
// TODO: Pipe through flag for compression type
CopyCompression compression = enableCompression ? CopyCompression.Gzip : CopyCompression.None;
long bytesReceived = 0L;
try
{
AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(new CopyFileRequest
{
TraceId = context.Id.ToString(),
HashType = (int)contentHash.HashType,
ContentHash = contentHash.ToByteString(),
// TODO: If `Drive` is expected to be the drive of the file on the source machine, then this should have nothing to do with the destination's drive
Drive = "B",
Offset = 0,
Compression = compression
});
bytesReceived = await StreamContentAsync(stream, response.ResponseStream);
}
catch (RpcException r) when(r.StatusCode == StatusCode.Unavailable && r.Message.Contains("Connect Failed"))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, r, $"Failed to connect to server {_host} at port {_grpcPort}"));
}
if (bytesReceived == 0)
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Received {bytesReceived} bytes for {contentHash}. Source file does not exist"));
}
else if (fileSize >= 0 && bytesReceived != fileSize)
{
return(new CopyFileResult(CopyFileResult.ResultCode.InvalidHash, $"Received {bytesReceived} bytes for {contentHash}, expected {fileSize}"));
}
return(CopyFileResult.SuccessWithSize(bytesReceived));
}
/// <inheritdoc />
public virtual async Task <CopyFileResult> CopyToAsync(OperationContext context, ContentLocation sourceLocation, Stream destinationStream, CopyOptions options)
{
var sourcePath = new AbsolutePath(sourceLocation.Machine.Path) / FileSystemContentStoreInternal.GetPrimaryRelativePath(sourceLocation.Hash, includeSharedFolder: false);
if (!FileUtilities.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
long startPosition = destinationStream.Position;
using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
}
/// <inheritdoc />
public async Task <CopyFileResult> CopyToAsync(AbsolutePath sourcePath, Stream destinationStream, long expectedContentSize, CancellationToken cancellationToken)
{
// NOTE: Assumes source is local
Contract.Assert(sourcePath.IsLocal);
if (!FileUtilities.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyFileResult.ResultCode.SourcePathError, $"Source file {sourcePath} doesn't exist."));
}
long startPosition = destinationStream.Position;
using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
return(await s.CopyToAsync(destinationStream, 81920, cancellationToken).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
}
public static void MakeInstalledPackageFileForDefPackage(string XmlPath, string _namespace, out LogDataList log, string rootDirectory = null, bool ignoreClash = false)
{
log = new LogDataList();
log.WriteInformationLog(MakeInstalledPackagesMessages.Info.CreateInstalledPack(XmlPath, rootDirectory), _namespace);
LogDataList outLog = null;
RMPackage newPack = null;
Exception exResult;
try
{
newPack = new RMPackage(XmlPath, _namespace, out outLog, rootDirectory);
}
catch (Exception ex)
{
log.AppendLogs(outLog);
Logger.WriteErrorLog(MakeInstalledPackagesMessages.Error.InvalidXML(XmlPath, rootDirectory), _namespace, ex, BasicDebugLogger.DebugErrorType.Error);
throw;
}
log.AppendLogs(outLog);
string origDir = newPack.XMLDirectory;
InstalledPackage installedPackage = GlobalPackages.FindByUID(newPack.UniqueID);
string newPackDir = PMFileSystem.PackMan_ManDir + "\\" + newPack.UniqueIDInMD5;
if (installedPackage != null)
{
if (ignoreClash)
{
Logger.WriteWarningLog(MakeInstalledPackagesMessages.Warning.InstalledPackExistsExit(newPack.Name), _namespace, null);
return;
}
else
{
log.WriteWarningLog(MakeInstalledPackagesMessages.Warning.InstalledPackExistsReplace(newPack.Name), _namespace);
if (Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, new DeleteFolderLogMessages(deleteFailed: MakeInstalledPackagesMessages.Error.UnableToDeleteExistingFolder)) == DeleteFolderResult.UserCancelled)
{
throw exResult;
}
}
}
string tempPackDir = PMFileSystem.PackMan_TempDir + "\\" + newPack.UniqueIDInMD5;
if (Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG) == DeleteFolderResult.UserCancelled)
{
throw exResult;
}
if (Helper.CreateFolderSafely(newPackDir, _namespace, out exResult, new CreateFolderLogMessages(createFailed: MakeInstalledPackagesMessages.Error.UnableCreateRootDir)) == CreateFolderResult.UserCancelled)
{
throw exResult;
}
if (Helper.CreateFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_CREATE_TEMP_DIR_ARG) == CreateFolderResult.UserCancelled)
{
throw exResult;
}
string backupXMLInstallerPath = tempPackDir + "\\" + Vars.INSTALL_FILE_DEFAULT_FILENAME;
makeInstPackageAnon onErrorBackup = delegate()
{
if (Helper.ShowMessageBox(MessageBoxStrings.PackageManagement.FailedMakeInstallPackageArch(newPack.Name), MessageBoxStrings.MESSAGEBOX_NAME, MessageBoxButtons.YesNo, MessageBoxIcon.Warning) == DialogResult.Yes)
{
return(true);
}
Logger.WriteErrorLog(MakeInstalledPackagesMessages.Error.UserRequiredOnFailedBackUpNo(newPack.Name), _namespace, null, BasicDebugLogger.DebugErrorType.Error);
Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG);
Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_UNUSED_DIR_ARG);
return(false);
};
string rootPathOfFiles;
if (rootDirectory == null)
{
rootPathOfFiles = newPack.XMLDirectory;
}
else
{
rootPathOfFiles = rootDirectory;
}
PackageUtil.ExplicitCopyAssetsAndLicFileTo(newPack, rootPathOfFiles, tempPackDir, _namespace);
try
{
newPack.SaveToFile(backupXMLInstallerPath, _namespace, RMPackage.SaveToFileMode.ImplicitAssetInfo,
logMessage: new WriteAllTextLogMessages(writeFailed: MakeInstalledPackagesMessages.Error.UnableCreateInstXML, logGroup: log));
}
catch
{
if (onErrorBackup())
{
goto onerrorContinue;
}
throw;
}
string archDest = newPackDir + "\\" + Vars.INSTALLED_ARCH_FILENAME;
try
{
ArchiveManagement.CreateNewZip(tempPackDir, archDest, _namespace);
}
catch (Exception ex)
{
log.WriteWarningLog(MakeInstalledPackagesMessages.Error.UnableMakeBackup(newPack.Name, tempPackDir, archDest), _namespace, ex);
if (onErrorBackup())
{
goto onerrorContinue;
}
throw;
}
onerrorContinue:
//newPack.Installed = true;
//newPack.SetInstalledPropertyAll(RMPackObject.InstallStatus.Installed);
if (newPack.License != null && newPack.License.LicenseSource == RMPackLic.Source.File && RMPackLic.IsAValidLicenseSourceFile(newPack.License.Data, origDir))
{
string licSourcePath = origDir + "\\" + newPack.License.Data;
string fileName = Path.GetFileName(licSourcePath);
string destPath = newPackDir + "\\" + fileName;
CopyFileResult copyRes = Helper.CopyFileSafely(licSourcePath, destPath, true, _namespace, out exResult, new CopyFileLogMessages(copyFileFailed: MakeInstalledPackagesMessages.Error.UnableCopyLicenseFile));
if (copyRes == CopyFileResult.UserCancelled || copyRes == CopyFileResult.SourceFileNotFound)
{
Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG);
Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_UNUSED_DIR_ARG);
throw exResult;
}
newPack.License.Data = fileName;
}
string installedXMLDest = newPackDir + "\\" + Vars.INSTALLED_XML_FILENAME;
newPack.Implicit = false;
try
{
newPack.SaveToFile(installedXMLDest, _namespace, RMPackage.SaveToFileMode.ExplicitAssetInfo,
logMessage: new WriteAllTextLogMessages(writeFailed: MakeInstalledPackagesMessages.Error.UnableMakeMainXML));
}
catch
{
Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG);
Helper.DeleteFolderSafely(newPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_UNUSED_DIR_ARG);
throw;
}
try
{
GlobalPackages.AddSafely(new InstalledPackage(newPackDir, _namespace, out outLog));
}
catch (Exception ex)
{
log.WriteWarningLog(LoggerMessages.PackageManagement.InstalledPackage.Error.ErrorTryLoad(newPackDir), _namespace, ex);
if (ex is InvalidInstalledPackageFile castedEx)
{
log.WriteErrorLog(castedEx.Message, _namespace, castedEx.InnerException);
}
else
{
log.WriteErrorLog(ex.Message, _namespace, ex);
}
}
log.AppendLogs(outLog);
try
{
NewProject.CopyPackageInstallInfo(tempPackDir, _namespace, out outLog);
}
catch (Exception ex)
{
log.WriteWarningLog(MakeInstalledPackagesMessages.Warning.UnableCopyNewProj(newPack.Name, newPackDir), _namespace, ex);
}
log.AppendLogs(outLog);
Helper.DeleteFolderSafely(tempPackDir, _namespace, out exResult, LoggerMessages.GeneralError.UNABLE_DELETE_TEMP_DIR_ARG);
log.WriteInformationLog(MakeInstalledPackagesMessages.Info.SUCCESS_CREATE + newPack.Name + ".", _namespace);
}
private void InvalidateResourceIfNeeded(Context context, CopyOptions options, CopyFileResult result, IResourceWrapperAdapter <GrpcCopyClient> clientWrapper)
{
if (!result)
{
switch (_configuration.GrpcCopyClientInvalidationPolicy)
{
case GrpcFileCopierConfiguration.ClientInvalidationPolicy.Disabled:
break;
case GrpcFileCopierConfiguration.ClientInvalidationPolicy.OnEveryError:
clientWrapper.Invalidate(context);
break;
case GrpcFileCopierConfiguration.ClientInvalidationPolicy.OnConnectivityErrors:
if ((result.Code == CopyResultCode.CopyBandwidthTimeoutError && options.TotalBytesCopied == 0) ||
result.Code == CopyResultCode.ConnectionTimeoutError)
{
if (options?.BandwidthConfiguration?.InvalidateOnTimeoutError ?? true)
{
clientWrapper.Invalidate(context);
}
}
break;
}
}
}
private async Task <CopyFileResult> CopyToCoreAsync(OperationContext context, ContentHash hash, CopyOptions options, Func <Stream> streamFactory, bool closeStream)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
var token = cts.Token;
bool exceptionThrown = false;
TimeSpan? headerResponseTime = null;
CopyFileResult?result = null;
try
{
CopyFileRequest request = new CopyFileRequest()
{
TraceId = context.TracingContext.Id.ToString(),
HashType = (int)hash.HashType,
ContentHash = hash.ToByteString(),
Offset = 0,
Compression = _configuration.UseGzipCompression ? CopyCompression.Gzip : CopyCompression.None,
FailFastIfBusy = options.BandwidthConfiguration?.FailFastIfServerIsBusy ?? false,
};
using AsyncServerStreamingCall <CopyFileResponse> response = _client.CopyFile(request, options: GetDefaultGrpcOptions(token));
Metadata headers;
var stopwatch = StopwatchSlim.Start();
try
{
var connectionTimeout = GetResponseHeadersTimeout(options);
headers = await response.ResponseHeadersAsync.WithTimeoutAsync(connectionTimeout, token);
headerResponseTime = stopwatch.Elapsed;
}
catch (TimeoutException t)
{
// Trying to cancel the back end operation as well.
cts.Cancel();
result = new CopyFileResult(GetCopyResultCodeForGetResponseHeaderTimeout(), t);
return(result);
}
// If the remote machine couldn't be contacted, GRPC returns an empty
// header collection. GRPC would throw an RpcException when we tried
// to stream response, but by that time we would have created target
// stream. To avoid that, exit early instead.
if (headers.Count == 0)
{
result = new CopyFileResult(
CopyResultCode.ServerUnavailable,
$"Failed to connect to copy server {Key.Host} at port {Key.GrpcPort}.");
return(result);
}
// Parse header collection.
string? exception = null;
string? message = null;
CopyCompression compression = CopyCompression.None;
foreach (Metadata.Entry header in headers)
{
switch (header.Key)
{
case "exception":
exception = header.Value;
break;
case "message":
message = header.Value;
break;
case "compression":
Enum.TryParse(header.Value, out compression);
break;
}
}
// Process reported server-side errors.
if (exception != null)
{
Contract.Assert(message != null);
switch (exception)
{
case "ContentNotFound":
result = new CopyFileResult(CopyResultCode.FileNotFoundError, message);
return(result);
default:
result = new CopyFileResult(CopyResultCode.UnknownServerError, message);
return(result);
}
}
// We got headers back with no errors, so create the target stream.
Stream targetStream;
try
{
targetStream = streamFactory();
}
catch (Exception targetException)
{
result = new CopyFileResult(CopyResultCode.DestinationPathError, targetException);
return(result);
}
result = await _bandwidthChecker.CheckBandwidthAtIntervalAsync(
context,
innerToken => copyToCoreImplementation(response, compression, targetStream, innerToken),
options,
getErrorResult : diagnostics => new CopyFileResult(CopyResultCode.CopyBandwidthTimeoutError, diagnostics));
return(result);
}
catch (RpcException r)
{
result = CreateResultFromException(r);
return(result);
}
catch (Exception)
{
exceptionThrown = true;
throw;
}
finally
{
// Even though we don't expect exceptions in this method, we can't assume they won't happen.
// So asserting that the result is not null only when the method completes successfully or with a known errors.
Contract.Assert(exceptionThrown || result != null);
if (result != null)
{
result.HeaderResponseTime = headerResponseTime;
}
}
async Task <CopyFileResult> copyToCoreImplementation(AsyncServerStreamingCall <CopyFileResponse> response, CopyCompression compression, Stream targetStream, CancellationToken token)
{
// Copy the content to the target stream.
try
{
switch (compression)
{
case CopyCompression.None:
await StreamContentAsync(response.ResponseStream, targetStream, options, token);
break;
case CopyCompression.Gzip:
await StreamContentWithCompressionAsync(response.ResponseStream, targetStream, options, token);
break;
default:
throw new NotSupportedException($"CopyCompression {compression} is not supported.");
}
}
finally
{
if (closeStream)
{
#pragma warning disable AsyncFixer02 // A disposable object used in a fire & forget async call
targetStream.Dispose();
#pragma warning restore AsyncFixer02 // A disposable object used in a fire & forget async call
}
}
return(CopyFileResult.Success);
}
}
public async Task <CopyFileResult> CheckBandwidthAtIntervalAsync(OperationContext context, Func <CancellationToken, Task <CopyFileResult> > copyTaskFactory, CopyToOptions options)
{
if (_historicalBandwidthLimitSource != null)
{
var timer = Stopwatch.StartNew();
var(result, bytesCopied) = await impl();
timer.Stop();
// Bandwidth checker expects speed in MiB/s, so convert it.
var speed = bytesCopied / timer.Elapsed.TotalSeconds / BytesInMb;
_historicalBandwidthLimitSource.AddBandwidthRecord(speed);
return(result);
}
else
{
return((await impl()).result);
}
async Task <(CopyFileResult result, long bytesCopied)> impl()
{
// This method should not fail with exceptions because the resulting task may be left unobserved causing an application to crash
// (given that the app is configured to fail on unobserved task exceptions).
var minimumSpeedInMbPerSec = _bandwidthLimitSource.GetMinimumSpeedInMbPerSec() * _config.BandwidthLimitMultiplier;
minimumSpeedInMbPerSec = Math.Min(minimumSpeedInMbPerSec, _config.MaxBandwidthLimit);
long startPosition = options.TotalBytesCopied;
long previousPosition = startPosition;
var copyCompleted = false;
using var copyCancellation = CancellationTokenSource.CreateLinkedTokenSource(context.Token);
Task <CopyFileResult> copyTask = copyTaskFactory(copyCancellation.Token);
// Subscribing for potential task failure here to avoid unobserved task exceptions.
traceCopyTaskFailures();
while (!copyCompleted)
{
// Wait some time for bytes to be copied
var firstCompletedTask = await Task.WhenAny(copyTask,
Task.Delay(_config.BandwidthCheckInterval, context.Token));
copyCompleted = firstCompletedTask == copyTask;
if (copyCompleted)
{
var result = await copyTask;
result.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec;
var bytesCopied = result.Size ?? options.TotalBytesCopied;
return(result, bytesCopied);
}
else if (context.Token.IsCancellationRequested)
{
context.Token.ThrowIfCancellationRequested();
}
// Copy is not completed and operation has not been canceled, perform
// bandwidth check
var position = options.TotalBytesCopied;
var receivedMiB = (position - previousPosition) / BytesInMb;
var currentSpeed = receivedMiB / _config.BandwidthCheckInterval.TotalSeconds;
if (currentSpeed == 0 || currentSpeed < minimumSpeedInMbPerSec)
{
copyCancellation.Cancel();
var totalBytesCopied = position - startPosition;
var result = new CopyFileResult(
CopyResultCode.CopyBandwidthTimeoutError,
$"Average speed was {currentSpeed}MiB/s - under {minimumSpeedInMbPerSec}MiB/s requirement. Aborting copy with {totalBytesCopied} bytes copied (received {position - previousPosition} bytes in {_config.BandwidthCheckInterval.TotalSeconds} seconds).");
return(result, totalBytesCopied);
}
previousPosition = position;
}
var copyFileResult = await copyTask;
copyFileResult.MinimumSpeedInMbPerSec = minimumSpeedInMbPerSec;
return(copyFileResult, previousPosition - startPosition);
void traceCopyTaskFailures()
{
// When the operation is cancelled, it is possible for the copy operation to fail.
// In this case we still want to trace the failure (but just with the debug severity and not with the error),
// but we should exclude ObjectDisposedException completely.
// That's why we don't use task.FireAndForget but tracing inside the task's continuation.
copyTask.ContinueWith(t =>
{
if (t.IsFaulted)
{
if (!(t.Exception?.InnerException is ObjectDisposedException))
{
context.TraceDebug($"Checked copy failed. {t.Exception}");
}
}
});
}
}
}
