public DiskTransferService(IDiskProvider diskProvider, Logger logger) { _diskProvider = diskProvider; _logger = logger; // TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer. // (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.) VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional; }
public DiskTransferService(IDiskProvider diskProvider, Logger logger) { _diskProvider = diskProvider; _logger = logger; // TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer. // (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.) VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional; }
public DiskTransferService(IDiskProvider diskProvider, Logger logger) { _diskProvider = diskProvider; _logger = logger; // TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer. // (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.) VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.Transactional; // TODO: Disabled the transactional logic entirely coz we have a couple of potential issues which we don't want in master. VerificationMode = DiskTransferVerificationMode.VerifyOnly; }
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode) { Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath(); if (!_diskProvider.FolderExists(targetPath)) { _diskProvider.CreateFolder(targetPath); } var result = mode; foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath)) { result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode); } foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath)) { var destFile = Path.Combine(targetPath, sourceFile.Name); result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode); } if (mode.HasFlag(TransferMode.Move)) { _diskProvider.DeleteFolder(sourcePath, true); } return(result); }
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize, DiskTransferVerificationMode verificationMode) { var backupPath = sourcePath + ".backup~"; var tempTargetPath = targetPath + ".partial~"; if (_diskProvider.FileExists(backupPath)) { _logger.Trace("Removing old backup."); _diskProvider.DeleteFile(backupPath); } if (_diskProvider.FileExists(tempTargetPath)) { _logger.Trace("Removing old partial."); _diskProvider.DeleteFile(tempTargetPath); } try { _logger.Trace("Attempting to move hardlinked backup."); if (_diskProvider.TryCreateHardLink(sourcePath, backupPath)) { _diskProvider.MoveFile(backupPath, tempTargetPath); if (_diskProvider.FileExists(tempTargetPath)) { var targetSize = _diskProvider.GetFileSize(tempTargetPath); if (targetSize == originalSize) { _diskProvider.MoveFile(tempTargetPath, targetPath); if (_diskProvider.FileExists(tempTargetPath)) { throw new IOException(string.Format("Temporary file '{0}' still exists, aborting.", tempTargetPath)); } _logger.Trace("Hardlink move succeeded, deleting source."); _diskProvider.DeleteFile(sourcePath); return(true); } } Thread.Sleep(5000); _diskProvider.DeleteFile(tempTargetPath); } } finally { if (_diskProvider.FileExists(backupPath)) { _diskProvider.DeleteFile(backupPath); } } if (verificationMode == DiskTransferVerificationMode.Transactional) { _logger.Trace("Hardlink move failed, reverting to copy."); if (TryCopyFileTransactional(sourcePath, targetPath, originalSize)) { _logger.Trace("Copy succeeded, deleting source."); _diskProvider.DeleteFile(sourcePath); return(true); } } else { _logger.Trace("Hardlink move failed, reverting to move."); TryMoveFileVerified(sourcePath, targetPath, originalSize); return(true); } _logger.Trace("Move failed."); return(false); }
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode) { Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath(); _logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath); var originalSize = _diskProvider.GetFileSize(sourcePath); if (sourcePath == targetPath) { throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); } if (sourcePath.PathEquals(targetPath, StringComparison.InvariantCultureIgnoreCase)) { if (mode.HasFlag(TransferMode.HardLink) || mode.HasFlag(TransferMode.Copy)) { throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); } if (mode.HasFlag(TransferMode.Move)) { var tempPath = sourcePath + ".backup~"; _diskProvider.MoveFile(sourcePath, tempPath, true); try { ClearTargetPath(targetPath, overwrite); _diskProvider.MoveFile(tempPath, targetPath); return(TransferMode.Move); } catch { RollbackMove(sourcePath, tempPath); throw; } } return(TransferMode.None); } if (sourcePath.GetParentPath() == targetPath.GetParentPath()) { if (mode.HasFlag(TransferMode.Move)) { TryMoveFileVerified(sourcePath, targetPath, originalSize); return(TransferMode.Move); } } if (sourcePath.IsParentPath(targetPath)) { throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath)); } ClearTargetPath(targetPath, overwrite); if (mode.HasFlag(TransferMode.HardLink)) { var createdHardlink = _diskProvider.TryCreateHardLink(sourcePath, targetPath); if (createdHardlink) { return(TransferMode.HardLink); } if (!mode.HasFlag(TransferMode.Copy)) { throw new IOException("Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed."); } } // We force a transactional transfer if the transfer occurs between mounts and one of the mounts is cifs, it would be a copy anyway. if (verificationMode == DiskTransferVerificationMode.TryTransactional && OsInfo.IsNotWindows) { var sourceMount = _diskProvider.GetMount(sourcePath); var targetMount = _diskProvider.GetMount(targetPath); if (sourceMount != null && targetMount != null && sourceMount.RootDirectory != targetMount.RootDirectory && (sourceMount.DriveFormat == "cifs" || targetMount.DriveFormat == "cifs")) { verificationMode = DiskTransferVerificationMode.Transactional; } } if (mode.HasFlag(TransferMode.Copy)) { if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) { if (TryCopyFileTransactional(sourcePath, targetPath, originalSize)) { return(TransferMode.Copy); } throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); } else if (verificationMode == DiskTransferVerificationMode.VerifyOnly) { TryCopyFileVerified(sourcePath, targetPath, originalSize); return(TransferMode.Copy); } else { _diskProvider.CopyFile(sourcePath, targetPath); return(TransferMode.Copy); } } if (mode.HasFlag(TransferMode.Move)) { if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) { if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode)) { return(TransferMode.Move); } throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); } else if (verificationMode == DiskTransferVerificationMode.VerifyOnly) { TryMoveFileVerified(sourcePath, targetPath, originalSize); return(TransferMode.Move); } else { _diskProvider.MoveFile(sourcePath, targetPath); return(TransferMode.Move); } } return(TransferMode.None); }
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize, DiskTransferVerificationMode verificationMode) { var backupPath = sourcePath + ".backup~"; var tempTargetPath = targetPath + ".partial~"; if (_diskProvider.FileExists(backupPath)) { _logger.Trace("Removing old backup."); _diskProvider.DeleteFile(backupPath); } if (_diskProvider.FileExists(tempTargetPath)) { _logger.Trace("Removing old partial."); _diskProvider.DeleteFile(tempTargetPath); } try { _logger.Trace("Attempting to move hardlinked backup."); if (_diskProvider.TryCreateHardLink(sourcePath, backupPath)) { _diskProvider.MoveFile(backupPath, tempTargetPath); if (_diskProvider.FileExists(tempTargetPath)) { var targetSize = _diskProvider.GetFileSize(tempTargetPath); if (targetSize == originalSize) { _diskProvider.MoveFile(tempTargetPath, targetPath); if (_diskProvider.FileExists(tempTargetPath)) { throw new IOException(string.Format("Temporary file '{0}' still exists, aborting.", tempTargetPath)); } _logger.Trace("Hardlink move succeeded, deleting source."); _diskProvider.DeleteFile(sourcePath); return true; } } Thread.Sleep(5000); _diskProvider.DeleteFile(tempTargetPath); } } finally { if (_diskProvider.FileExists(backupPath)) { _diskProvider.DeleteFile(backupPath); } } if (verificationMode == DiskTransferVerificationMode.Transactional) { _logger.Trace("Hardlink move failed, reverting to copy."); if (TryCopyFileTransactional(sourcePath, targetPath, originalSize)) { _logger.Trace("Copy succeeded, deleting source."); _diskProvider.DeleteFile(sourcePath); return true; } } else { _logger.Trace("Hardlink move failed, reverting to move."); TryMoveFileVerified(sourcePath, targetPath, originalSize); return true; } _logger.Trace("Move failed."); return false; }
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode) { Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath(); if (!_diskProvider.FolderExists(targetPath)) { _diskProvider.CreateFolder(targetPath); } var result = mode; foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath)) { result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode); } foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath)) { var destFile = Path.Combine(targetPath, sourceFile.Name); result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode); } if (mode.HasFlag(TransferMode.Move)) { _diskProvider.DeleteFolder(sourcePath, true); } return result; }
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode) { Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath(); _logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath); var originalSize = _diskProvider.GetFileSize(sourcePath); if (sourcePath == targetPath) { throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); } if (sourcePath.PathEquals(targetPath, StringComparison.InvariantCultureIgnoreCase)) { if (mode.HasFlag(TransferMode.HardLink) || mode.HasFlag(TransferMode.Copy)) { throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); } if (mode.HasFlag(TransferMode.Move)) { var tempPath = sourcePath + ".backup~"; _diskProvider.MoveFile(sourcePath, tempPath, true); try { ClearTargetPath(targetPath, overwrite); _diskProvider.MoveFile(tempPath, targetPath); return TransferMode.Move; } catch { RollbackMove(sourcePath, tempPath); throw; } } return TransferMode.None; } if (sourcePath.GetParentPath() == targetPath.GetParentPath()) { if (mode.HasFlag(TransferMode.Move)) { TryMoveFileVerified(sourcePath, targetPath, originalSize); return TransferMode.Move; } } if (sourcePath.IsParentPath(targetPath)) { throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath)); } ClearTargetPath(targetPath, overwrite); if (mode.HasFlag(TransferMode.HardLink)) { var createdHardlink = _diskProvider.TryCreateHardLink(sourcePath, targetPath); if (createdHardlink) { return TransferMode.HardLink; } if (!mode.HasFlag(TransferMode.Copy)) { throw new IOException("Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed."); } } // We force a transactional transfer if the transfer occurs between mounts and one of the mounts is cifs, it would be a copy anyway. if (verificationMode == DiskTransferVerificationMode.TryTransactional && OsInfo.IsNotWindows) { var sourceMount = _diskProvider.GetMount(sourcePath); var targetMount = _diskProvider.GetMount(targetPath); if (sourceMount != null && targetMount != null && sourceMount.RootDirectory != targetMount.RootDirectory && (sourceMount.DriveFormat == "cifs" || targetMount.DriveFormat == "cifs")) { verificationMode = DiskTransferVerificationMode.Transactional; } } if (mode.HasFlag(TransferMode.Copy)) { if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) { if (TryCopyFileTransactional(sourcePath, targetPath, originalSize)) { return TransferMode.Copy; } throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); } else if (verificationMode == DiskTransferVerificationMode.VerifyOnly) { TryCopyFileVerified(sourcePath, targetPath, originalSize); return TransferMode.Copy; } else { _diskProvider.CopyFile(sourcePath, targetPath); return TransferMode.Copy; } } if (mode.HasFlag(TransferMode.Move)) { if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) { if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode)) { return TransferMode.Move; } throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); } else if (verificationMode == DiskTransferVerificationMode.VerifyOnly) { TryMoveFileVerified(sourcePath, targetPath, originalSize); return TransferMode.Move; } else { _diskProvider.MoveFile(sourcePath, targetPath); return TransferMode.Move; } } return TransferMode.None; }
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode) { Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath(); if (mode == TransferMode.Move && !_diskProvider.FolderExists(targetPath)) { if (verificationMode == DiskTransferVerificationMode.TryTransactional || verificationMode == DiskTransferVerificationMode.VerifyOnly) { var sourceMount = _diskProvider.GetMount(sourcePath); var targetMount = _diskProvider.GetMount(targetPath); // If we're on the same mount, do a simple folder move. if (sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory) { _logger.Debug("Move Directory [{0}] > [{1}]", sourcePath, targetPath); _diskProvider.MoveFolder(sourcePath, targetPath); return(mode); } } } if (!_diskProvider.FolderExists(targetPath)) { _diskProvider.CreateFolder(targetPath); } var result = mode; foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath)) { if (ShouldIgnore(subDir)) { continue; } result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode); } foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath)) { if (ShouldIgnore(sourceFile)) { continue; } var destFile = Path.Combine(targetPath, sourceFile.Name); result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode); } if (mode.HasFlag(TransferMode.Move)) { _diskProvider.DeleteFolder(sourcePath, true); } return(result); }
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode) { Ensure.That(sourcePath, () => sourcePath).IsValidPath(); Ensure.That(targetPath, () => targetPath).IsValidPath(); _logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath); var originalSize = _diskProvider.GetFileSize(sourcePath); if (sourcePath == targetPath) { throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); } if (sourcePath.PathEquals(targetPath, StringComparison.InvariantCultureIgnoreCase)) { if (mode.HasFlag(TransferMode.HardLink) || mode.HasFlag(TransferMode.Copy)) { throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath)); } if (mode.HasFlag(TransferMode.Move)) { var tempPath = sourcePath + ".backup~"; _diskProvider.MoveFile(sourcePath, tempPath, true); try { ClearTargetPath(sourcePath, targetPath, overwrite); _diskProvider.MoveFile(tempPath, targetPath); return(TransferMode.Move); } catch { RollbackMove(sourcePath, tempPath); throw; } } return(TransferMode.None); } if (sourcePath.GetParentPath() == targetPath.GetParentPath()) { if (mode.HasFlag(TransferMode.Move)) { TryMoveFileVerified(sourcePath, targetPath, originalSize); return(TransferMode.Move); } } if (sourcePath.IsParentPath(targetPath)) { throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath)); } ClearTargetPath(sourcePath, targetPath, overwrite); if (mode.HasFlag(TransferMode.HardLink)) { var createdHardlink = _diskProvider.TryCreateHardLink(sourcePath, targetPath); if (createdHardlink) { return(TransferMode.HardLink); } if (!mode.HasFlag(TransferMode.Copy)) { throw new IOException("Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed."); } } // Adjust the transfer mode depending on the filesystems if (verificationMode == DiskTransferVerificationMode.TryTransactional) { var sourceMount = _diskProvider.GetMount(sourcePath); var targetMount = _diskProvider.GetMount(targetPath); var isSameMount = (sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory); var sourceDriveFormat = sourceMount?.DriveFormat ?? string.Empty; var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty; if (isSameMount) { // No transaction needed for operations on same mount, force VerifyOnly verificationMode = DiskTransferVerificationMode.VerifyOnly; } else if (sourceDriveFormat.Contains("mergerfs") || sourceDriveFormat.Contains("rclone") || targetDriveFormat.Contains("mergerfs") || targetDriveFormat.Contains("rclone")) { // Cloud storage filesystems don't need any Transactional stuff and it hurts performance, force VerifyOnly verificationMode = DiskTransferVerificationMode.VerifyOnly; } else if ((sourceDriveFormat == "cifs" || targetDriveFormat == "cifs") && OsInfo.IsNotWindows) { // Force Transactional on a cifs mount due to the likeliness of move failures on certain scenario's on mono verificationMode = DiskTransferVerificationMode.Transactional; } } if (mode.HasFlag(TransferMode.Copy)) { if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) { if (TryCopyFileTransactional(sourcePath, targetPath, originalSize)) { return(TransferMode.Copy); } throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); } else if (verificationMode == DiskTransferVerificationMode.VerifyOnly) { TryCopyFileVerified(sourcePath, targetPath, originalSize); return(TransferMode.Copy); } else { _diskProvider.CopyFile(sourcePath, targetPath); return(TransferMode.Copy); } } if (mode.HasFlag(TransferMode.Move)) { if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional) { if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode)) { return(TransferMode.Move); } throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath)); } else if (verificationMode == DiskTransferVerificationMode.VerifyOnly) { TryMoveFileVerified(sourcePath, targetPath, originalSize); return(TransferMode.Move); } else { _diskProvider.MoveFile(sourcePath, targetPath); return(TransferMode.Move); } } return(TransferMode.None); }