public DiskTransferService(IDiskProvider diskProvider, Logger logger)
{
_diskProvider = diskProvider;
_logger = logger;
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional;
}
public DiskTransferService(IDiskProvider diskProvider, Logger logger)
{
_diskProvider = diskProvider;
_logger = logger;
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.TryTransactional;
}
public DiskTransferService(IDiskProvider diskProvider, Logger logger)
{
_diskProvider = diskProvider;
_logger = logger;
// TODO: Atm we haven't seen partial transfers on windows so we disable verified transfer.
// (If enabled in the future, be sure to check specifically for ReFS, which doesn't support hardlinks.)
VerificationMode = OsInfo.IsWindows ? DiskTransferVerificationMode.VerifyOnly : DiskTransferVerificationMode.Transactional;
// TODO: Disabled the transactional logic entirely coz we have a couple of potential issues which we don't want in master.
VerificationMode = DiskTransferVerificationMode.VerifyOnly;
}
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
if (!_diskProvider.FolderExists(targetPath))
{
_diskProvider.CreateFolder(targetPath);
}
var result = mode;
foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath))
{
result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode);
}
foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath))
{
var destFile = Path.Combine(targetPath, sourceFile.Name);
result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode);
}
if (mode.HasFlag(TransferMode.Move))
{
_diskProvider.DeleteFolder(sourcePath, true);
}
return(result);
}
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize, DiskTransferVerificationMode verificationMode)
{
var backupPath = sourcePath + ".backup~";
var tempTargetPath = targetPath + ".partial~";
if (_diskProvider.FileExists(backupPath))
{
_logger.Trace("Removing old backup.");
_diskProvider.DeleteFile(backupPath);
}
if (_diskProvider.FileExists(tempTargetPath))
{
_logger.Trace("Removing old partial.");
_diskProvider.DeleteFile(tempTargetPath);
}
try
{
_logger.Trace("Attempting to move hardlinked backup.");
if (_diskProvider.TryCreateHardLink(sourcePath, backupPath))
{
_diskProvider.MoveFile(backupPath, tempTargetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
var targetSize = _diskProvider.GetFileSize(tempTargetPath);
if (targetSize == originalSize)
{
_diskProvider.MoveFile(tempTargetPath, targetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
throw new IOException(string.Format("Temporary file '{0}' still exists, aborting.", tempTargetPath));
}
_logger.Trace("Hardlink move succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return(true);
}
}
Thread.Sleep(5000);
_diskProvider.DeleteFile(tempTargetPath);
}
}
finally
{
if (_diskProvider.FileExists(backupPath))
{
_diskProvider.DeleteFile(backupPath);
}
}
if (verificationMode == DiskTransferVerificationMode.Transactional)
{
_logger.Trace("Hardlink move failed, reverting to copy.");
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
_logger.Trace("Copy succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return(true);
}
}
else
{
_logger.Trace("Hardlink move failed, reverting to move.");
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return(true);
}
_logger.Trace("Move failed.");
return(false);
}
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
var originalSize = _diskProvider.GetFileSize(sourcePath);
if (sourcePath == targetPath)
{
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
}
if (sourcePath.PathEquals(targetPath, StringComparison.InvariantCultureIgnoreCase))
{
if (mode.HasFlag(TransferMode.HardLink) || mode.HasFlag(TransferMode.Copy))
{
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
}
if (mode.HasFlag(TransferMode.Move))
{
var tempPath = sourcePath + ".backup~";
_diskProvider.MoveFile(sourcePath, tempPath, true);
try
{
ClearTargetPath(targetPath, overwrite);
_diskProvider.MoveFile(tempPath, targetPath);
return(TransferMode.Move);
}
catch
{
RollbackMove(sourcePath, tempPath);
throw;
}
}
return(TransferMode.None);
}
if (sourcePath.GetParentPath() == targetPath.GetParentPath())
{
if (mode.HasFlag(TransferMode.Move))
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return(TransferMode.Move);
}
}
if (sourcePath.IsParentPath(targetPath))
{
throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath));
}
ClearTargetPath(targetPath, overwrite);
if (mode.HasFlag(TransferMode.HardLink))
{
var createdHardlink = _diskProvider.TryCreateHardLink(sourcePath, targetPath);
if (createdHardlink)
{
return(TransferMode.HardLink);
}
if (!mode.HasFlag(TransferMode.Copy))
{
throw new IOException("Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed.");
}
}
// We force a transactional transfer if the transfer occurs between mounts and one of the mounts is cifs, it would be a copy anyway.
if (verificationMode == DiskTransferVerificationMode.TryTransactional && OsInfo.IsNotWindows)
{
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
if (sourceMount != null && targetMount != null && sourceMount.RootDirectory != targetMount.RootDirectory &&
(sourceMount.DriveFormat == "cifs" || targetMount.DriveFormat == "cifs"))
{
verificationMode = DiskTransferVerificationMode.Transactional;
}
}
if (mode.HasFlag(TransferMode.Copy))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
return(TransferMode.Copy);
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryCopyFileVerified(sourcePath, targetPath, originalSize);
return(TransferMode.Copy);
}
else
{
_diskProvider.CopyFile(sourcePath, targetPath);
return(TransferMode.Copy);
}
}
if (mode.HasFlag(TransferMode.Move))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode))
{
return(TransferMode.Move);
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return(TransferMode.Move);
}
else
{
_diskProvider.MoveFile(sourcePath, targetPath);
return(TransferMode.Move);
}
}
return(TransferMode.None);
}
private bool TryMoveFileTransactional(string sourcePath, string targetPath, long originalSize, DiskTransferVerificationMode verificationMode)
{
var backupPath = sourcePath + ".backup~";
var tempTargetPath = targetPath + ".partial~";
if (_diskProvider.FileExists(backupPath))
{
_logger.Trace("Removing old backup.");
_diskProvider.DeleteFile(backupPath);
}
if (_diskProvider.FileExists(tempTargetPath))
{
_logger.Trace("Removing old partial.");
_diskProvider.DeleteFile(tempTargetPath);
}
try
{
_logger.Trace("Attempting to move hardlinked backup.");
if (_diskProvider.TryCreateHardLink(sourcePath, backupPath))
{
_diskProvider.MoveFile(backupPath, tempTargetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
var targetSize = _diskProvider.GetFileSize(tempTargetPath);
if (targetSize == originalSize)
{
_diskProvider.MoveFile(tempTargetPath, targetPath);
if (_diskProvider.FileExists(tempTargetPath))
{
throw new IOException(string.Format("Temporary file '{0}' still exists, aborting.", tempTargetPath));
}
_logger.Trace("Hardlink move succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return true;
}
}
Thread.Sleep(5000);
_diskProvider.DeleteFile(tempTargetPath);
}
}
finally
{
if (_diskProvider.FileExists(backupPath))
{
_diskProvider.DeleteFile(backupPath);
}
}
if (verificationMode == DiskTransferVerificationMode.Transactional)
{
_logger.Trace("Hardlink move failed, reverting to copy.");
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
_logger.Trace("Copy succeeded, deleting source.");
_diskProvider.DeleteFile(sourcePath);
return true;
}
}
else
{
_logger.Trace("Hardlink move failed, reverting to move.");
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return true;
}
_logger.Trace("Move failed.");
return false;
}
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
if (!_diskProvider.FolderExists(targetPath))
{
_diskProvider.CreateFolder(targetPath);
}
var result = mode;
foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath))
{
result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode);
}
foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath))
{
var destFile = Path.Combine(targetPath, sourceFile.Name);
result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode);
}
if (mode.HasFlag(TransferMode.Move))
{
_diskProvider.DeleteFolder(sourcePath, true);
}
return result;
}
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
var originalSize = _diskProvider.GetFileSize(sourcePath);
if (sourcePath == targetPath)
{
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
}
if (sourcePath.PathEquals(targetPath, StringComparison.InvariantCultureIgnoreCase))
{
if (mode.HasFlag(TransferMode.HardLink) || mode.HasFlag(TransferMode.Copy))
{
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
}
if (mode.HasFlag(TransferMode.Move))
{
var tempPath = sourcePath + ".backup~";
_diskProvider.MoveFile(sourcePath, tempPath, true);
try
{
ClearTargetPath(targetPath, overwrite);
_diskProvider.MoveFile(tempPath, targetPath);
return TransferMode.Move;
}
catch
{
RollbackMove(sourcePath, tempPath);
throw;
}
}
return TransferMode.None;
}
if (sourcePath.GetParentPath() == targetPath.GetParentPath())
{
if (mode.HasFlag(TransferMode.Move))
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
}
}
if (sourcePath.IsParentPath(targetPath))
{
throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath));
}
ClearTargetPath(targetPath, overwrite);
if (mode.HasFlag(TransferMode.HardLink))
{
var createdHardlink = _diskProvider.TryCreateHardLink(sourcePath, targetPath);
if (createdHardlink)
{
return TransferMode.HardLink;
}
if (!mode.HasFlag(TransferMode.Copy))
{
throw new IOException("Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed.");
}
}
// We force a transactional transfer if the transfer occurs between mounts and one of the mounts is cifs, it would be a copy anyway.
if (verificationMode == DiskTransferVerificationMode.TryTransactional && OsInfo.IsNotWindows)
{
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
if (sourceMount != null && targetMount != null && sourceMount.RootDirectory != targetMount.RootDirectory &&
(sourceMount.DriveFormat == "cifs" || targetMount.DriveFormat == "cifs"))
{
verificationMode = DiskTransferVerificationMode.Transactional;
}
}
if (mode.HasFlag(TransferMode.Copy))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
return TransferMode.Copy;
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryCopyFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Copy;
}
else
{
_diskProvider.CopyFile(sourcePath, targetPath);
return TransferMode.Copy;
}
}
if (mode.HasFlag(TransferMode.Move))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode))
{
return TransferMode.Move;
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return TransferMode.Move;
}
else
{
_diskProvider.MoveFile(sourcePath, targetPath);
return TransferMode.Move;
}
}
return TransferMode.None;
}
public TransferMode TransferFolder(string sourcePath, string targetPath, TransferMode mode, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
if (mode == TransferMode.Move && !_diskProvider.FolderExists(targetPath))
{
if (verificationMode == DiskTransferVerificationMode.TryTransactional || verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
// If we're on the same mount, do a simple folder move.
if (sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory)
{
_logger.Debug("Move Directory [{0}] > [{1}]", sourcePath, targetPath);
_diskProvider.MoveFolder(sourcePath, targetPath);
return(mode);
}
}
}
if (!_diskProvider.FolderExists(targetPath))
{
_diskProvider.CreateFolder(targetPath);
}
var result = mode;
foreach (var subDir in _diskProvider.GetDirectoryInfos(sourcePath))
{
if (ShouldIgnore(subDir))
{
continue;
}
result &= TransferFolder(subDir.FullName, Path.Combine(targetPath, subDir.Name), mode, verificationMode);
}
foreach (var sourceFile in _diskProvider.GetFileInfos(sourcePath))
{
if (ShouldIgnore(sourceFile))
{
continue;
}
var destFile = Path.Combine(targetPath, sourceFile.Name);
result &= TransferFile(sourceFile.FullName, destFile, mode, true, verificationMode);
}
if (mode.HasFlag(TransferMode.Move))
{
_diskProvider.DeleteFolder(sourcePath, true);
}
return(result);
}
public TransferMode TransferFile(string sourcePath, string targetPath, TransferMode mode, bool overwrite, DiskTransferVerificationMode verificationMode)
{
Ensure.That(sourcePath, () => sourcePath).IsValidPath();
Ensure.That(targetPath, () => targetPath).IsValidPath();
_logger.Debug("{0} [{1}] > [{2}]", mode, sourcePath, targetPath);
var originalSize = _diskProvider.GetFileSize(sourcePath);
if (sourcePath == targetPath)
{
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
}
if (sourcePath.PathEquals(targetPath, StringComparison.InvariantCultureIgnoreCase))
{
if (mode.HasFlag(TransferMode.HardLink) || mode.HasFlag(TransferMode.Copy))
{
throw new IOException(string.Format("Source and destination can't be the same {0}", sourcePath));
}
if (mode.HasFlag(TransferMode.Move))
{
var tempPath = sourcePath + ".backup~";
_diskProvider.MoveFile(sourcePath, tempPath, true);
try
{
ClearTargetPath(sourcePath, targetPath, overwrite);
_diskProvider.MoveFile(tempPath, targetPath);
return(TransferMode.Move);
}
catch
{
RollbackMove(sourcePath, tempPath);
throw;
}
}
return(TransferMode.None);
}
if (sourcePath.GetParentPath() == targetPath.GetParentPath())
{
if (mode.HasFlag(TransferMode.Move))
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return(TransferMode.Move);
}
}
if (sourcePath.IsParentPath(targetPath))
{
throw new IOException(string.Format("Destination cannot be a child of the source [{0}] => [{1}]", sourcePath, targetPath));
}
ClearTargetPath(sourcePath, targetPath, overwrite);
if (mode.HasFlag(TransferMode.HardLink))
{
var createdHardlink = _diskProvider.TryCreateHardLink(sourcePath, targetPath);
if (createdHardlink)
{
return(TransferMode.HardLink);
}
if (!mode.HasFlag(TransferMode.Copy))
{
throw new IOException("Hardlinking from '" + sourcePath + "' to '" + targetPath + "' failed.");
}
}
// Adjust the transfer mode depending on the filesystems
if (verificationMode == DiskTransferVerificationMode.TryTransactional)
{
var sourceMount = _diskProvider.GetMount(sourcePath);
var targetMount = _diskProvider.GetMount(targetPath);
var isSameMount = (sourceMount != null && targetMount != null && sourceMount.RootDirectory == targetMount.RootDirectory);
var sourceDriveFormat = sourceMount?.DriveFormat ?? string.Empty;
var targetDriveFormat = targetMount?.DriveFormat ?? string.Empty;
if (isSameMount)
{
// No transaction needed for operations on same mount, force VerifyOnly
verificationMode = DiskTransferVerificationMode.VerifyOnly;
}
else if (sourceDriveFormat.Contains("mergerfs") || sourceDriveFormat.Contains("rclone") ||
targetDriveFormat.Contains("mergerfs") || targetDriveFormat.Contains("rclone"))
{
// Cloud storage filesystems don't need any Transactional stuff and it hurts performance, force VerifyOnly
verificationMode = DiskTransferVerificationMode.VerifyOnly;
}
else if ((sourceDriveFormat == "cifs" || targetDriveFormat == "cifs") && OsInfo.IsNotWindows)
{
// Force Transactional on a cifs mount due to the likeliness of move failures on certain scenario's on mono
verificationMode = DiskTransferVerificationMode.Transactional;
}
}
if (mode.HasFlag(TransferMode.Copy))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryCopyFileTransactional(sourcePath, targetPath, originalSize))
{
return(TransferMode.Copy);
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryCopyFileVerified(sourcePath, targetPath, originalSize);
return(TransferMode.Copy);
}
else
{
_diskProvider.CopyFile(sourcePath, targetPath);
return(TransferMode.Copy);
}
}
if (mode.HasFlag(TransferMode.Move))
{
if (verificationMode == DiskTransferVerificationMode.Transactional || verificationMode == DiskTransferVerificationMode.TryTransactional)
{
if (TryMoveFileTransactional(sourcePath, targetPath, originalSize, verificationMode))
{
return(TransferMode.Move);
}
throw new IOException(string.Format("Failed to completely transfer [{0}] to [{1}], aborting.", sourcePath, targetPath));
}
else if (verificationMode == DiskTransferVerificationMode.VerifyOnly)
{
TryMoveFileVerified(sourcePath, targetPath, originalSize);
return(TransferMode.Move);
}
else
{
_diskProvider.MoveFile(sourcePath, targetPath);
return(TransferMode.Move);
}
}
return(TransferMode.None);
}
