private void GenerateDir(DirNode dirNode, CloudFileDirectory cloudFileDir, string parentPath)
{
string dirPath = Path.Combine(parentPath, dirNode.Name);
DMLibDataHelper.CreateLocalDirIfNotExists(dirPath);
cloudFileDir.CreateIfNotExists(HelperConst.DefaultFileOptions);
foreach (var subDir in dirNode.DirNodes)
{
CloudFileDirectory subCloudFileDir = cloudFileDir.GetDirectoryReference(subDir.Name);
this.GenerateDir(subDir, subCloudFileDir, dirPath);
}
foreach (var file in dirNode.FileNodes)
{
CloudFile cloudFile = cloudFileDir.GetFileReference(file.Name);
this.GenerateFile(file, cloudFile, dirPath);
}
}
public void FollowSymlink_1_SymlinkDir()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
var dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
sourceDataInfo.RootNode.AddDirNode(dirNode);
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", dirNode.Name, dirNode));
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
(transferOptions as UploadDirectoryOptions).FollowSymlink = true;
item.Options = transferOptions;
},
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
private void BuildSymlinkedDirNode(DirNode parent, string parentPath)
{
if (!CrossPlatformHelpers.IsLinux)
{
throw new PlatformNotSupportedException();
}
foreach (FileNode fileNode in parent.FileNodes)
{
Test.Info("Building file info of {0}", Path.Combine(parentPath, fileNode.Name));
FileInfo fileInfo = new FileInfo(Path.Combine(parentPath, fileNode.Name));
this.BuildFileNode(fileInfo, fileNode, false, PreserveSMBPermissions.None);
}
foreach (DirNode dirNode in parent.DirNodes)
{
BuildSymlinkedDirNode(dirNode, Path.Combine(parentPath, dirNode.Name));
}
}
private void BuildDirNode(CloudFileDirectory cloudDir, DirNode dirNode)
{
cloudDir.FetchAttributes(options: HelperConst.DefaultFileOptions);
dirNode.LastWriteTime = cloudDir.Properties.LastWriteTime;
dirNode.CreationTime = cloudDir.Properties.CreationTime;
dirNode.SMBAttributes = cloudDir.Properties.NtfsAttributes;
if (!string.IsNullOrEmpty(cloudDir.FilePermission))
{
dirNode.PortableSDDL = cloudDir.FilePermission;
}
else if (!string.IsNullOrEmpty(cloudDir.Properties.FilePermissionKey))
{
dirNode.PortableSDDL = cloudDir.Share.GetFilePermission(cloudDir.Properties.FilePermissionKey, options: HelperConst.DefaultFileOptions);
}
if (cloudDir.Metadata.Count > 0)
{
dirNode.Metadata = new Dictionary <string, string>(cloudDir.Metadata);
}
foreach (IListFileItem item in cloudDir.ListFilesAndDirectories(HelperConst.DefaultFileOptions))
{
CloudFile cloudFile = item as CloudFile;
CloudFileDirectory subCloudDir = item as CloudFileDirectory;
if (cloudFile != null)
{
// Cannot fetch attributes while listing, so do it for each cloud file.
cloudFile.FetchAttributes(options: HelperConst.DefaultFileOptions);
FileNode fileNode = new FileNode(cloudFile.Name);
this.BuildFileNode(cloudFile, fileNode);
dirNode.AddFileNode(fileNode);
}
else if (subCloudDir != null)
{
DirNode subDirNode = new DirNode(subCloudDir.Name);
this.BuildDirNode(subCloudDir, subDirNode);
dirNode.AddDirNode(subDirNode);
}
}
}
private static void PrepareDirSourceData(long fileSizeInB)
{
foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections())
{
string dirName = GetTransferDirName(direction);
string fileName = dirName;
string sourceDataInfoKey = GetSourceDataInfoKey(direction);
DMLibDataInfo sourceDataInfo = GetSourceDataInfo(sourceDataInfoKey);
DirNode subDirNode = new DirNode(dirName);
DMLibDataHelper.AddOneFileInBytes(subDirNode, fileName, fileSizeInB);
sourceDataInfo.RootNode.AddDirNode(subDirNode);
directoryNodes.Add(dirName, subDirNode);
expectedFileNodes.Add(fileName, subDirNode.GetFileNode(fileName));
}
}
private static List <TransferItem> GetTransformItemsForAllDirTransferDirections(bool resume)
{
List <TransferItem> allItems = new List <TransferItem>();
foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections())
{
if ((direction.SourceType != DMLibDataType.CloudFile) ||
(direction.DestType != DMLibDataType.Local))
{
continue;
}
string dirName = GetTransferDirName(direction);
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
DirNode dirNode = directoryNodes[dirName];
dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType);
options.Recursive = true;
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
CopyMethod = direction.CopyMethod,
IsDirectoryTransfer = true,
Options = options,
TransferContext = new DirectoryTransferContext()
{
SetAttributesCallbackAsync = AllTransferDirectionTest.SetAttributesCallbackMethodAsync
}
};
allItems.Add(item);
}
return(allItems);
}
public CloudBlobDirectory GetCloudBlobDirReference(string rootPath, DirNode dirNode, StorageCredentials credentials = null)
{
var container = this.BlobHelper.BlobClient.GetContainerReference(this.ContainerName);
if (credentials != null)
{
container = new CloudBlobContainer(container.StorageUri, credentials);
}
var dirName = dirNode.GetURLRelativePath();
if (dirName.StartsWith("/"))
{
dirName = dirName.Substring(1, dirName.Length - 1);
}
if (!string.IsNullOrEmpty(rootPath))
{
dirName = rootPath + "/" + dirName;
}
return(container.GetDirectoryReference(dirName));
}
private void GenerateDir(DirNode dirNode, CloudBlobDirectory cloudBlobDir)
{
DMLibDataHelper.CreateLocalDirIfNotExists(this.TempFolder);
foreach (var subDir in dirNode.DirNodes)
{
CloudBlobDirectory subCloudBlobDir = cloudBlobDir.GetDirectoryReference(subDir.Name);
this.GenerateDir(subDir, subCloudBlobDir);
}
List <FileNode> snapshotList = new List <FileNode>();
foreach (var file in dirNode.FileNodes)
{
CloudBlob cloudBlob = CloudBlobHelper.GetCloudBlobReference(cloudBlobDir, file.Name, this.BlobType);
this.GenerateFile(file, cloudBlob, snapshotList);
}
foreach (var snapshot in snapshotList)
{
dirNode.AddFileNode(snapshot);
}
}
private void BuildDirNode(string dirPath, DirNode parent, bool handleSMBAttributes, PreserveSMBPermissions getSMBPermissions)
{
dirPath = AppendDirectorySeparator(dirPath);
DateTimeOffset?creationTime = null;
DateTimeOffset?lastWriteTime = null;
FileAttributes?fileAttributes = null;
#if DOTNET5_4
LongPathFileExtension.GetFileProperties(dirPath, out creationTime, out lastWriteTime, out fileAttributes, true);
#else
LongPathFileExtension.GetFileProperties(dirPath, out creationTime, out lastWriteTime, out fileAttributes);
#endif
parent.CreationTime = creationTime;
parent.LastWriteTime = lastWriteTime;
if (PreserveSMBPermissions.None != getSMBPermissions)
{
parent.PortableSDDL = LongPathFileExtension.GetFilePortableSDDL(dirPath, getSMBPermissions);
}
foreach (var fileInfo in LongPathDirectoryExtension.GetFiles(dirPath))
{
FileNode fileNode = new FileNode(fileInfo.Remove(0, dirPath.Length));
this.BuildFileNode(fileInfo, fileNode, handleSMBAttributes, getSMBPermissions);
parent.AddFileNode(fileNode);
}
foreach (var subDirInfo in LongPathDirectoryExtension.GetDirectories(dirPath))
{
DirNode subDirNode = new DirNode(subDirInfo.Remove(0, dirPath.Length));
this.BuildDirNode(subDirInfo, subDirNode, handleSMBAttributes, getSMBPermissions);
parent.AddDirNode(subDirNode);
}
}
public CloudFileDirectory GetCloudFileDirReference(string rootPath, DirNode dirNode)
{
var share = this.fileHelper.FileClient.GetShareReference(this.shareName);
string dirName = dirNode.GetURLRelativePath();
if (dirName.StartsWith("/"))
{
dirName = dirName.Substring(1, dirName.Length - 1);
}
if (!string.IsNullOrEmpty(rootPath))
{
dirName = rootPath + "/" + dirName;
}
if (string.IsNullOrEmpty(dirName))
{
return(share.GetRootDirectoryReference());
}
else
{
return(share.GetRootDirectoryReference().GetDirectoryReference(dirName));
}
}
private void BuildDirNode(CloudFileDirectory cloudDir, DirNode dirNode)
{
foreach (IListFileItem item in cloudDir.ListFilesAndDirectories(HelperConst.DefaultFileOptions))
{
CloudFile cloudFile = item as CloudFile;
CloudFileDirectory subCloudDir = item as CloudFileDirectory;
if (cloudFile != null)
{
// Cannot fetch attributes while listing, so do it for each cloud file.
cloudFile.FetchAttributes(options: HelperConst.DefaultFileOptions);
FileNode fileNode = new FileNode(cloudFile.Name);
this.BuildFileNode(cloudFile, fileNode);
dirNode.AddFileNode(fileNode);
}
else if (subCloudDir != null)
{
DirNode subDirNode = new DirNode(subCloudDir.Name);
this.BuildDirNode(subCloudDir, subDirNode);
dirNode.AddDirNode(subDirNode);
}
}
}
public static void AddMultipleFilesBigSize(DirNode dirNode, string filePrefix)
{
int[] fileSizes = new int[] { 32000, 64 * 1024 };
AddMultipleFilesDifferentSize(dirNode, filePrefix, fileSizes);
}
public static void AddMultipleFilesNormalSize(DirNode dirNode, string filePrefix)
{
int[] fileSizes = new int[] { 0, 1, 4000, 4 * 1024, 10000 };
AddMultipleFilesDifferentSize(dirNode, filePrefix, fileSizes);
}
public static void AddMultipleFiles(DirNode dirNode, string filePrefix, int fileNumber, int fileSizeInKB, FileAttributes?fa = null, DateTime?lmt = null)
{
DMLibDataHelper.AddTree(dirNode, string.Empty, filePrefix, fileNumber, 0, fileSizeInKB, fa, lmt);
}
public static DirNode RemoveOneDir(DirNode parentNode, string dirNodeToDelete)
{
return(parentNode.DeleteDirNode(dirNodeToDelete));
}
public static FileNode RemoveOneFile(DirNode dirNode, string fileName)
{
return(dirNode.DeleteFileNode(fileName));
}
public static void AddOneFile(DirNode dirNode, string fileName, long fileSizeInKB, FileAttributes?fa = null, DateTime?lmt = null)
{
AddOneFileInBytes(dirNode, fileName, 1024L * fileSizeInKB, fa, lmt);
}
private static void PrepareSourceData()
{
DMLibDataInfo sourceFileTree = new DMLibDataInfo(string.Empty);
DirNode dirNode1 = new DirNode("folder1");
DirNode subDir1 = new DirNode("subfolder1");
subDir1.AddDirNode(new DirNode("subfolder3"));
subDir1.AddFileNode(GenerateFileNode("testfile2"));
subDir1.AddFileNode(GenerateFileNode("4testfile"));
dirNode1.AddDirNode(subDir1);
DirNode subDir2 = new DirNode("subfolder2");
DirNode subDir4 = new DirNode("subfolder4");
subDir4.AddFileNode(GenerateFileNode("test5"));
subDir2.AddDirNode(subDir4);
subDir2.AddFileNode(GenerateFileNode("TESTFILE345"));
subDir2.AddFileNode(GenerateFileNode("testfile234"));
subDir2.AddFileNode(GenerateFileNode("testYfile"));
subDir2.AddFileNode(GenerateFileNode("f_arbitrary.exe"));
subDir2.AddFileNode(GenerateFileNode("测试x文件"));
dirNode1.AddDirNode(subDir2);
dirNode1.AddFileNode(GenerateFileNode("testfile1"));
dirNode1.AddFileNode(GenerateFileNode("TestFile2"));
dirNode1.AddFileNode(GenerateFileNode("测试文件2"));
sourceFileTree.RootNode.AddDirNode(dirNode1);
DirNode dirNode2 = new DirNode("folder2");
dirNode2.AddFileNode(GenerateFileNode("folder_file"));
dirNode2.AddDirNode(new DirNode("testfile1"));
dirNode2.AddFileNode(GenerateFileNode("测试文件三"));
dirNode2.AddFileNode(GenerateFileNode("测试四文件"));
sourceFileTree.RootNode.AddDirNode(dirNode2);
DirNode dirNode3 = new DirNode("folder3");
sourceFileTree.RootNode.AddDirNode(dirNode3);
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile1"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile2"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXfile"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXXfile"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("测试文件"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("..a123"));
DMLibDataInfo blobSourceFileTree = sourceFileTree.Clone();
blobSourceFileTree.RootNode.AddFileNode(GenerateFileNode("TeSTfIle"));
Test.Info("Start to generate test data, will take a while...");
foreach (DMLibDataType dataType in sourceDataTypes)
{
if (IsCloudBlob(dataType))
{
PrepareSourceData(dataType, blobSourceFileTree.Clone());
}
else
{
PrepareSourceData(dataType, sourceFileTree.Clone());
}
}
Test.Info("Done");
}
public void TestDirectoryResume()
{
int bigFileSizeInKB = 5 * 1024; // 5 MB
int smallFileSizeInKB = 1; // 1 KB
int bigFileNum = 5;
int smallFileNum = 50;
long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
int totalFileNum = bigFileNum + smallFileNum;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
options.IsDirectoryTransfer = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Test.Assert(e.Exception.Message.Contains("cancel"), "Verify task is canceled: {0}", e.Exception.Message);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(100);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
secondCheckpoint = transferContext.LastCheckpoint;
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
public override object GetTransferObject(DirNode dirNode)
{
throw new InvalidOperationException("Can't get directory transfer object in URI data adaptor.");
}
public abstract object GetTransferObject(string rootPath, DirNode dirNode, StorageCredentials credentials = null);
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options)
{
if (options.DisableSourceCleaner)
{
this.CleanupData(false, true);
}
else
{
this.CleanupData();
}
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
string sourceRootPath = string.Empty;
DirNode sourceRootNode = new DirNode(string.Empty);
if (sourceDataInfo != null)
{
sourceRootPath = sourceDataInfo.RootPath;
sourceRootNode = sourceDataInfo.RootNode;
if (!options.DisableSourceGenerator)
{
SourceAdaptor.GenerateData(sourceDataInfo);
}
}
string destRootPath = string.Empty;
if (options.DestTransferDataInfo != null)
{
destRootPath = options.DestTransferDataInfo.RootPath;
DestAdaptor.GenerateData(options.DestTransferDataInfo);
}
if (options.AfterDataPrepared != null)
{
options.AfterDataPrepared();
}
List <TransferItem> allItems = new List <TransferItem>();
if (options.IsDirectoryTransfer)
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceRootPath, sourceRootNode, options.SourceCredentials),
DestObject = DestAdaptor.GetTransferObject(destRootPath, sourceRootNode, options.DestCredentials),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
IsDirectoryTransfer = true,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(null, item);
}
allItems.Add(item);
}
else
{
foreach (var fileNode in sourceDataInfo.EnumerateFileNodes())
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, fileNode, options.SourceCredentials),
DestObject = DestAdaptor.GetTransferObject(destRootPath, fileNode, options.DestCredentials),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(fileNode, item);
}
allItems.Add(item);
}
}
return(this.RunTransferItems(allItems, options));
}
public override object GetTransferObject(DirNode dirNode)
{
return(this.GetCloudFileDirReference(dirNode));
}
public override object GetTransferObject(string rootPath, DirNode dirNode, StorageCredentials credentials = null)
{
throw new InvalidOperationException("Can't get directory transfer object in URI data adaptor.");
}
public override object GetTransferObject(string rootPath, DirNode dirNode, StorageCredentials credentials = null)
{
return(this.GetCloudFileDirReference(rootPath, dirNode, credentials));
}
public void TestDirectoryResume()
{
int bigFileSizeInKB = 5 * 1024; // 5 MB
int smallFileSizeInKB = 1; // 1 KB
int bigFileNum = 5;
int smallFileNum = 50;
long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
int totalFileNum = bigFileNum + smallFileNum;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
options.IsDirectoryTransfer = true;
using (Stream journalStream = new MemoryStream())
{
bool IsStreamJournal = random.Next(0, 2) == 0;
var transferContext = IsStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext();
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Helper.VerifyCancelException(e.Exception);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
dirOptions.PreserveSMBAttributes = true;
dirOptions.PreserveSMBPermissions = true;
}
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
if (!IsStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = null;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (!IsStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
Helper.CompareSMBPermissions(
sourceDataInfo.RootNode,
result.DataInfo.RootNode,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
}
}
public override object GetTransferObject(string rootPath, DirNode dirNode)
{
return(this.GetCloudBlobDirReference(rootPath, dirNode));
}
public static bool Equals(DirNode dirNodeA, DirNode dirNodeB)
{
// The same node
if (dirNodeA == dirNodeB)
{
return(true);
}
// Empty node equals to null
if ((dirNodeA == null || dirNodeA.IsEmpty) &&
(dirNodeB == null || dirNodeB.IsEmpty))
{
return(true);
}
// Compare two nodes
if (null != dirNodeA && null != dirNodeB)
{
if (dirNodeA.FileNodeCount != dirNodeB.FileNodeCount ||
dirNodeA.NonEmptyDirNodeCount != dirNodeB.NonEmptyDirNodeCount)
{
return(false);
}
if ((null != dirNodeA.Metadata) && (dirNodeA.Metadata.Count > 0))
{
if (null == dirNodeB.Metadata)
{
return(false);
}
if (dirNodeA.Metadata.Count != dirNodeB.Metadata.Count)
{
return(false);
}
foreach (var keyValue in dirNodeA.Metadata)
{
if (!string.Equals(dirNodeB.Metadata[keyValue.Key], keyValue.Value))
{
return(false);
}
}
}
else
{
if ((null != dirNodeB.Metadata) && (dirNodeB.Metadata.Count > 0))
{
return(false);
}
}
foreach (FileNode fileNodeA in dirNodeA.FileNodes)
{
FileNode fileNodeB = dirNodeB.GetFileNode(fileNodeA.Name);
FileNode fileNodeAA = fileNodeA;
if (null == fileNodeB)
{
fileNodeB = dirNodeB.GetFileNode(DMLibTestHelper.EscapeInvalidCharacters(fileNodeA.Name));
if (null != fileNodeB)
{
fileNodeAA = fileNodeA.Clone(DMLibTestHelper.EscapeInvalidCharacters(fileNodeA.Name));
}
}
if (!DMLibDataHelper.Equals(fileNodeAA, fileNodeB))
{
return(false);
}
}
foreach (DirNode subDirNodeA in dirNodeA.DirNodes)
{
Test.Info("Verifying subfolder: {0} ", subDirNodeA.Name);
DirNode subDirNodeB = dirNodeB.GetDirNode(subDirNodeA.Name);
if (null == subDirNodeB)
{
subDirNodeB = dirNodeB.GetDirNode(DMLibTestHelper.EscapeInvalidCharacters(subDirNodeA.Name));
}
if (!DMLibDataHelper.Equals(subDirNodeA, subDirNodeB))
{
return(false);
}
}
return(true);
}
return(false);
}