public static void AddOneFileInBytes(DirNode dirNode, string fileName, long fileSizeInB, FileAttributes? fa = null, DateTime? lmt = null)
{
FileNode fileNode = new FileNode(fileName)
{
SizeInByte = fileSizeInB,
FileAttr = fa,
LastModifiedTime = lmt,
};
dirNode.AddFileNode(fileNode);
}
public static void AddMultipleFilesDifferentSize(DirNode dirNode, string filePrefix, int[] fileSizes)
{
for (int i = 0; i < fileSizes.Length; ++i)
{
FileNode fileNode = new FileNode(filePrefix + "_" + i)
{
SizeInByte = fileSizes[i] * 1024
};
dirNode.AddFileNode(fileNode);
}
}
private void TestDelimiter(char delimiter)
{
Test.Info("Test delimiter: {0}", delimiter);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
string fileName = DMLibTestBase.FolderName + delimiter + DMLibTestBase.FolderName + delimiter + DMLibTestBase.FileName;
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, fileName, 1);
TransferContext context = new DirectoryTransferContext();
context.FileFailed += (sender, e) =>
{
Test.Info(e.Exception.StackTrace);
};
TestExecutionOptions <DMLibDataInfo> options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (node, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
dirOptions.Delimiter = delimiter;
item.Options = dirOptions;
item.TransferContext = context;
}
};
TestResult <DMLibDataInfo> result = this.ExecuteTestCase(sourceDataInfo, options);
DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty);
DirNode dirNode1 = new DirNode(FolderName);
DirNode dirNode2 = new DirNode(FolderName);
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName).Clone(DMLibTestBase.FileName);
dirNode2.AddFileNode(fileNode);
dirNode1.AddDirNode(dirNode2);
expectedDataInfo.RootNode.AddDirNode(dirNode1);
VerificationHelper.VerifySingleTransferStatus(result, 1, 0, 0, null);
VerificationHelper.VerifyTransferSucceed(result, expectedDataInfo);
}
private static void PrepareSourceData()
{
DMLibDataInfo sourceFileTree = new DMLibDataInfo(string.Empty);
DirNode dirNode1 = new DirNode("folder1");
DirNode subDir1 = new DirNode("subfolder1");
subDir1.AddDirNode(new DirNode("subfolder3"));
subDir1.AddFileNode(GenerateFileNode("testfile2"));
subDir1.AddFileNode(GenerateFileNode("4testfile"));
dirNode1.AddDirNode(subDir1);
DirNode subDir2 = new DirNode("subfolder2");
DirNode subDir4 = new DirNode("subfolder4");
subDir4.AddFileNode(GenerateFileNode("test5"));
subDir2.AddDirNode(subDir4);
subDir2.AddFileNode(GenerateFileNode("TESTFILE345"));
subDir2.AddFileNode(GenerateFileNode("testYfile"));
subDir2.AddFileNode(GenerateFileNode("f_arbitrary.exe"));
subDir2.AddFileNode(GenerateFileNode("测试x文件"));
dirNode1.AddDirNode(subDir2);
dirNode1.AddFileNode(GenerateFileNode("testfile1"));
dirNode1.AddFileNode(GenerateFileNode("TestFile2"));
dirNode1.AddFileNode(GenerateFileNode("测试文件2"));
sourceFileTree.RootNode.AddDirNode(dirNode1);
DirNode dirNode2 = new DirNode("folder2");
dirNode2.AddFileNode(GenerateFileNode("folder_file"));
dirNode2.AddDirNode(new DirNode("testfile1"));
dirNode2.AddFileNode(GenerateFileNode("测试文件三"));
dirNode2.AddFileNode(GenerateFileNode("测试四文件"));
sourceFileTree.RootNode.AddDirNode(dirNode2);
DirNode dirNode3 = new DirNode("folder3");
sourceFileTree.RootNode.AddDirNode(dirNode3);
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile1"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile2"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXfile"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXXfile"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("测试文件"));
sourceFileTree.RootNode.AddFileNode(GenerateFileNode("..a123"));
DMLibDataInfo blobSourceFileTree = sourceFileTree.Clone();
blobSourceFileTree.RootNode.AddFileNode(GenerateFileNode("TeSTfIle"));
Test.Info("Start to generate test data, will take a while...");
foreach (DMLibDataType dataType in sourceDataTypes)
{
if (IsCloudBlob(dataType))
{
PrepareSourceData(dataType, blobSourceFileTree.Clone());
}
else
{
PrepareSourceData(dataType, sourceFileTree.Clone());
}
}
Test.Info("Done");
}
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5)
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string wrongMD5File = "wrongMD5File";
string correctMD5File = "correctMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
DestAdaptor.Cleanup();
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
using (var resumeStream = new MemoryStream())
{
TransferContext context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti,
totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = !checkMD5,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, executionOption);
if (null != testResult.Exceptions)
{
foreach (var exception in testResult.Exceptions)
{
Test.Info("Got exception during transferring. {0}", exception);
}
}
eventChecker = new TransferEventChecker();
resumeStream.Position = 0;
context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(wrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
checkMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected.");
VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0);
if (checkMD5)
{
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
else
{
Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5");
}
}
}
public void TestDirectoryCheckContentMD5Resume()
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
// Prepare data for transfer items with disabling MD5 check
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
notCheckMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
TransferContext context = new DirectoryTransferContext();
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, executionOption);
eventChecker = new TransferEventChecker();
context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint));
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
notCheckMD5Item = notCheckMD5Item.Clone();
checkMD5Item.TransferContext = context;
notCheckMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
public static void AddTree(DirNode dirNode, string dirPrefix, string filePrefix, int width, int depth, int fileSizeInKB, FileAttributes? fa = null, DateTime? lmt = null)
{
for (int i = 0; i < width; ++i)
{
string fileName = i == 0 ? filePrefix : filePrefix + "_" + i;
FileNode fileNode = new FileNode(fileName)
{
SizeInByte = 1024L * fileSizeInKB,
FileAttr = fa,
LastModifiedTime = lmt,
};
dirNode.AddFileNode(fileNode);
}
if (depth > 0)
{
for (int i = 0; i < width; ++i)
{
string dirName = i == 0 ? dirPrefix : dirPrefix + "_" + i;
DirNode subDirNode = dirNode.GetDirNode(dirName);
if (subDirNode == null)
{
subDirNode = new DirNode(dirName);
dirNode.AddDirNode(subDirNode);
}
DMLibDataHelper.AddTree(subDirNode, dirPrefix, filePrefix, width, depth - 1, fileSizeInKB, fa, lmt: lmt);
}
}
}
