public static void AddOneFileInBytes(DirNode dirNode, string fileName, long fileSizeInB, FileAttributes? fa = null, DateTime? lmt = null) { FileNode fileNode = new FileNode(fileName) { SizeInByte = fileSizeInB, FileAttr = fa, LastModifiedTime = lmt, }; dirNode.AddFileNode(fileNode); }
public static void AddMultipleFilesDifferentSize(DirNode dirNode, string filePrefix, int[] fileSizes) { for (int i = 0; i < fileSizes.Length; ++i) { FileNode fileNode = new FileNode(filePrefix + "_" + i) { SizeInByte = fileSizes[i] * 1024 }; dirNode.AddFileNode(fileNode); } }
private void TestDelimiter(char delimiter) { Test.Info("Test delimiter: {0}", delimiter); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); string fileName = DMLibTestBase.FolderName + delimiter + DMLibTestBase.FolderName + delimiter + DMLibTestBase.FileName; DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, fileName, 1); TransferContext context = new DirectoryTransferContext(); context.FileFailed += (sender, e) => { Test.Info(e.Exception.StackTrace); }; TestExecutionOptions <DMLibDataInfo> options = new TestExecutionOptions <DMLibDataInfo>() { IsDirectoryTransfer = true, TransferItemModifier = (node, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; dirOptions.Delimiter = delimiter; item.Options = dirOptions; item.TransferContext = context; } }; TestResult <DMLibDataInfo> result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); DirNode dirNode1 = new DirNode(FolderName); DirNode dirNode2 = new DirNode(FolderName); FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName).Clone(DMLibTestBase.FileName); dirNode2.AddFileNode(fileNode); dirNode1.AddDirNode(dirNode2); expectedDataInfo.RootNode.AddDirNode(dirNode1); VerificationHelper.VerifySingleTransferStatus(result, 1, 0, 0, null); VerificationHelper.VerifyTransferSucceed(result, expectedDataInfo); }
private static void PrepareSourceData() { DMLibDataInfo sourceFileTree = new DMLibDataInfo(string.Empty); DirNode dirNode1 = new DirNode("folder1"); DirNode subDir1 = new DirNode("subfolder1"); subDir1.AddDirNode(new DirNode("subfolder3")); subDir1.AddFileNode(GenerateFileNode("testfile2")); subDir1.AddFileNode(GenerateFileNode("4testfile")); dirNode1.AddDirNode(subDir1); DirNode subDir2 = new DirNode("subfolder2"); DirNode subDir4 = new DirNode("subfolder4"); subDir4.AddFileNode(GenerateFileNode("test5")); subDir2.AddDirNode(subDir4); subDir2.AddFileNode(GenerateFileNode("TESTFILE345")); subDir2.AddFileNode(GenerateFileNode("testYfile")); subDir2.AddFileNode(GenerateFileNode("f_arbitrary.exe")); subDir2.AddFileNode(GenerateFileNode("测试x文件")); dirNode1.AddDirNode(subDir2); dirNode1.AddFileNode(GenerateFileNode("testfile1")); dirNode1.AddFileNode(GenerateFileNode("TestFile2")); dirNode1.AddFileNode(GenerateFileNode("测试文件2")); sourceFileTree.RootNode.AddDirNode(dirNode1); DirNode dirNode2 = new DirNode("folder2"); dirNode2.AddFileNode(GenerateFileNode("folder_file")); dirNode2.AddDirNode(new DirNode("testfile1")); dirNode2.AddFileNode(GenerateFileNode("测试文件三")); dirNode2.AddFileNode(GenerateFileNode("测试四文件")); sourceFileTree.RootNode.AddDirNode(dirNode2); DirNode dirNode3 = new DirNode("folder3"); sourceFileTree.RootNode.AddDirNode(dirNode3); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile1")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile2")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXfile")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXXfile")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("测试文件")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("..a123")); DMLibDataInfo blobSourceFileTree = sourceFileTree.Clone(); blobSourceFileTree.RootNode.AddFileNode(GenerateFileNode("TeSTfIle")); Test.Info("Start to generate test data, will take a while..."); foreach (DMLibDataType dataType in sourceDataTypes) { if (IsCloudBlob(dataType)) { PrepareSourceData(dataType, blobSourceFileTree.Clone()); } else { PrepareSourceData(dataType, sourceFileTree.Clone()); } } Test.Info("Done"); }
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5) { long fileSize = 5 * 1024; int fileCountMulti = 32; long totalSize = fileSize * 4 * fileCountMulti; string wrongMD5 = "wrongMD5"; string wrongMD5File = "wrongMD5File"; string correctMD5File = "correctMD5File"; // Prepare data for transfer items with checkMD5 DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; checkMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); SourceAdaptor.GenerateData(sourceDataInfo); DestAdaptor.Cleanup(); TransferEventChecker eventChecker = new TransferEventChecker(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); using (var resumeStream = new MemoryStream()) { TransferContext context = new DirectoryTransferContext(resumeStream); eventChecker.Apply(context); ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti, totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = !checkMD5, Recursive = true, }, CancellationToken = cancellationTokenSource.Token, }; var executionOption = new TestExecutionOptions <DMLibDataInfo>(); executionOption.AfterAllItemAdded = () => { // Wait until there are data transferred if (!progressChecker.DataTransferred.WaitOne(30000)) { Test.Error("No progress in 30s."); } // Cancel the transfer and store the second checkpoint cancellationTokenSource.Cancel(); }; executionOption.LimitSpeed = true; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item }, executionOption); if (null != testResult.Exceptions) { foreach (var exception in testResult.Exceptions) { Test.Info("Got exception during transferring. {0}", exception); } } eventChecker = new TransferEventChecker(); resumeStream.Position = 0; context = new DirectoryTransferContext(resumeStream); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(wrongMD5File); } transferExceptions.Add(args.Exception); }; progressChecker.Reset(); context.ProgressHandler = progressChecker.GetProgressHandler(); checkMD5Item = checkMD5Item.Clone(); checkMD5Item.TransferContext = context; testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DMLibDataInfo actualDataInfo = testResult.DataInfo; for (int i = 0; i < fileCountMulti; ++i) { expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); } Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected."); VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0); if (checkMD5) { if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) { Test.Error("Expect one exception but actually no exception is thrown."); } else { for (int i = 0; i < fileCountMulti; ++i) { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } } else { Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5"); } } }
public void TestDirectoryCheckContentMD5Resume() { long fileSize = 5 * 1024; int fileCountMulti = 32; long totalSize = fileSize * 4 * fileCountMulti; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; // Prepare data for transfer items with checkMD5 DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; checkMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); // Prepare data for transfer items with disabling MD5 check DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; notCheckMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); TransferContext context = new DirectoryTransferContext(); eventChecker.Apply(context); ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, CancellationToken = cancellationTokenSource.Token, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, CancellationToken = cancellationTokenSource.Token }; var executionOption = new TestExecutionOptions <DMLibDataInfo>(); executionOption.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Cancel the transfer and store the second checkpoint cancellationTokenSource.Cancel(); }; executionOption.LimitSpeed = true; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, executionOption); eventChecker = new TransferEventChecker(); context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint)); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } transferExceptions.Add(args.Exception); }; progressChecker.Reset(); context.ProgressHandler = progressChecker.GetProgressHandler(); checkMD5Item = checkMD5Item.Clone(); notCheckMD5Item = notCheckMD5Item.Clone(); checkMD5Item.TransferContext = context; notCheckMD5Item.TransferContext = context; testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DMLibDataInfo actualDataInfo = testResult.DataInfo; for (int i = 0; i < fileCountMulti; ++i) { expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); } Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti); if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) { Test.Error("Expect one exception but actually no exception is thrown."); } else { for (int i = 0; i < fileCountMulti; ++i) { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } }
public static void AddTree(DirNode dirNode, string dirPrefix, string filePrefix, int width, int depth, int fileSizeInKB, FileAttributes? fa = null, DateTime? lmt = null) { for (int i = 0; i < width; ++i) { string fileName = i == 0 ? filePrefix : filePrefix + "_" + i; FileNode fileNode = new FileNode(fileName) { SizeInByte = 1024L * fileSizeInKB, FileAttr = fa, LastModifiedTime = lmt, }; dirNode.AddFileNode(fileNode); } if (depth > 0) { for (int i = 0; i < width; ++i) { string dirName = i == 0 ? dirPrefix : dirPrefix + "_" + i; DirNode subDirNode = dirNode.GetDirNode(dirName); if (subDirNode == null) { subDirNode = new DirNode(dirName); dirNode.AddDirNode(subDirNode); } DMLibDataHelper.AddTree(subDirNode, dirPrefix, filePrefix, width, depth - 1, fileSizeInKB, fa, lmt: lmt); } } }