public void ValidateDestinationMD5ByDownloading(DMLibDataInfo destDataInfo, TestExecutionOptions <DMLibDataInfo> options) { foreach (var fileNode in destDataInfo.EnumerateFileNodes()) { var dest = DestAdaptor.GetTransferObject(destDataInfo.RootPath, fileNode, options.DestCredentials); DestAdaptor.ValidateMD5ByDownloading(dest); } }
public void Upload_Symlinked_RootDir() { #if DNXCORE50 if (!CrossPlatformHelpers.IsLinux) { return; } DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); DMLibDataInfo targetDataInfo = new DMLibDataInfo("target"); UnicodeFileName = "TempTestName"; DirNode dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}"); dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}") { SizeInByte = 1024 }); targetDataInfo.RootNode.AddDirNode(dirNode); SourceAdaptor.CreateIfNotExists(); DestAdaptor.CreateIfNotExists(); SourceAdaptor.GenerateData(targetDataInfo); sourceDataInfo.RootNode = DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", "target", targetDataInfo.RootNode); SourceAdaptor.GenerateData(sourceDataInfo); TransferItem item = new TransferItem() { SourceObject = Path.Combine(SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as string, sourceDataInfo.RootNode.Name), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), TransferContext = new DirectoryTransferContext(), Options = DefaultTransferDirectoryOptions }; (item.Options as UploadDirectoryOptions).Recursive = true; var result = this.RunTransferItems(new List <TransferItem>() { item }, new TestExecutionOptions <DMLibDataInfo>()); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); #endif }
public void TestFileShareSnapshotsDest() { string failError1 = "Failed to validate destination"; string failError2 = "Cannot perform this operation on a share representing a snapshot."; //Prepare Data string snapshotFile = "snapshotFile"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, snapshotFile, 1024); SourceAdaptor.GenerateData(sourceDataInfo); DMLibDataInfo DestDataInfo = new DMLibDataInfo(string.Empty); DestDataInfo.IsFileShareSnapshot = true; DestAdaptor.GenerateData(DestDataInfo); CloudBlobDirectory SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as CloudBlobDirectory; CloudFileDirectory DestObject = DestAdaptor.GetTransferObject(DestDataInfo.RootPath, DestDataInfo.RootNode) as CloudFileDirectory; // transfer File and failed with expected Error Task <TransferStatus> task = TransferManager.CopyDirectoryAsync( SourceObject, DestObject, DMLibTestContext.IsAsync, new CopyDirectoryOptions() { Recursive = true }, new DirectoryTransferContext() { ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite }); try { task.Wait(15 * 60 * 100); } catch (Exception e) { Test.Assert(e.InnerException.Message.Contains(failError1), "Tansfer Exception should contain:" + failError1); Test.Assert(e.InnerException.InnerException.Message.Contains(failError2), "Tansfer Exception should contain:" + failError2); } }
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options) { this.CleanupData(); SourceAdaptor.CreateIfNotExists(); DestAdaptor.CreateIfNotExists(); if (sourceDataInfo != null) { SourceAdaptor.GenerateData(sourceDataInfo); } if (options.DestTransferDataInfo != null) { DestAdaptor.GenerateData(options.DestTransferDataInfo); } if (options.AfterDataPrepared != null) { options.AfterDataPrepared(); } List <TransferItem> allItems = new List <TransferItem>(); foreach (var fileNode in sourceDataInfo.EnumerateFileNodes()) { TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(fileNode), DestObject = DestAdaptor.GetTransferObject(fileNode), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, }; if (options.TransferItemModifier != null) { options.TransferItemModifier(fileNode, item); } allItems.Add(item); } return(this.RunTransferItems(allItems, options)); }
private void TestSASTokenOfEachVersion(string targetSASVersion, bool isDirectoryTransfer) { Test.Info("Testing version of {0}", targetSASVersion); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = isDirectoryTransfer; options.TransferItemModifier = (fileNode, transferItem) => { dynamic transferOptions = isDirectoryTransfer ? DefaultTransferDirectoryOptions : DefaultTransferOptions; transferItem.Options = transferOptions; if (isDirectoryTransfer) { transferItem.TransferContext = new DirectoryTransferContext(); transferItem.TransferContext.FileFailed += (source, e) => { Test.Error(e.Exception.ToString()); }; DirectoryOptions dirOptions = transferItem.Options as DirectoryOptions; dirOptions.Recursive = true; } else { transferItem.TransferContext = new SingleTransferContext(); } transferItem.TransferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite; }; string sourceSAS = null; string destSAS = null; switch (DMLibTestContext.SourceType) { case DMLibDataType.CloudBlob: case DMLibDataType.AppendBlob: case DMLibDataType.BlockBlob: case DMLibDataType.PageBlob: if ((DMLibTestContext.SourceType == DMLibDataType.AppendBlob) && (string.CompareOrdinal(targetSASVersion, "2015-04-05") < 0)) { break; } SourceAdaptor.CreateIfNotExists(); CloudBlobDataAdaptor blobAdaptor = SourceAdaptor as CloudBlobDataAdaptor; sourceSAS = Util.SASGenerator.GetSharedAccessSignature(blobAdaptor.GetBaseContainer(), new SharedAccessBlobPolicy { Permissions = isDirectoryTransfer ? SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List : SharedAccessBlobPermissions.Read, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; case DMLibDataType.CloudFile: if (string.CompareOrdinal(targetSASVersion, "2015-02-21") < 0) { break; } SourceAdaptor.CreateIfNotExists(); CloudFileDataAdaptor fileAdaptor = SourceAdaptor as CloudFileDataAdaptor; sourceSAS = Util.SASGenerator.GetSharedAccessSignature( fileAdaptor.GetBaseShare(), new SharedAccessFilePolicy { Permissions = isDirectoryTransfer ? SharedAccessFilePermissions.List | SharedAccessFilePermissions.Read : SharedAccessFilePermissions.Read, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; default: break; } if (!DMLibTestContext.IsAsync || (string.CompareOrdinal(targetSASVersion, "2014-02-14") >= 0)) { switch (DMLibTestContext.DestType) { case DMLibDataType.CloudBlob: case DMLibDataType.AppendBlob: case DMLibDataType.BlockBlob: case DMLibDataType.PageBlob: if ((DMLibTestContext.DestType == DMLibDataType.AppendBlob) && (string.CompareOrdinal(targetSASVersion, "2015-04-05") < 0)) { break; } DestAdaptor.CreateIfNotExists(); CloudBlobDataAdaptor blobAdaptor = DestAdaptor as CloudBlobDataAdaptor; destSAS = Util.SASGenerator.GetSharedAccessSignature(blobAdaptor.GetBaseContainer(), new SharedAccessBlobPolicy { Permissions = DMLibTestContext.IsAsync ? SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read : SharedAccessBlobPermissions.Write, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; case DMLibDataType.CloudFile: if (string.CompareOrdinal(targetSASVersion, "2015-02-21") < 0) { break; } DestAdaptor.CreateIfNotExists(); CloudFileDataAdaptor fileAdaptor = DestAdaptor as CloudFileDataAdaptor; destSAS = Util.SASGenerator.GetSharedAccessSignature( fileAdaptor.GetBaseShare(), new SharedAccessFilePolicy { Permissions = DMLibTestContext.IsAsync ? SharedAccessFilePermissions.Write | SharedAccessFilePermissions.Read : SharedAccessFilePermissions.Write, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; default: break; } } if (null != sourceSAS) { options.SourceCredentials = new StorageCredentials(sourceSAS); } if (null != destSAS) { options.DestCredentials = new StorageCredentials(destSAS); } var result = this.ExecuteTestCase(sourceDataInfo, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); }
public void TestDirectoryPreserveSMBProperties() { if (!CrossPlatformHelpers.IsWindows) { return; } CloudFileNtfsAttributes[] SMBFileAttributes = { CloudFileNtfsAttributes.ReadOnly, CloudFileNtfsAttributes.Hidden, #if DEBUG CloudFileNtfsAttributes.System, CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Normal, CloudFileNtfsAttributes.Offline, CloudFileNtfsAttributes.NotContentIndexed, CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden | CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData #endif }; for (int i = 0; i < SMBFileAttributes.Length; ++i) { Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]); DirectoryTransferContext dirTransferContext = new DirectoryTransferContext(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = dirTransferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; transferItem.Options = transferOptions; }; #if DEBUG TestHookCallbacks.GetFileAttributesCallback = (path) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { return(FileAttributes.Directory); } else { return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory)); } } else { return(Helper.ToFileAttributes(SMBFileAttributes[i])); } }; TestHookCallbacks.SetFileAttributesCallback = (path, attributes) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot")) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory), "Directory attributes should be expected {0}", attributes); } } } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]), "File attributes should be expected {0}", attributes); } }; #endif try { // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), IsDirectoryTransfer = true, Options = DefaultTransferDirectoryOptions }; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; item.Options = transferOptions; item.TransferContext = new DirectoryTransferContext(); item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) => { return(false); }; List <TransferItem> items = new List <TransferItem>(); items.Add(item); result = this.RunTransferItems(items, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.Local) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false); } else { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); } } finally { #if DEBUG TestHookCallbacks.GetFileAttributesCallback = null; TestHookCallbacks.SetFileAttributesCallback = null; #endif } } }
public void TestDirectoryPreserveSMBPropertiesResume() { if (!CrossPlatformHelpers.IsWindows) { return; } CloudFileNtfsAttributes[] SMBFileAttributes = { CloudFileNtfsAttributes.ReadOnly, CloudFileNtfsAttributes.Hidden, #if DEBUG CloudFileNtfsAttributes.System, CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Normal, CloudFileNtfsAttributes.Offline, CloudFileNtfsAttributes.NotContentIndexed, CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden | CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData #endif }; for (int i = 0; i < SMBFileAttributes.Length; ++i) { CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; bool IsStreamJournal = random.Next(0, 2) == 0; using (Stream journalStream = new MemoryStream()) { Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]); DirectoryTransferContext dirTransferContext = null; if (IsStreamJournal) { dirTransferContext = new DirectoryTransferContext(journalStream); } else { dirTransferContext = new DirectoryTransferContext(); } var progressChecker = new ProgressChecker(14, 14 * 1024); dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, item) => { item.TransferContext = dirTransferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; item.Options = transferOptions; item.CancellationToken = tokenSource.Token; transferItem = item; }; TransferCheckpoint checkpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!IsStreamJournal) { checkpoint = dirTransferContext.LastCheckpoint; } // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; #if DEBUG TestHookCallbacks.GetFileAttributesCallback = (path) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { return(FileAttributes.Directory); } else { return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory)); } } else { return(Helper.ToFileAttributes(SMBFileAttributes[i])); } }; TestHookCallbacks.SetFileAttributesCallback = (path, attributes) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot")) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory), "Directory attributes should be expected {0}", attributes); } } } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]), "File attributes should be expected {0}", attributes); } }; #endif try { // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; Helper.VerifyCancelException(exception); TransferItem resumeItem = transferItem.Clone(); DirectoryTransferContext resumeContext = null; journalStream.Position = 0; if (IsStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream); } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint)); } resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), IsDirectoryTransfer = true, Options = DefaultTransferDirectoryOptions }; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; item.Options = transferOptions; item.TransferContext = new DirectoryTransferContext(); item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) => { return(false); }; List <TransferItem> items = new List <TransferItem>(); items.Add(item); result = this.RunTransferItems(items, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.Local) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false); } else { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); } } finally { #if DEBUG TestHookCallbacks.GetFileAttributesCallback = null; TestHookCallbacks.SetFileAttributesCallback = null; #endif } } } }
public void TestDirectoryCheckContentMD5() { long fileSize = 5 * 1024 * 1024; long totalSize = fileSize * 4; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File); tmpFileNode.MD5 = wrongMD5; tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File); tmpFileNode.MD5 = wrongMD5; SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); TransferContext context = new TransferContext(); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } }; ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); context.FileFailed += (eventSource, eventArgs) => { transferExceptions.Add(eventArgs.Exception); }; TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, }; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); DMLibDataInfo actualDataInfo = testResult.DataInfo; actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1); if (testResult.Exceptions.Count != 0 || transferExceptions.Count != 1) { Test.Error("Expect one exception but actually no exception is thrown."); } else { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } }
public void DirectoryForceOverwriteTest() { string destExistName = "destExist"; string destNotExistName = "destNotExist"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistName, 1024); TransferContext transferContext = new DirectoryTransferContext(); transferContext.ShouldOverwriteCallback = TransferContext.ForceOverwrite; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } if (IsCloudService(DMLibTestContext.DestType)) { SharedAccessPermissions permissions; if (DMLibTestContext.IsAsync) { permissions = SharedAccessPermissions.Write | SharedAccessPermissions.Read; } else { permissions = SharedAccessPermissions.Write; } StorageCredentials destSAS = new StorageCredentials(DestAdaptor.GenerateSAS(permissions, (int)new TimeSpan(1, 0, 0, 0).TotalSeconds)); options.DestCredentials = destSAS; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // Verify transfer result Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); VerificationHelper.VerifySingleTransferStatus(result, 2, 0, 0, 1024 * 2); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 0, "Verify skipped transfer"); }
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5) { long fileSize = 5 * 1024; int fileCountMulti = 32; long totalSize = fileSize * 4 * fileCountMulti; string wrongMD5 = "wrongMD5"; string wrongMD5File = "wrongMD5File"; string correctMD5File = "correctMD5File"; // Prepare data for transfer items with checkMD5 DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; checkMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); SourceAdaptor.GenerateData(sourceDataInfo); DestAdaptor.Cleanup(); TransferEventChecker eventChecker = new TransferEventChecker(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); using (var resumeStream = new MemoryStream()) { TransferContext context = new DirectoryTransferContext(resumeStream); eventChecker.Apply(context); ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti, totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = !checkMD5, Recursive = true, }, CancellationToken = cancellationTokenSource.Token, }; var executionOption = new TestExecutionOptions <DMLibDataInfo>(); executionOption.AfterAllItemAdded = () => { // Wait until there are data transferred if (!progressChecker.DataTransferred.WaitOne(30000)) { Test.Error("No progress in 30s."); } // Cancel the transfer and store the second checkpoint cancellationTokenSource.Cancel(); }; executionOption.LimitSpeed = true; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item }, executionOption); if (null != testResult.Exceptions) { foreach (var exception in testResult.Exceptions) { Test.Info("Got exception during transferring. {0}", exception); } } eventChecker = new TransferEventChecker(); resumeStream.Position = 0; context = new DirectoryTransferContext(resumeStream); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(wrongMD5File); } transferExceptions.Add(args.Exception); }; progressChecker.Reset(); context.ProgressHandler = progressChecker.GetProgressHandler(); checkMD5Item = checkMD5Item.Clone(); checkMD5Item.TransferContext = context; testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DMLibDataInfo actualDataInfo = testResult.DataInfo; for (int i = 0; i < fileCountMulti; ++i) { expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); } Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected."); VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0); if (checkMD5) { if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) { Test.Error("Expect one exception but actually no exception is thrown."); } else { for (int i = 0; i < fileCountMulti; ++i) { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } } else { Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5"); } } }
public void TestDirectoryCheckContentMD5Resume() { long fileSize = 5 * 1024; int fileCountMulti = 32; long totalSize = fileSize * 4 * fileCountMulti; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; // Prepare data for transfer items with checkMD5 DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; checkMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); // Prepare data for transfer items with disabling MD5 check DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; notCheckMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); TransferContext context = new DirectoryTransferContext(); eventChecker.Apply(context); ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, CancellationToken = cancellationTokenSource.Token, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, CancellationToken = cancellationTokenSource.Token }; var executionOption = new TestExecutionOptions <DMLibDataInfo>(); executionOption.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Cancel the transfer and store the second checkpoint cancellationTokenSource.Cancel(); }; executionOption.LimitSpeed = true; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, executionOption); eventChecker = new TransferEventChecker(); context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint)); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } transferExceptions.Add(args.Exception); }; progressChecker.Reset(); context.ProgressHandler = progressChecker.GetProgressHandler(); checkMD5Item = checkMD5Item.Clone(); notCheckMD5Item = notCheckMD5Item.Clone(); checkMD5Item.TransferContext = context; notCheckMD5Item.TransferContext = context; testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DMLibDataInfo actualDataInfo = testResult.DataInfo; for (int i = 0; i < fileCountMulti; ++i) { expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); } Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti); if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) { Test.Error("Expect one exception but actually no exception is thrown."); } else { for (int i = 0; i < fileCountMulti; ++i) { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } }
public void ForceOverwriteTest() { string destExistName = "destExist"; string destNotExistName = "destNotExist"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistName, 1024); TransferContext transferContext = new SingleTransferContext(); transferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } if (IsCloudService(DMLibTestContext.DestType)) { SharedAccessPermissions permissions; if (DMLibTestContext.CopyMethod == DMLibCopyMethod.ServiceSideAsyncCopy) { permissions = SharedAccessPermissions.Write | SharedAccessPermissions.Read; } else { permissions = SharedAccessPermissions.Write; } StorageCredentials destSAS = new StorageCredentials(DestAdaptor.GenerateSAS(permissions, (int)new TimeSpan(1, 0, 0, 0).TotalSeconds)); options.DestCredentials = destSAS; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferOptions; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { transferOptions.PreserveSMBAttributes = true; transferOptions.PreserveSMBPermissions = true; } transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // Verify transfer result Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 0, "Verify skipped transfer"); if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( sourceDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } }
public TestResult <DMLibDataInfo> RunTransferItems(List <TransferItem> items, TestExecutionOptions <DMLibDataInfo> options) { Dictionary <TransferItem, Task <TransferStatus> > allTasks = new Dictionary <TransferItem, Task <TransferStatus> >(); var testResult = new TestResult <DMLibDataInfo>(); testResult.TransferItems = items; try { foreach (TransferItem item in items) { DMLibWrapper wrapper = GetDMLibWrapper(item.SourceType, item.DestType, DMLibTestContext.IsAsync); if (item.BeforeStarted != null) { item.BeforeStarted(); } try { if (options.LimitSpeed) { OperationContext.GlobalSendingRequest += this.LimitSpeed; TransferManager.Configurations.ParallelOperations = DMLibTestConstants.LimitedSpeedNC; } if (options.BlockSize.HasValue) { TransferManager.Configurations.BlockSize = options.BlockSize.Value; } allTasks.Add(item, wrapper.DoTransfer(item)); } catch (Exception e) { testResult.AddException(e); } if (item.AfterStarted != null) { item.AfterStarted(); } } if (options.AfterAllItemAdded != null) { options.AfterAllItemAdded(); } try { Task.WaitAll(allTasks.Values.ToArray(), options.TimeoutInMs); } catch (Exception e) { AggregateException ae = e as AggregateException; if (ae != null) { ae = ae.Flatten(); foreach (var innerE in ae.InnerExceptions) { testResult.AddException(innerE); } } else { testResult.AddException(e); } } } finally { if (options.LimitSpeed) { OperationContext.GlobalSendingRequest -= this.LimitSpeed; TransferManager.Configurations.ParallelOperations = DMLibTestConstants.DefaultNC; } TransferManager.Configurations.BlockSize = DMLibTestConstants.DefaultBlockSize; } Parallel.ForEach(allTasks, pair => { TransferItem transferItem = pair.Key; Task <TransferStatus> task = pair.Value; transferItem.CloseStreamIfNecessary(); try { transferItem.FinalStatus = task.Result; } catch (Exception e) { transferItem.Exception = e; } }); if (!options.DisableDestinationFetch) { testResult.DataInfo = DestAdaptor.GetTransferDataInfo(string.Empty); } foreach (var exception in testResult.Exceptions) { Test.Info("Exception from DMLib: {0}", exception.ToString()); } return(testResult); }
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options) { if (options.DisableSourceCleaner) { this.CleanupData(false, true); } else { this.CleanupData(); } SourceAdaptor.CreateIfNotExists(); DestAdaptor.CreateIfNotExists(); string sourceRootPath = string.Empty; DirNode sourceRootNode = new DirNode(string.Empty); if (sourceDataInfo != null) { sourceRootPath = sourceDataInfo.RootPath; sourceRootNode = sourceDataInfo.RootNode; if (!options.DisableSourceGenerator) { SourceAdaptor.GenerateData(sourceDataInfo); } } string destRootPath = string.Empty; if (options.DestTransferDataInfo != null) { destRootPath = options.DestTransferDataInfo.RootPath; DestAdaptor.GenerateData(options.DestTransferDataInfo); } if (options.AfterDataPrepared != null) { options.AfterDataPrepared(); } List <TransferItem> allItems = new List <TransferItem>(); if (options.IsDirectoryTransfer) { TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceRootPath, sourceRootNode, options.SourceCredentials), DestObject = DestAdaptor.GetTransferObject(destRootPath, sourceRootNode, options.DestCredentials), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, IsDirectoryTransfer = true, }; if (options.TransferItemModifier != null) { options.TransferItemModifier(null, item); } allItems.Add(item); } else { foreach (var fileNode in sourceDataInfo.EnumerateFileNodes()) { TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, fileNode, options.SourceCredentials), DestObject = DestAdaptor.GetTransferObject(destRootPath, fileNode, options.DestCredentials), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, }; if (options.TransferItemModifier != null) { options.TransferItemModifier(fileNode, item); } allItems.Add(item); } } return(this.RunTransferItems(allItems, options)); }