public void TestDirectorySetContentType() { string contentType = "contenttype"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); int[] fileSizes = new int[] {1024, 1024, 1024 }; DMLibDataHelper.AddMultipleFilesDifferentSize(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizes); var options = new TestExecutionOptions<DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, transferItem) => { dynamic uploadOptions = DefaultTransferDirectoryOptions; uploadOptions.ContentType = "contenttype"; transferItem.Options = uploadOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); foreach (FileNode destFileNode in result.DataInfo.RootNode.FileNodes) { Test.Assert(contentType.Equals(destFileNode.ContentType), "Verify content type: {0}, expected {1}", destFileNode.ContentType, contentType); } }
public void TransferDirectoryDifferentSizeObject() { DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder"); DMLibDataHelper.AddMultipleFilesNormalSize(sourceDataInfo.RootNode, BVT.UnicodeFileName); var options = new TestExecutionOptions<DMLibDataInfo>() { IsDirectoryTransfer = true, TransferItemModifier = (notUsed, item) => { dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; item.Options = transferOptions; }, }; var result = this.ExecuteTestCase(sourceDataInfo, options); // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void TestMetadata() { Dictionary<string, string> metadata = new Dictionary<string, string>(); metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random)); metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random)); Test.Info("Metadata is ====================="); foreach (var keyValue in metadata) { Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value); } DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); FileNode fileNode = new FileNode(DMLibTestBase.FileName) { SizeInByte = DMLibTestBase.FileSizeInKB * 1024L, Metadata = metadata }; sourceDataInfo.RootNode.AddFileNode(fileNode); var result = this.ExecuteTestCase(sourceDataInfo, new TestExecutionOptions<DMLibDataInfo>()); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void TransferBigSizeObject() { DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddMultipleFilesBigSize(sourceDataInfo.RootNode, DMLibTestBase.FileName); var result = this.ExecuteTestCase(sourceDataInfo, new TestExecutionOptions<DMLibDataInfo>()); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
private void TestSetAttributesToLocal(bool IsDirectoryTransfer) { DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); FileNode fileNode = new FileNode(DMLibTestBase.FileName) { SizeInByte = DMLibTestBase.FileSizeInKB * 1024L, }; sourceDataInfo.RootNode.AddFileNode(fileNode); TransferContext context; if (IsDirectoryTransfer) { context = new DirectoryTransferContext() { SetAttributesCallbackAsync = async(destObj) => { Test.Error("SetAttributes callback should not be invoked when destination is local"); } }; } else { context = new SingleTransferContext() { SetAttributesCallbackAsync = async(destObj) => { Test.Error("SetAttributes callback should not be invoked when destination is local"); } }; } var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (node, transferItem) => { dynamic transferOptions = IsDirectoryTransfer ? DefaultTransferDirectoryOptions : DefaultTransferOptions; if (IsDirectoryTransfer) { transferOptions.Recursive = true; } transferItem.Options = transferOptions; transferItem.TransferContext = context; }; options.IsDirectoryTransfer = IsDirectoryTransfer; var result = this.ExecuteTestCase(sourceDataInfo, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); }
public void TestFileShareSnapshotsCopyToBase() { int fileCount = 3; string snapshotFile = "snapshotFile"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); sourceDataInfo.IsFileShareSnapshot = true; DMLibDataHelper.AddMultipleFiles(sourceDataInfo.RootNode, snapshotFile, fileCount, 1024); SourceAdaptor.Cleanup(); SourceAdaptor.CreateIfNotExists(); SourceAdaptor.GenerateData(sourceDataInfo); CloudFileDataAdaptor fileAdaptor = (SourceAdaptor as CloudFileDataAdaptor); CloudFileDirectory SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as CloudFileDirectory; CloudFileDirectory DestObject = fileAdaptor.fileHelper.FileClient.GetShareReference(fileAdaptor.ShareName).GetRootDirectoryReference(); // transfer File and finished succssfully Task <TransferStatus> task = TransferManager.CopyDirectoryAsync( SourceObject, DestObject, DMLibTestContext.IsAsync, new CopyDirectoryOptions() { Recursive = true }, new DirectoryTransferContext() { ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite }); Test.Assert(task.Wait(15 * 60 * 100), "Tansfer finished in time."); Test.Assert(task.Result.NumberOfFilesFailed == 0, "No Failed File."); Test.Assert(task.Result.NumberOfFilesSkipped == 0, "No Skipped File."); Test.Assert(task.Result.NumberOfFilesTransferred == fileCount, string.Format("Transferred file :{0} == {1}", task.Result.NumberOfFilesTransferred, fileCount)); // verify that Files in Share Snapshot is transferred IEnumerable <IListFileItem> sourceFiles = SourceObject.ListFilesAndDirectories(HelperConst.DefaultFileOptions); foreach (IListFileItem item in sourceFiles) { if (item is CloudFile) { CloudFile srcFile = item as CloudFile; CloudFile destFile = DestObject.GetFileReference(srcFile.Name); srcFile.FetchAttributes(); destFile.FetchAttributes(); Test.Assert(srcFile.Properties.ContentMD5 == destFile.Properties.ContentMD5, string.Format("File {0} MD5 :{1} == {2}", srcFile.Name, srcFile.Properties.ContentMD5, destFile.Properties.ContentMD5)); } } }
public void TestSetAttributes() { DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); FileNode fileNode = new FileNode(DMLibTestBase.FileName) { SizeInByte = DMLibTestBase.FileSizeInKB * 1024L }; if (DMLibTestContext.SourceType != DMLibDataType.Local) { fileNode.Metadata = new Dictionary <string, string>(); fileNode.Metadata.Add("foo", "bar"); fileNode.ContentLanguage = SetAttributesTest.TestContentLanguage; } sourceDataInfo.RootNode.AddFileNode(fileNode); TransferContext context = new SingleTransferContext() { SetAttributesCallbackAsync = async(destObj) => { dynamic destCloudObj = destObj; destCloudObj.Properties.ContentType = SetAttributesTest.TestContentType; destCloudObj.Metadata.Add("aa", "bb"); } }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (node, transferItem) => { dynamic transferOptions = DefaultTransferOptions; transferItem.Options = transferOptions; transferItem.TransferContext = context; }; var result = this.ExecuteTestCase(sourceDataInfo, options); fileNode.Metadata.Add("aa", "bb"); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); FileNode destFileNode = result.DataInfo.RootNode.GetFileNode(DMLibTestBase.FileName); Test.Assert(TestContentType.Equals(destFileNode.ContentType), "Verify content type: {0}, expected {1}", destFileNode.ContentType, TestContentType); if (DMLibTestContext.SourceType != DMLibDataType.Local) { Test.Assert(SetAttributesTest.TestContentLanguage.Equals(destFileNode.ContentLanguage), "Verify ContentLanguage: {0}, expected {1}", destFileNode.ContentLanguage, SetAttributesTest.TestContentLanguage); } }
private static DMLibDataInfo GetSourceDataInfo(string key) { DMLibDataInfo result; if (!sourceDataInfos.ContainsKey(key)) { result = new DMLibDataInfo(string.Empty); sourceDataInfos.Add(key, result); } return sourceDataInfos[key]; }
public void TestCheckContentMD5() { long fileSize = 10 * 1024 * 1024; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, checkWrongMD5File, fileSize); FileNode tmpFileNode = sourceDataInfo.RootNode.GetFileNode(checkWrongMD5File); tmpFileNode.MD5 = wrongMD5; DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, notCheckWrongMD5File, fileSize); tmpFileNode = sourceDataInfo.RootNode.GetFileNode(notCheckWrongMD5File); tmpFileNode.MD5 = wrongMD5; DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, checkCorrectMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, notCheckCorrectMD5File, fileSize); var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; DownloadOptions downloadOptions = new DownloadOptions(); if (fileName.Equals(checkWrongMD5File) || fileName.Equals(checkCorrectMD5File)) { downloadOptions.DisableContentMD5Validation = false; } else if (fileName.Equals(notCheckWrongMD5File) || fileName.Equals(notCheckCorrectMD5File)) { downloadOptions.DisableContentMD5Validation = true; } transferItem.Options = downloadOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify there's one exception."); Exception exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify it's an invalid operation exception."); VerificationHelper.VerifyExceptionErrorMessage(exception, "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored", checkWrongMD5File); }
// TODO: add more test cases public void TestUnsupportedDirection() { DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); var result = this.ExecuteTestCase(sourceDataInfo, new TestExecutionOptions <DMLibDataInfo>()); Test.Assert(result.Exceptions.Count == 1, "Verify exception is thrown."); Exception exception = result.Exceptions[0]; if (DMLibCopyMethod.ServiceSideSyncCopy == DMLibTestContext.CopyMethod) { Test.Assert(exception is NotSupportedException, "Verify exception is NotSupportedException."); if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { VerificationHelper.VerifyExceptionErrorMessage(exception, "Copying to Azure File Storage with service side synchronous copying is not supported."); } else { VerificationHelper.VerifyExceptionErrorMessage(exception, "Copying from Azure File Storage with service side synchronous copying is not supported."); } } else if (DMLibTestContext.SourceType == DMLibDataType.URI) { Test.Assert(exception is NotSupportedException, "Verify exception is NotSupportedException."); if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { VerificationHelper.VerifyExceptionErrorMessage(exception, "Copying from uri to Azure File Storage synchronously is not supported"); } else { VerificationHelper.VerifyExceptionErrorMessage(exception, "Copying from uri to Azure Blob Storage synchronously is not supported"); } } else if (DMLibTestBase.IsCloudBlob(DMLibTestContext.SourceType) && DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType)) { Test.Assert(exception is InvalidOperationException, "Verify exception is InvalidOperationException."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Blob type of source and destination must be the same."); } else { Test.Assert(exception is InvalidOperationException, "Verify exception is InvalidOperationException."); VerificationHelper.VerifyExceptionErrorMessage(exception, string.Format("Copying from File Storage to {0} Blob Storage asynchronously is not supported.", MapBlobDataTypeToBlobType(DMLibTestContext.DestType))); } Test.Assert(DMLibDataHelper.Equals(new DMLibDataInfo(string.Empty), result.DataInfo), "Verify no file is transfered."); }
public static void VerifySingleObjectResumeResult(TestResult<DMLibDataInfo> result, DMLibDataInfo expectedDataInfo) { if (DMLibTestContext.SourceType != DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); } else { Test.Assert(result.Exceptions.Count == 1, "Verify stream resume is not supported"); Exception exception = result.Exceptions[0]; Test.Assert(exception is NotSupportedException, "Verify stream resume is not supported"); } }
private static void PrepareData(SourceOrDest sourceOrDest, DMLibDataType dataType, DMLibDataInfo dataInfo) { var dataInfos = GetDataInfos(sourceOrDest); CleanupData(sourceOrDest, dataType); if (SourceOrDest.Source == sourceOrDest) { GetSourceAdaptor(dataType).GenerateData(dataInfo); SetSourceDataInfo(dataType, dataInfo); } else { GetDestAdaptor(dataType).GenerateData(dataInfo); SetDestDataInfo(dataType, dataInfo); } }
public void TestCheckContentMD5() { long fileSize = 10 * 1024 * 1024; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, checkWrongMD5File, fileSize); FileNode tmpFileNode = sourceDataInfo.RootNode.GetFileNode(checkWrongMD5File); tmpFileNode.MD5 = wrongMD5; DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, notCheckWrongMD5File, fileSize); tmpFileNode = sourceDataInfo.RootNode.GetFileNode(notCheckWrongMD5File); tmpFileNode.MD5 = wrongMD5; DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, checkCorrectMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, notCheckCorrectMD5File, fileSize); var options = new TestExecutionOptions<DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; DownloadOptions downloadOptions = new DownloadOptions(); if (fileName.Equals(checkWrongMD5File) || fileName.Equals(checkCorrectMD5File)) { downloadOptions.DisableContentMD5Validation = false; } else if (fileName.Equals(notCheckWrongMD5File) || fileName.Equals(notCheckCorrectMD5File)) { downloadOptions.DisableContentMD5Validation = true; } transferItem.Options = downloadOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify there's one exception."); Exception exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify it's an invalid operation exception."); VerificationHelper.VerifyExceptionErrorMessage(exception, "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored", checkWrongMD5File); }
public void TransferDifferentSizeObject() { DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddMultipleFilesNormalSize(sourceDataInfo.RootNode, BVT.UnicodeFileName); var options = new TestExecutionOptions<DMLibDataInfo>(); options.AfterDataPrepared = () => { if ((DMLibTestContext.SourceType == DMLibDataType.CloudFile || DMLibTestContext.SourceType == DMLibDataType.PageBlob) && !DMLibTestContext.IsAsync) { string sparseFileName = "SparseFile"; DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, sparseFileName, 1); FileNode sparseFileNode = sourceDataInfo.RootNode.GetFileNode(sparseFileName); if (DMLibTestContext.SourceType == DMLibDataType.CloudFile) { CloudFileDataAdaptor cloudFileDataAdaptor = SourceAdaptor as CloudFileDataAdaptor; CloudFile sparseCloudFile = cloudFileDataAdaptor.GetCloudFileReference(sparseFileNode); this.PrepareCloudFileWithDifferentSizeRange(sparseCloudFile); sparseFileNode.MD5 = sparseCloudFile.Properties.ContentMD5; sparseFileNode.Metadata = sparseCloudFile.Metadata; } else if (DMLibTestContext.SourceType == DMLibDataType.PageBlob) { CloudBlobDataAdaptor cloudBlobDataAdaptor = SourceAdaptor as CloudBlobDataAdaptor; CloudPageBlob sparsePageBlob = cloudBlobDataAdaptor.GetCloudBlobReference(sparseFileNode) as CloudPageBlob; this.PreparePageBlobWithDifferenSizePage(sparsePageBlob); sparseFileNode.MD5 = sparsePageBlob.Properties.ContentMD5; sparseFileNode.Metadata = sparsePageBlob.Metadata; } } }; var result = this.ExecuteTestCase(sourceDataInfo, options); // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void TestFileShareSnapshotsDest() { string failError1 = "Failed to validate destination"; string failError2 = "Cannot perform this operation on a share representing a snapshot."; //Prepare Data string snapshotFile = "snapshotFile"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, snapshotFile, 1024); SourceAdaptor.GenerateData(sourceDataInfo); DMLibDataInfo DestDataInfo = new DMLibDataInfo(string.Empty); DestDataInfo.IsFileShareSnapshot = true; DestAdaptor.GenerateData(DestDataInfo); CloudBlobDirectory SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as CloudBlobDirectory; CloudFileDirectory DestObject = DestAdaptor.GetTransferObject(DestDataInfo.RootPath, DestDataInfo.RootNode) as CloudFileDirectory; // transfer File and failed with expected Error Task <TransferStatus> task = TransferManager.CopyDirectoryAsync( SourceObject, DestObject, DMLibTestContext.IsAsync, new CopyDirectoryOptions() { Recursive = true }, new DirectoryTransferContext() { ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite }); try { task.Wait(15 * 60 * 100); } catch (Exception e) { Test.Assert(e.InnerException.Message.Contains(failError1), "Tansfer Exception should contain:" + failError1); Test.Assert(e.InnerException.InnerException.Message.Contains(failError2), "Tansfer Exception should contain:" + failError2); } }
private void TestDelimiter(char delimiter) { Test.Info("Test delimiter: {0}", delimiter); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); string fileName = DMLibTestBase.FolderName + delimiter + DMLibTestBase.FolderName + delimiter + DMLibTestBase.FileName; DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, fileName, 1); TransferContext context = new DirectoryTransferContext(); context.FileFailed += (sender, e) => { Test.Info(e.Exception.StackTrace); }; TestExecutionOptions <DMLibDataInfo> options = new TestExecutionOptions <DMLibDataInfo>() { IsDirectoryTransfer = true, TransferItemModifier = (node, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; dirOptions.Delimiter = delimiter; item.Options = dirOptions; item.TransferContext = context; } }; TestResult <DMLibDataInfo> result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); DirNode dirNode1 = new DirNode(FolderName); DirNode dirNode2 = new DirNode(FolderName); FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName).Clone(DMLibTestBase.FileName); dirNode2.AddFileNode(fileNode); dirNode1.AddDirNode(dirNode2); expectedDataInfo.RootNode.AddDirNode(dirNode1); VerificationHelper.VerifySingleTransferStatus(result, 1, 0, 0, null); VerificationHelper.VerifyTransferSucceed(result, expectedDataInfo); }
public void TestDirectoryIncludeSnapshots() { string snapshotFile1 = "snapshotFile1"; string snapshotFile2 = "snapshotFile2"; string snapshotFile3 = "snapshotFile3"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); // the 1st file has 1 snapshot DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile1, 1024); FileNode fileNode1 = sourceDataInfo.RootNode.GetFileNode(snapshotFile1); fileNode1.SnapshotsCount = 1; // the 2nd file has 2 snapshots DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile2, 1024); FileNode fileNode2 = sourceDataInfo.RootNode.GetFileNode(snapshotFile2); fileNode2.SnapshotsCount = 2; // the 3rd file has no snapshot DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile3, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; // transfer with IncludeSnapshots = true options.TransferItemModifier = (fileNode, transferItem) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; dirOptions.IncludeSnapshots = true; transferItem.Options = dirOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // verify that snapshots are transferred Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void UploadLargeObject_Nonseekable_PageBlob_Nagitive() { DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(DMLibDataType.PageBlob); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{1}B") { SizeInByte = 1, }); sourceAdaptor.GenerateData(sourceDataInfo); destAdaptor.CreateIfNotExists(); List <TransferItem> uploadItems = new List <TransferItem>(); string fileName = $"{FileName}_{1}B"; FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName); uploadItems.Add(new TransferItem() { SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, false, true), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = DMLibDataType.Stream, DestType = DMLibDataType.PageBlob, IsServiceCopy = false }); // Execution var result = this.RunTransferItems( uploadItems, new TestExecutionOptions <DMLibDataInfo>() { DisableDestinationFetch = true }); Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("must be a multiple of 512 bytes."), "Verify error is expected."); sourceAdaptor.Cleanup(); destAdaptor.Cleanup(); }
public void TestProgressHandlerTest() { long fileSize = 10 * 1024 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize); var options = new TestExecutionOptions<DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { TransferContext transferContext = new TransferContext(); ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void TestDirectoryIncludeSnapshots() { string snapshotFile1 = "snapshotFile1"; string snapshotFile2 = "snapshotFile2"; string snapshotFile3 = "snapshotFile3"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); // the 1st file has 1 snapshot DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile1, 1024); FileNode fileNode1 = sourceDataInfo.RootNode.GetFileNode(snapshotFile1); fileNode1.SnapshotsCount = 1; // the 2nd file has 2 snapshots DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile2, 1024); FileNode fileNode2 = sourceDataInfo.RootNode.GetFileNode(snapshotFile2); fileNode2.SnapshotsCount = 2; // the 3rd file has no snapshot DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile3, 1024); var options = new TestExecutionOptions<DMLibDataInfo>(); options.IsDirectoryTransfer = true; // transfer with IncludeSnapshots = true options.TransferItemModifier = (fileNode, transferItem) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; dirOptions.IncludeSnapshots = true; transferItem.Options = dirOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // verify that snapshots are transferred Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void TestProgressHandlerTest() { long fileSize = 10 * 1024 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize); var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { TransferContext transferContext = new SingleTransferContext(); ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
public void TestSetContentType() { string contentType = "contenttype"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); var options = new TestExecutionOptions<DMLibDataInfo>(); options.TransferItemModifier = (fileNode, transferItem) => { UploadOptions uploadOptions = new UploadOptions(); uploadOptions.ContentType = "contenttype"; transferItem.Options = uploadOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); FileNode destFileNode = result.DataInfo.RootNode.GetFileNode(DMLibTestBase.FileName); Test.Assert(contentType.Equals(destFileNode.ContentType), "Verify content type: {0}, expected {1}", destFileNode.ContentType, contentType); }
public void TestUnsupportedDirection() { DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); var result = this.ExecuteTestCase(sourceDataInfo, new TestExecutionOptions<DMLibDataInfo>()); Test.Assert(result.Exceptions.Count == 1, "Verify no exception is thrown."); Exception exception = result.Exceptions[0]; if (DMLibTestContext.SourceType == DMLibDataType.URI) { Test.Assert(exception is NotSupportedException, "Verify exception is NotSupportedException."); if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { VerificationHelper.VerifyExceptionErrorMessage(exception, "Copying from uri to Azure File Storage synchronously is not supported"); } else { VerificationHelper.VerifyExceptionErrorMessage(exception, "Copying from uri to Azure Blob Storage synchronously is not supported"); } } else if (DMLibTestBase.IsCloudBlob(DMLibTestContext.SourceType) && DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType)) { Test.Assert(exception is InvalidOperationException, "Verify exception is InvalidOperationException."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Blob type of source and destination must be the same."); } else { Test.Assert(exception is InvalidOperationException, "Verify exception is InvalidOperationException."); VerificationHelper.VerifyExceptionErrorMessage(exception, string.Format("Copying from File Storage to {0} Blob Storage asynchronously is not supported.", MapBlobDataTypeToBlobType(DMLibTestContext.DestType))); } Test.Assert(DMLibDataHelper.Equals(new DMLibDataInfo(string.Empty), result.DataInfo), "Verify no file is transfered."); }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions<DMLibDataInfo>(); options.LimitSpeed = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); secondCheckpoint = transferContext.LastCheckpoint; }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled."); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && DMLibTestContext.DestType != DMLibDataType.AppendBlob) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = new TransferContext(firstResumeCheckpoint) { ProgressHandler = progressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>()); VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new TransferContext(secondResumeCheckpoint) { ProgressHandler = progressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>()); if (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.SourceType == DMLibDataType.Stream) { VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } else { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } }
protected static void SetSourceDataInfo(DMLibDataType dataType, DMLibDataInfo dataInfo) { sourceDataInfos[dataType.ToString()] = dataInfo; }
public void DirectoryOverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); TransferContext transferContext = new DirectoryTransferContext(); transferContext.ShouldOverwriteCallbackAsync = async(source, destination) => { if (DMLibTestHelper.TransferInstanceToString(source).EndsWith(destExistNName)) { return(false); } else { return(true); } }; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); TransferException transferException = args.Exception as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skipped file", destExistNName); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { transferOptions.PreserveSMBAttributes = true; transferOptions.PreserveSMBPermissions = true; } transferItem.Options = transferOptions; transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { VerificationHelper.VerifySingleTransferStatus(result, 2, 1, 0, 1024 * 2); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 1, "Verify skipped transfer"); } else { VerificationHelper.VerifySingleTransferStatus(result, 3, 0, 0, 1024 * 3); Test.Assert(successCount == 3, "Very all transfers are success"); Test.Assert(skipCount == 0, "Very no transfer is skipped"); } if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(expectedDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( expectedDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } }
public void DirectoryOverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); TransferContext transferContext = new TransferContext(); transferContext.OverwriteCallback = (string sourcePath, string destinationPath) => { if (sourcePath.EndsWith(destExistNName)) { return(false); } else { return(true); } }; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); TransferException transferException = args.Exception as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 1, "Verify skipped transfer"); } else { Test.Assert(successCount == 3, "Very all transfers are success"); Test.Assert(skipCount == 0, "Very no transfer is skipped"); } }
public void OverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); var options = new TestExecutionOptions<DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; TransferContext transferContext = new TransferContext(); if (fileName.Equals(destExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } else if (fileName.Equals(destExistNName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackN(); } else if (fileName.Equals(destNotExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify there's only one exceptions."); TransferException transferException = result.Exceptions[0] as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); } }
public void DirectoryForceOverwriteTest() { string destExistName = "destExist"; string destNotExistName = "destNotExist"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistName, 1024); TransferContext transferContext = new DirectoryTransferContext(); transferContext.ShouldOverwriteCallback = TransferContext.ForceOverwrite; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } if (IsCloudService(DMLibTestContext.DestType)) { SharedAccessPermissions permissions; if (DMLibTestContext.IsAsync) { permissions = SharedAccessPermissions.Write | SharedAccessPermissions.Read; } else { permissions = SharedAccessPermissions.Write; } StorageCredentials destSAS = new StorageCredentials(DestAdaptor.GenerateSAS(permissions, (int)new TimeSpan(1, 0, 0, 0).TotalSeconds)); options.DestCredentials = destSAS; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // Verify transfer result Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); VerificationHelper.VerifySingleTransferStatus(result, 2, 0, 0, 1024 * 2); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 0, "Verify skipped transfer"); }
public void TestDirectoryNotIncludeSnapshots() { string snapshotFile1 = "snapshotFile1"; string snapshotFile2 = "snapshotFile2"; string snapshotFile3 = "snapshotFile3"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile1, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile2, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, snapshotFile3, 1024); // the 1st file has 1 snapshot FileNode fileNode1 = sourceDataInfo.RootNode.GetFileNode(snapshotFile1); fileNode1.SnapshotsCount = 1; // the 2nd file has 2 snapshots FileNode fileNode2 = sourceDataInfo.RootNode.GetFileNode(snapshotFile2); fileNode2.SnapshotsCount = 2; var options = new TestExecutionOptions<DMLibDataInfo>(); options.IsDirectoryTransfer = true; // transfer with default options, or IncludeSnapshots = false options.TransferItemModifier = (fileNode, transferItem) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; if (random.Next(0, 2) == 0) { Test.Info("Transfer directory with IncludeSnapshots=false"); dirOptions.IncludeSnapshots = false; } else { Test.Info("Transfer directory with defaullt IncludeSnapshots"); } transferItem.Options = dirOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // verify that only non-snapshot blobs are transferred DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); expectedDataInfo.RootNode.AddFileNode(fileNode1); expectedDataInfo.RootNode.AddFileNode(fileNode2); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(snapshotFile3)); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); }
private Dictionary<string, FileNode> PrepareSourceData(long fileSizeInB) { var sourceFileNodes = new Dictionary<string, FileNode>(); var sourceDataInfos = new Dictionary<string, DMLibDataInfo>(); // Prepare source data info foreach (DMLibTransferDirection direction in GetAllValidDirections()) { string fileName = GetTransferFileName(direction); DMLibDataInfo sourceDataInfo; string sourceDataInfoKey; if (direction.SourceType != DMLibDataType.URI) { sourceDataInfoKey = direction.SourceType.ToString(); } else { sourceDataInfoKey = GetTransferFileName(direction); } if (sourceDataInfos.ContainsKey(sourceDataInfoKey)) { sourceDataInfo = sourceDataInfos[sourceDataInfoKey]; } else { sourceDataInfo = new DMLibDataInfo(string.Empty); sourceDataInfos[sourceDataInfoKey] = sourceDataInfo; } DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, fileName, fileSizeInB); FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(fileName); sourceFileNodes.Add(fileName, sourceFileNode); } // Generate source data foreach (var pair in sourceDataInfos) { DMLibDataType sourceDataType; if (Enum.TryParse<DMLibDataType>(pair.Key, out sourceDataType)) { DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(sourceDataType); sourceAdaptor.Cleanup(); sourceAdaptor.CreateIfNotExists(); sourceAdaptor.GenerateData(pair.Value); } } // Generate source data for URI source separately since it's destination related DataAdaptor<DMLibDataInfo> uriSourceAdaptor = GetSourceAdaptor(DMLibDataType.URI); uriSourceAdaptor.Cleanup(); uriSourceAdaptor.CreateIfNotExists(); DMLibTestContext.SourceType = DMLibDataType.URI; DMLibTestContext.IsAsync = true; DMLibDataType[] uriDestDataTypes = { DMLibDataType.CloudFile, DMLibDataType.BlockBlob, DMLibDataType.PageBlob, DMLibDataType.AppendBlob }; foreach (DMLibDataType uriDestDataType in uriDestDataTypes) { DMLibTestContext.DestType = uriDestDataType; string sourceDataInfoKey = GetTransferFileName(DMLibDataType.URI, uriDestDataType, true); uriSourceAdaptor.GenerateData(sourceDataInfos[sourceDataInfoKey]); } // Clean up destination foreach (DMLibDataType destDataType in DataTypes) { if (destDataType != DMLibDataType.URI) { DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType); destAdaptor.Cleanup(); destAdaptor.CreateIfNotExists(); } } return sourceFileNodes; }
private void TestSearchPattern(bool recursive, DMLibDataInfo expectedResult, string searchPattern = null) { var options = new TestExecutionOptions<DMLibDataInfo>() { IsDirectoryTransfer = true, TransferItemModifier = (notUsed, item) => { dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = recursive; transferOptions.SearchPattern = searchPattern; item.Options = transferOptions; }, }; var testResult = this.ExecuteTestCase(null, options); VerificationHelper.VerifyTransferSucceed(testResult, expectedResult); }
private static void PrepareSourceData() { DMLibDataInfo sourceFileTree = new DMLibDataInfo(string.Empty); DirNode dirNode1 = new DirNode("folder1"); DirNode subDir1 = new DirNode("subfolder1"); subDir1.AddDirNode(new DirNode("subfolder3")); subDir1.AddFileNode(GenerateFileNode("testfile2")); subDir1.AddFileNode(GenerateFileNode("4testfile")); dirNode1.AddDirNode(subDir1); DirNode subDir2 = new DirNode("subfolder2"); DirNode subDir4 = new DirNode("subfolder4"); subDir4.AddFileNode(GenerateFileNode("test5")); subDir2.AddDirNode(subDir4); subDir2.AddFileNode(GenerateFileNode("TESTFILE345")); subDir2.AddFileNode(GenerateFileNode("testYfile")); subDir2.AddFileNode(GenerateFileNode("f_arbitrary.exe")); subDir2.AddFileNode(GenerateFileNode("测试x文件")); dirNode1.AddDirNode(subDir2); dirNode1.AddFileNode(GenerateFileNode("testfile1")); dirNode1.AddFileNode(GenerateFileNode("TestFile2")); dirNode1.AddFileNode(GenerateFileNode("测试文件2")); sourceFileTree.RootNode.AddDirNode(dirNode1); DirNode dirNode2 = new DirNode("folder2"); dirNode2.AddFileNode(GenerateFileNode("folder_file")); dirNode2.AddDirNode(new DirNode("testfile1")); dirNode2.AddFileNode(GenerateFileNode("测试文件三")); dirNode2.AddFileNode(GenerateFileNode("测试四文件")); sourceFileTree.RootNode.AddDirNode(dirNode2); DirNode dirNode3 = new DirNode("folder3"); sourceFileTree.RootNode.AddDirNode(dirNode3); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile1")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testfile2")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXfile")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("testXXfile")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("测试文件")); sourceFileTree.RootNode.AddFileNode(GenerateFileNode("..a123")); DMLibDataInfo blobSourceFileTree = sourceFileTree.Clone(); blobSourceFileTree.RootNode.AddFileNode(GenerateFileNode("TeSTfIle")); Test.Info("Start to generate test data, will take a while..."); foreach (DMLibDataType dataType in sourceDataTypes) { if (IsCloudBlob(dataType)) { PrepareSourceData(dataType, blobSourceFileTree.Clone()); } else { PrepareSourceData(dataType, sourceFileTree.Clone()); } } Test.Info("Done"); }
public void FilePattern_Local_UnMatchedPattern() { DMLibDataInfo expectedResult = new DMLibDataInfo(string.Empty); this.TestSearchPattern(true, expectedResult, "unmatched*"); }
private void TestDirectorySetAttribute_Restart( int bigFileSizeInKB, int smallFileSizeInKB, int bigFileNum, int smallFileNum, Action <DirNode> bigFileDirAddFileAction, Action <DirNode> smallFileDirAddFileAction, SetAttributesCallbackAsync setAttributesCallback = null, Action <DMLibDataInfo> sourceDataInfoDecorator = null) { int totalFileNum = bigFileNum + smallFileNum; long totalSizeInBytes = ((bigFileSizeInKB * bigFileNum) + (smallFileSizeInKB * smallFileNum)) * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode bigFileDirNode = new DirNode("big"); DirNode smallFileDirNode = new DirNode("small"); sourceDataInfo.RootNode.AddDirNode(bigFileDirNode); sourceDataInfo.RootNode.AddDirNode(smallFileDirNode); bigFileDirAddFileAction(bigFileDirNode); smallFileDirAddFileAction(smallFileDirNode); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo> { LimitSpeed = true, IsDirectoryTransfer = true }; using (Stream journalStream = new MemoryStream()) { bool isStreamJournal = random.Next(0, 2) == 0; var transferContext = isStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext(); transferContext.SetAttributesCallbackAsync = setAttributesCallback; var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); var eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); transferContext.FileFailed += (sender, e) => { Helper.VerifyCancelException(e.Exception); }; options.TransferItemModifier = (fileName, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!isStreamJournal) { // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; } Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); if (progressChecker.FailedFilesNumber <= 0) { Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber); } TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; if (!isStreamJournal) { secondCheckpoint = transferContext.LastCheckpoint; Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = null; if (isStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream) { ProgressHandler = progressChecker.GetProgressHandler() }; } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler() }; } resumeContext.SetAttributesCallbackAsync = setAttributesCallback; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); sourceDataInfoDecorator?.Invoke(sourceDataInfo); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (!isStreamJournal) { // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler(), // Need this overwrite callback since some files is already transferred to destination ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(), SetAttributesCallbackAsync = setAttributesCallback }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); } } }
public static bool Equals(DMLibDataInfo infoA, DMLibDataInfo infoB) { bool result; bool aIsEmpty = infoA == null || infoA.RootNode.IsEmpty; bool bIsEmpty = infoB == null || infoB.RootNode.IsEmpty; if (aIsEmpty && bIsEmpty) { result = true; } else if(aIsEmpty || bIsEmpty) { result = false; } else { result = Equals(infoA.RootNode, infoB.RootNode); } if (!result) { Test.Info("-----Data Info A-----"); MultiDirectionTestHelper.PrintTransferDataInfo(infoA); Test.Info("-----Data Info B-----"); MultiDirectionTestHelper.PrintTransferDataInfo(infoB); } return result; }
public void DirectoryShouldTransfer() { // Prepare data int totaFileNumber = DMLibTestConstants.FlatFileCount; int expectedTransferred = totaFileNumber, transferred = 0; int expectedSkipped = 0, skipped = 0; int expectedFailed = 0, failed = 0; DMLibDataInfo sourceDataInfo = this.GenerateSourceDataInfo(FileNumOption.FlatFolder, 1024); DirectoryTransferContext dirTransferContext = new DirectoryTransferContext(); List <String> notTransferredFileNames = new List <String>(); dirTransferContext.ShouldTransferCallbackAsync = async(source, dest) => { if (Helper.RandomBoolean()) { return(true); } else { Interlocked.Decrement(ref expectedTransferred); string fullName = DMLibTestHelper.TransferInstanceToString(source); string fileName = fullName.Substring(fullName.IndexOf(DMLibTestBase.FileName)); lock (notTransferredFileNames) { notTransferredFileNames.Add(fileName); } Test.Info("{0} is filterred in ShouldTransfer.", fileName); return(false); } }; dirTransferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref transferred); }; dirTransferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipped); }; dirTransferContext.FileFailed += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref failed); }; var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = dirTransferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); // Verify result DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DirNode expectedRootNode = expectedDataInfo.RootNode; foreach (string fileNames in notTransferredFileNames) { expectedRootNode.DeleteFileNode(fileNames); } VerificationHelper.VerifyTransferSucceed(result, expectedDataInfo); Test.Assert(expectedTransferred == transferred, string.Format("Verify transferred number. Expected: {0}, Actual: {1}", expectedTransferred, transferred)); Test.Assert(expectedSkipped == skipped, string.Format("Verify skipped number. Expected: {0}, Actual: {1}", expectedSkipped, skipped)); Test.Assert(expectedFailed == failed, string.Format("Verify failed number. Expected: {0}, Actual: {1}", expectedFailed, failed)); }
public void TestDirectoryCheckContentMD5Resume() { long fileSize = 5 * 1024; int fileCountMulti = 32; long totalSize = fileSize * 4 * fileCountMulti; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; // Prepare data for transfer items with checkMD5 DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; checkMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); // Prepare data for transfer items with disabling MD5 check DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; notCheckMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); TransferContext context = new DirectoryTransferContext(); eventChecker.Apply(context); ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, CancellationToken = cancellationTokenSource.Token, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, CancellationToken = cancellationTokenSource.Token }; var executionOption = new TestExecutionOptions <DMLibDataInfo>(); executionOption.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Cancel the transfer and store the second checkpoint cancellationTokenSource.Cancel(); }; executionOption.LimitSpeed = true; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, executionOption); eventChecker = new TransferEventChecker(); context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint)); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } transferExceptions.Add(args.Exception); }; progressChecker.Reset(); context.ProgressHandler = progressChecker.GetProgressHandler(); checkMD5Item = checkMD5Item.Clone(); notCheckMD5Item = notCheckMD5Item.Clone(); checkMD5Item.TransferContext = context; notCheckMD5Item.TransferContext = context; testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DMLibDataInfo actualDataInfo = testResult.DataInfo; for (int i = 0; i < fileCountMulti; ++i) { expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); } Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti); if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) { Test.Error("Expect one exception but actually no exception is thrown."); } else { for (int i = 0; i < fileCountMulti; ++i) { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } }
public void DirectoryOverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); TransferContext transferContext = new TransferContext(); transferContext.OverwriteCallback = (string sourcePath, string destinationPath) => { if (sourcePath.EndsWith(destExistNName)) { return false; } else { return true; } }; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); TransferException transferException = args.Exception as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions<DMLibDataInfo>(); options.IsDirectoryTransfer = true; if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 1, "Verify skipped transfer"); } else { Test.Assert(successCount == 3, "Very all transfers are success"); Test.Assert(skipCount == 0, "Very no transfer is skipped"); } }
public void TestDirectoryCheckContentMD5() { long fileSize = 5 * 1024 * 1024; long totalSize = fileSize * 4; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File); tmpFileNode.MD5 = wrongMD5; tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File); tmpFileNode.MD5 = wrongMD5; SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); TransferContext context = new TransferContext(); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } }; ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, }; var testResult = this.RunTransferItems(new List<TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions<DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); DMLibDataInfo actualDataInfo = testResult.DataInfo; actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1); if (testResult.Exceptions.Count != 1) { Test.Error("Expect one exception but actually no exception is thrown."); } else { VerificationHelper.VerifyTransferException(testResult.Exceptions[0], TransferErrorCode.SubTransferFails, "1 sub transfer(s) failed."); } }
private void TestAccessCondition(SourceOrDest sourceOrDest) { string eTag = "notmatch"; AccessCondition accessCondition = new AccessCondition() { IfMatchETag = eTag }; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, DMLibTestBase.FileName, 1024); var options = new TestExecutionOptions<DMLibDataInfo>(); if (sourceOrDest == SourceOrDest.Dest) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { dynamic transferOptions = DefaultTransferOptions; if (sourceOrDest == SourceOrDest.Source) { transferOptions.SourceAccessCondition = accessCondition; } else { transferOptions.DestinationAccessCondition = accessCondition; } transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); if (sourceOrDest == SourceOrDest.Dest) { Test.Assert(DMLibDataHelper.Equals(destDataInfo, result.DataInfo), "Verify no file is transferred."); } else { if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(DMLibDataHelper.Equals(new DMLibDataInfo(string.Empty), result.DataInfo), "Verify no file is transferred."); } else { foreach(var fileNode in result.DataInfo.EnumerateFileNodes()) { Test.Assert(fileNode.SizeInByte == 0, "Verify file {0} is empty", fileNode.Name); } } } // Verify TransferException if (result.Exceptions.Count != 1) { Test.Error("There should be exactly one exceptions."); return; } Exception exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(exception, TransferErrorCode.Unknown); // Verify innner StorageException VerificationHelper.VerifyStorageException(exception.InnerException, (int)HttpStatusCode.PreconditionFailed, "The condition specified using HTTP conditional header(s) is not met."); }
public void TestPreserveSMBPermissions() { if (!CrossPlatformHelpers.IsWindows) { return; } try { PreserveSMBPermissions preserveSMBPermissions = PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL; string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)"; #if DEBUG TestHookCallbacks.UnderTesting = true; TestHookCallbacks.GetFilePermissionsCallback = (path, SMBPermissionType) => { Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected."); return(sampleSDDL); }; TestHookCallbacks.SetFilePermissionsCallback = (path, portableSDDL, SMBPermissionType) => { Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected."); Test.Assert(portableSDDL.StartsWith(sampleSDDL), "The SDDL value should be expected."); }; #endif // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); FileNode fileNode = new FileNode(DMLibTestBase.FileName); fileNode.SizeInByte = 1024; fileNode.PortableSDDL = sampleSDDL; sourceDataInfo.RootNode.AddFileNode(fileNode); SingleTransferContext transferContext = new SingleTransferContext(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (fileNodeVar, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferOptions; transferOptions.PreserveSMBPermissions = preserveSMBPermissions; transferItem.Options = transferOptions; }; // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBPermissions(sourceDataInfo.RootNode, result.DataInfo.RootNode, preserveSMBPermissions); } } finally { #if DEBUG TestHookCallbacks.UnderTesting = false; TestHookCallbacks.GetFilePermissionsCallback = null; TestHookCallbacks.SetFilePermissionsCallback = null; #endif } }
public void OverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; TransferContext transferContext = new TransferContext(); if (fileName.Equals(destExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } else if (fileName.Equals(destExistNName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackN(); } else if (fileName.Equals(destNotExistYName)) { transferContext.OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } transferItem.TransferContext = transferContext; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify there's only one exceptions."); TransferException transferException = result.Exceptions[0] as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skiped file", destExistNName); } }
private void TestSASTokenOfEachVersion(string targetSASVersion, bool isDirectoryTransfer) { Test.Info("Testing version of {0}", targetSASVersion); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = isDirectoryTransfer; options.TransferItemModifier = (fileNode, transferItem) => { dynamic transferOptions = isDirectoryTransfer ? DefaultTransferDirectoryOptions : DefaultTransferOptions; transferItem.Options = transferOptions; if (isDirectoryTransfer) { transferItem.TransferContext = new DirectoryTransferContext(); transferItem.TransferContext.FileFailed += (source, e) => { Test.Error(e.Exception.ToString()); }; DirectoryOptions dirOptions = transferItem.Options as DirectoryOptions; dirOptions.Recursive = true; } else { transferItem.TransferContext = new SingleTransferContext(); } transferItem.TransferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite; }; string sourceSAS = null; string destSAS = null; switch (DMLibTestContext.SourceType) { case DMLibDataType.CloudBlob: case DMLibDataType.AppendBlob: case DMLibDataType.BlockBlob: case DMLibDataType.PageBlob: if ((DMLibTestContext.SourceType == DMLibDataType.AppendBlob) && (string.CompareOrdinal(targetSASVersion, "2015-04-05") < 0)) { break; } SourceAdaptor.CreateIfNotExists(); CloudBlobDataAdaptor blobAdaptor = SourceAdaptor as CloudBlobDataAdaptor; sourceSAS = Util.SASGenerator.GetSharedAccessSignature(blobAdaptor.GetBaseContainer(), new SharedAccessBlobPolicy { Permissions = isDirectoryTransfer ? SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List : SharedAccessBlobPermissions.Read, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; case DMLibDataType.CloudFile: if (string.CompareOrdinal(targetSASVersion, "2015-02-21") < 0) { break; } SourceAdaptor.CreateIfNotExists(); CloudFileDataAdaptor fileAdaptor = SourceAdaptor as CloudFileDataAdaptor; sourceSAS = Util.SASGenerator.GetSharedAccessSignature( fileAdaptor.GetBaseShare(), new SharedAccessFilePolicy { Permissions = isDirectoryTransfer ? SharedAccessFilePermissions.List | SharedAccessFilePermissions.Read : SharedAccessFilePermissions.Read, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; default: break; } if (!DMLibTestContext.IsAsync || (string.CompareOrdinal(targetSASVersion, "2014-02-14") >= 0)) { switch (DMLibTestContext.DestType) { case DMLibDataType.CloudBlob: case DMLibDataType.AppendBlob: case DMLibDataType.BlockBlob: case DMLibDataType.PageBlob: if ((DMLibTestContext.DestType == DMLibDataType.AppendBlob) && (string.CompareOrdinal(targetSASVersion, "2015-04-05") < 0)) { break; } DestAdaptor.CreateIfNotExists(); CloudBlobDataAdaptor blobAdaptor = DestAdaptor as CloudBlobDataAdaptor; destSAS = Util.SASGenerator.GetSharedAccessSignature(blobAdaptor.GetBaseContainer(), new SharedAccessBlobPolicy { Permissions = DMLibTestContext.IsAsync ? SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read : SharedAccessBlobPermissions.Write, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; case DMLibDataType.CloudFile: if (string.CompareOrdinal(targetSASVersion, "2015-02-21") < 0) { break; } DestAdaptor.CreateIfNotExists(); CloudFileDataAdaptor fileAdaptor = DestAdaptor as CloudFileDataAdaptor; destSAS = Util.SASGenerator.GetSharedAccessSignature( fileAdaptor.GetBaseShare(), new SharedAccessFilePolicy { Permissions = DMLibTestContext.IsAsync ? SharedAccessFilePermissions.Write | SharedAccessFilePermissions.Read : SharedAccessFilePermissions.Write, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1) }, null, null, null, targetSASVersion); break; default: break; } } if (null != sourceSAS) { options.SourceCredentials = new StorageCredentials(sourceSAS); } if (null != destSAS) { options.DestCredentials = new StorageCredentials(destSAS); } var result = this.ExecuteTestCase(sourceDataInfo, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); }
private void TestAccessCondition(SourceOrDest sourceOrDest) { string eTag = "\"notmatch\""; AccessCondition accessCondition = new AccessCondition() { IfMatchETag = eTag }; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, DMLibTestBase.FileName, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); if (sourceOrDest == SourceOrDest.Dest) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { dynamic transferOptions = DefaultTransferOptions; if (sourceOrDest == SourceOrDest.Source) { transferOptions.SourceAccessCondition = accessCondition; } else { transferOptions.DestinationAccessCondition = accessCondition; } transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); if (sourceOrDest == SourceOrDest.Dest) { Test.Assert(DMLibDataHelper.Equals(destDataInfo, result.DataInfo), "Verify no file is transferred."); } else { if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(DMLibDataHelper.Equals(new DMLibDataInfo(string.Empty), result.DataInfo), "Verify no file is transferred."); } else { foreach (var fileNode in result.DataInfo.EnumerateFileNodes()) { Test.Assert(fileNode.SizeInByte == 0, "Verify file {0} is empty", fileNode.Name); } } } // Verify TransferException if (result.Exceptions.Count != 1) { Test.Error("There should be exactly one exceptions."); return; } Exception exception = result.Exceptions[0]; #if DOTNET5_4 VerificationHelper.VerifyTransferException(exception, TransferErrorCode.Unknown); // Verify innner StorageException VerificationHelper.VerifyStorageException(exception.InnerException, (int)HttpStatusCode.PreconditionFailed, "The condition specified using HTTP conditional header(s) is not met."); #else VerificationHelper.VerifyTransferException(exception, TransferErrorCode.Unknown); // Verify innner StorageException VerificationHelper.VerifyStorageException(exception.InnerException, (int)HttpStatusCode.PreconditionFailed, "The condition specified using HTTP conditional header(s) is not met."); #endif }
public void OverwriteDestination() { string destExistYName = "destExistY"; string destExistNName = "destExistN"; string destNotExistYName = "destNotExistY"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024); var options = new TestExecutionOptions <DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } options.TransferItemModifier = (fileNode, transferItem) => { string fileName = fileNode.Name; TransferContext transferContext = new SingleTransferContext(); if (fileName.Equals(destExistYName)) { transferContext.ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } else if (fileName.Equals(destExistNName)) { transferContext.ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackN(); } else if (fileName.Equals(destNotExistYName)) { transferContext.ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(); } transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferOptions; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { transferOptions.PreserveSMBAttributes = true; transferOptions.PreserveSMBPermissions = true; } transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty); if (DMLibTestContext.DestType != DMLibDataType.Stream) { expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName)); expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName)); expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName)); } else { expectedDataInfo = sourceDataInfo; } // Verify transfer result Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); // Verify exception if (DMLibTestContext.DestType != DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify there's only one exceptions."); TransferException transferException = result.Exceptions[0] as TransferException; Test.Assert(transferException != null, "Verify the exception is a TransferException"); VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination, "Skipped file", destExistNName); } if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(expectedDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( expectedDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } }
protected static void PrepareDestData(DMLibDataType dataType, DMLibDataInfo dataInfo) { PrepareData(SourceOrDest.Dest, dataType, dataInfo); }
public void TestPreserveSMBProperties() { if (!CrossPlatformHelpers.IsWindows) { return; } CloudFileNtfsAttributes[] SMBFileAttributes = { CloudFileNtfsAttributes.ReadOnly, CloudFileNtfsAttributes.Hidden, #if DEBUG CloudFileNtfsAttributes.System, CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Normal, CloudFileNtfsAttributes.Temporary, CloudFileNtfsAttributes.Offline, CloudFileNtfsAttributes.NotContentIndexed, CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Temporary | CloudFileNtfsAttributes.Offline, CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden | CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive | CloudFileNtfsAttributes.Temporary | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData #endif }; try { for (int i = 0; i < SMBFileAttributes.Length; ++i) { #if DEBUG TestHookCallbacks.GetFileAttributesCallback = (path) => { if (path.EndsWith("\\") || path.EndsWith(DMLibTestBase.DirName)) { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { return(FileAttributes.Directory); } else { return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory)); } } else { return(Helper.ToFileAttributes(SMBFileAttributes[i])); } }; TestHookCallbacks.SetFileAttributesCallback = (path, attributes) => { if (path.EndsWith("\\") || path.EndsWith(DMLibTestBase.DirName)) { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory), "Directory attributes should be expected {0}", attributes); } } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]), "File attributes should be expected {0}", attributes); } }; #endif // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); FileNode fileNode = new FileNode(DMLibTestBase.FileName); fileNode.SizeInByte = 1024; fileNode.SMBAttributes = SMBFileAttributes[i]; sourceDataInfo.RootNode.AddFileNode(fileNode); SingleTransferContext transferContext = new SingleTransferContext(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.TransferItemModifier = (fileNodeVar, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferOptions; transferOptions.PreserveSMBAttributes = true; transferItem.Options = transferOptions; }; // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.Local) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false); } else { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); } } } finally { #if DEBUG TestHookCallbacks.GetFileAttributesCallback = null; TestHookCallbacks.SetFileAttributesCallback = null; #endif } }
protected static void SetDestDataInfo(DMLibDataType dataType, DMLibDataInfo dataInfo) { destDataInfos[dataType.ToString()] = dataInfo; }
public void TestDirectoryPreserveSMBProperties() { if (!CrossPlatformHelpers.IsWindows) { return; } CloudFileNtfsAttributes[] SMBFileAttributes = { CloudFileNtfsAttributes.ReadOnly, CloudFileNtfsAttributes.Hidden, #if DEBUG CloudFileNtfsAttributes.System, CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Normal, CloudFileNtfsAttributes.Offline, CloudFileNtfsAttributes.NotContentIndexed, CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden | CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData #endif }; for (int i = 0; i < SMBFileAttributes.Length; ++i) { Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]); DirectoryTransferContext dirTransferContext = new DirectoryTransferContext(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = dirTransferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; transferItem.Options = transferOptions; }; #if DEBUG TestHookCallbacks.GetFileAttributesCallback = (path) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { return(FileAttributes.Directory); } else { return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory)); } } else { return(Helper.ToFileAttributes(SMBFileAttributes[i])); } }; TestHookCallbacks.SetFileAttributesCallback = (path, attributes) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot")) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory), "Directory attributes should be expected {0}", attributes); } } } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]), "File attributes should be expected {0}", attributes); } }; #endif try { // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), IsDirectoryTransfer = true, Options = DefaultTransferDirectoryOptions }; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; item.Options = transferOptions; item.TransferContext = new DirectoryTransferContext(); item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) => { return(false); }; List <TransferItem> items = new List <TransferItem>(); items.Add(item); result = this.RunTransferItems(items, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.Local) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false); } else { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); } } finally { #if DEBUG TestHookCallbacks.GetFileAttributesCallback = null; TestHookCallbacks.SetFileAttributesCallback = null; #endif } } }
public static void SetCalculatedFileMD5(DMLibDataInfo dataInfo, DataAdaptor<DMLibDataInfo> destAdaptor, bool disableMD5Check = false) { foreach (FileNode fileNode in dataInfo.EnumerateFileNodes()) { if (DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType)) { CloudBlobDataAdaptor cloudBlobDataAdaptor = destAdaptor as CloudBlobDataAdaptor; CloudBlob cloudBlob = cloudBlobDataAdaptor.GetCloudBlobReference(dataInfo.RootPath, fileNode); fileNode.MD5 = CloudBlobHelper.CalculateMD5ByDownloading(cloudBlob, disableMD5Check); } else if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { CloudFileDataAdaptor cloudFileDataAdaptor = destAdaptor as CloudFileDataAdaptor; CloudFile cloudFile = cloudFileDataAdaptor.GetCloudFileReference(dataInfo.RootPath, fileNode); fileNode.MD5 = CloudFileHelper.CalculateMD5ByDownloading(cloudFile, disableMD5Check); } // No need to set md5 for local destination } }
public void TestDirectoryWithSpecialCharNamedBlobs() { #if DNXCORE50 // TODO: There's a known issue that signature for URI with '[' or ']' doesn't work. string specialChars = "~`!@#$%()-_+={};?.^&"; #else string specialChars = "~`!@#$%()-_+={}[];?.^&"; #endif DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); var subDirWithDot = new DirNode(DMLibTestBase.FolderName + "." + DMLibTestBase.FolderName); for (int i = 0; i < specialChars.Length; ++i) { string fileName = DMLibTestBase.FileName + specialChars[i] + DMLibTestBase.FileName; if (random.Next(2) == 0) { if ((specialChars[i] != '.') && (random.Next(2) == 0)) { string folderName = DMLibTestBase.FolderName + specialChars[i] + DMLibTestBase.FolderName; var subDir = new DirNode(folderName); DMLibDataHelper.AddOneFileInBytes(subDir, fileName, 1024); FileNode fileNode1 = subDir.GetFileNode(fileName); fileNode1.SnapshotsCount = 1; sourceDataInfo.RootNode.AddDirNode(subDir); } else { DMLibDataHelper.AddOneFileInBytes(subDirWithDot, fileName, 1024); FileNode fileNode1 = subDirWithDot.GetFileNode(fileName); fileNode1.SnapshotsCount = 1; } } else { DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, fileName, 1024); FileNode fileNode1 = sourceDataInfo.RootNode.GetFileNode(fileName); fileNode1.SnapshotsCount = 1; } } sourceDataInfo.RootNode.AddDirNode(subDirWithDot); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; // transfer with IncludeSnapshots = true options.TransferItemModifier = (fileNode, transferItem) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; dirOptions.IncludeSnapshots = true; transferItem.Options = dirOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // verify that snapshots are transferred Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); }
private void UploadFromStreamTest(DMLibDataType destType, long[] variedSourceLength, int?blockSize, bool seekable, bool fixedSized) { DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(DMLibDataType.Stream); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destType); DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); foreach (long fileSizeInByte in variedSourceLength) { sourceDataInfo.RootNode.AddFileNode(new FileNode($"{FileName}_{fileSizeInByte}") { SizeInByte = fileSizeInByte, }); } sourceAdaptor.GenerateData(sourceDataInfo); destAdaptor.CreateIfNotExists(); List <TransferItem> uploadItems = new List <TransferItem>(); foreach (long fileSizeInByte in variedSourceLength) { string fileName = $"{FileName}_{fileSizeInByte}"; FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName); uploadItems.Add(new TransferItem() { SourceObject = new DMLibTestStream(sourceAdaptor.GetTransferObject(string.Empty, fileNode) as FileStream, seekable, fixedSized), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = DMLibDataType.Stream, DestType = destType, IsServiceCopy = false }); } // Execution var result = this.RunTransferItems( uploadItems, new TestExecutionOptions <DMLibDataInfo>() { DisableDestinationFetch = true, BlockSize = blockSize.HasValue ? blockSize.Value : 4 * 1024 * 1024 }); if (!fixedSized && (destType == DMLibDataType.PageBlob || destType == DMLibDataType.CloudFile)) { Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0].Message.Contains("Source must be fixed size"), "Verify error is expected."); } else { // Verify all files are transfered successfully Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs."); DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { FileNode sourceFileNode = sourceDataInfo.RootNode.GetFileNode(destFileNode.Name); Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result."); } } sourceAdaptor.Cleanup(); destAdaptor.Cleanup(); }
public void ForceOverwriteTest() { string destExistName = "destExist"; string destNotExistName = "destNotExist"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistName, 1024); DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistName, 1024); DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistName, 1024); TransferContext transferContext = new SingleTransferContext(); transferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite; int skipCount = 0; int successCount = 0; transferContext.FileSkipped += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref skipCount); }; transferContext.FileTransferred += (object sender, TransferEventArgs args) => { Interlocked.Increment(ref successCount); }; var options = new TestExecutionOptions <DMLibDataInfo>(); if (DMLibTestContext.DestType != DMLibDataType.Stream) { options.DestTransferDataInfo = destDataInfo; } if (IsCloudService(DMLibTestContext.DestType)) { SharedAccessPermissions permissions; if (DMLibTestContext.CopyMethod == DMLibCopyMethod.ServiceSideAsyncCopy) { permissions = SharedAccessPermissions.Write | SharedAccessPermissions.Read; } else { permissions = SharedAccessPermissions.Write; } StorageCredentials destSAS = new StorageCredentials(DestAdaptor.GenerateSAS(permissions, (int)new TimeSpan(1, 0, 0, 0).TotalSeconds)); options.DestCredentials = destSAS; } options.TransferItemModifier = (fileNode, transferItem) => { transferItem.TransferContext = transferContext; dynamic transferOptions = DefaultTransferOptions; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { transferOptions.PreserveSMBAttributes = true; transferOptions.PreserveSMBPermissions = true; } transferItem.Options = transferOptions; }; var result = this.ExecuteTestCase(sourceDataInfo, options); // Verify transfer result Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); Test.Assert(successCount == 2, "Verify success transfers"); Test.Assert(skipCount == 0, "Verify skipped transfer"); if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( sourceDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } }
public void TestDirectoryPreserveSMBPermissionsResume() { if (!CrossPlatformHelpers.IsWindows) { return; } PreserveSMBPermissions preserveSMBPermissions = PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL; string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)"; CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; bool IsStreamJournal = random.Next(0, 2) == 0; using (Stream journalStream = new MemoryStream()) { // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); GenerateDirNodeWithPermissions(sourceDataInfo.RootNode, 2, sampleSDDL); DirectoryTransferContext dirTransferContext = null; if (IsStreamJournal) { dirTransferContext = new DirectoryTransferContext(journalStream); } else { dirTransferContext = new DirectoryTransferContext(); } var progressChecker = new ProgressChecker(14, 14 * 1024); dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, item) => { item.TransferContext = dirTransferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBPermissions = preserveSMBPermissions; item.Options = transferOptions; item.CancellationToken = tokenSource.Token; transferItem = item; }; TransferCheckpoint checkpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!IsStreamJournal) { checkpoint = dirTransferContext.LastCheckpoint; } // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; #if DEBUG TestHookCallbacks.UnderTesting = true; TestHookCallbacks.GetFilePermissionsCallback = (path, SMBPermissionType) => { Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected."); return(sampleSDDL); }; TestHookCallbacks.SetFilePermissionsCallback = (path, portableSDDL, SMBPermissionType) => { Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected."); Test.Assert(portableSDDL.StartsWith(sampleSDDL), "The SDDL value should be expected."); }; #endif try { // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; Helper.VerifyCancelException(exception); TransferItem resumeItem = transferItem.Clone(); DirectoryTransferContext resumeContext = null; journalStream.Position = 0; if (IsStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream); } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint)); } resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBPermissions(sourceDataInfo.RootNode, result.DataInfo.RootNode, preserveSMBPermissions); } } finally { #if DEBUG TestHookCallbacks.UnderTesting = false; TestHookCallbacks.GetFilePermissionsCallback = null; TestHookCallbacks.SetFilePermissionsCallback = null; #endif } } }
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5) { long fileSize = 5 * 1024; int fileCountMulti = 32; long totalSize = fileSize * 4 * fileCountMulti; string wrongMD5 = "wrongMD5"; string wrongMD5File = "wrongMD5File"; string correctMD5File = "correctMD5File"; // Prepare data for transfer items with checkMD5 DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5"); for (int i = 0; i < fileCountMulti; ++i) { var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}") { SizeInByte = fileSize, MD5 = wrongMD5 }; checkMD5Folder.AddFileNode(wrongMD5FileNode); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize); } sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); SourceAdaptor.GenerateData(sourceDataInfo); DestAdaptor.Cleanup(); TransferEventChecker eventChecker = new TransferEventChecker(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); using (var resumeStream = new MemoryStream()) { TransferContext context = new DirectoryTransferContext(resumeStream); eventChecker.Apply(context); ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti, totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = !checkMD5, Recursive = true, }, CancellationToken = cancellationTokenSource.Token, }; var executionOption = new TestExecutionOptions <DMLibDataInfo>(); executionOption.AfterAllItemAdded = () => { // Wait until there are data transferred if (!progressChecker.DataTransferred.WaitOne(30000)) { Test.Error("No progress in 30s."); } // Cancel the transfer and store the second checkpoint cancellationTokenSource.Cancel(); }; executionOption.LimitSpeed = true; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item }, executionOption); if (null != testResult.Exceptions) { foreach (var exception in testResult.Exceptions) { Test.Info("Got exception during transferring. {0}", exception); } } eventChecker = new TransferEventChecker(); resumeStream.Position = 0; context = new DirectoryTransferContext(resumeStream); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(wrongMD5File); } transferExceptions.Add(args.Exception); }; progressChecker.Reset(); context.ProgressHandler = progressChecker.GetProgressHandler(); checkMD5Item = checkMD5Item.Clone(); checkMD5Item.TransferContext = context; testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); DMLibDataInfo actualDataInfo = testResult.DataInfo; for (int i = 0; i < fileCountMulti; ++i) { expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); } Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected."); VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0); if (checkMD5) { if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) { Test.Error("Expect one exception but actually no exception is thrown."); } else { for (int i = 0; i < fileCountMulti; ++i) { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } } else { Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5"); } } }
public void TestDirectoryCheckContentMD5() { long fileSize = 5 * 1024 * 1024; long totalSize = fileSize * 4; string wrongMD5 = "wrongMD5"; string checkWrongMD5File = "checkWrongMD5File"; string checkCorrectMD5File = "checkCorrectMD5File"; string notCheckWrongMD5File = "notCheckWrongMD5File"; string notCheckCorrectMD5File = "notCheckCorrectMD5File"; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode checkMD5Folder = new DirNode("checkMD5"); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize); DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize); sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File); tmpFileNode.MD5 = wrongMD5; tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File); tmpFileNode.MD5 = wrongMD5; SourceAdaptor.GenerateData(sourceDataInfo); TransferEventChecker eventChecker = new TransferEventChecker(); TransferContext context = new TransferContext(); eventChecker.Apply(context); bool failureReported = false; context.FileFailed += (sender, args) => { if (args.Exception != null) { failureReported = args.Exception.Message.Contains(checkWrongMD5File); } }; ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); List <Exception> transferExceptions = new List <Exception>(); context.FileFailed += (eventSource, eventArgs) => { transferExceptions.Add(eventArgs.Exception); }; TransferItem checkMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = false, Recursive = true, }, }; TransferItem notCheckMD5Item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, TransferContext = context, Options = new DownloadDirectoryOptions() { DisableContentMD5Validation = true, Recursive = true, }, }; var testResult = this.RunTransferItems(new List <TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions <DMLibDataInfo>()); DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); DMLibDataInfo actualDataInfo = testResult.DataInfo; actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File); actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); Test.Assert(failureReported, "Verify md5 check failure is reported."); VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1); if (testResult.Exceptions.Count != 0 || transferExceptions.Count != 1) { Test.Error("Expect one exception but actually no exception is thrown."); } else { VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } }
public void TestDirectoryPreserveSMBPropertiesResume() { if (!CrossPlatformHelpers.IsWindows) { return; } CloudFileNtfsAttributes[] SMBFileAttributes = { CloudFileNtfsAttributes.ReadOnly, CloudFileNtfsAttributes.Hidden, #if DEBUG CloudFileNtfsAttributes.System, CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Normal, CloudFileNtfsAttributes.Offline, CloudFileNtfsAttributes.NotContentIndexed, CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden, CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive, CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData, CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden | CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData #endif }; for (int i = 0; i < SMBFileAttributes.Length; ++i) { CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; bool IsStreamJournal = random.Next(0, 2) == 0; using (Stream journalStream = new MemoryStream()) { Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]); // Prepare data DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]); DirectoryTransferContext dirTransferContext = null; if (IsStreamJournal) { dirTransferContext = new DirectoryTransferContext(journalStream); } else { dirTransferContext = new DirectoryTransferContext(); } var progressChecker = new ProgressChecker(14, 14 * 1024); dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler(); var options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; options.TransferItemModifier = (fileNode, item) => { item.TransferContext = dirTransferContext; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; item.Options = transferOptions; item.CancellationToken = tokenSource.Token; transferItem = item; }; TransferCheckpoint checkpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!IsStreamJournal) { checkpoint = dirTransferContext.LastCheckpoint; } // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; #if DEBUG TestHookCallbacks.GetFileAttributesCallback = (path) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { return(FileAttributes.Directory); } else { return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory)); } } else { return(Helper.ToFileAttributes(SMBFileAttributes[i])); } }; TestHookCallbacks.SetFileAttributesCallback = (path, attributes) => { if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName)) { if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot")) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal) { Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes); } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory), "Directory attributes should be expected {0}", attributes); } } } else { Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]), "File attributes should be expected {0}", attributes); } }; #endif try { // Execute test case var result = this.ExecuteTestCase(sourceDataInfo, options); Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; Helper.VerifyCancelException(exception); TransferItem resumeItem = transferItem.Clone(); DirectoryTransferContext resumeContext = null; journalStream.Position = 0; if (IsStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream); } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint)); } resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); options = new TestExecutionOptions <DMLibDataInfo>(); options.IsDirectoryTransfer = true; TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), IsDirectoryTransfer = true, Options = DefaultTransferDirectoryOptions }; dynamic transferOptions = DefaultTransferDirectoryOptions; transferOptions.Recursive = true; transferOptions.PreserveSMBAttributes = true; item.Options = transferOptions; item.TransferContext = new DirectoryTransferContext(); item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) => { return(false); }; List <TransferItem> items = new List <TransferItem>(); items.Add(item); result = this.RunTransferItems(items, options); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (DMLibTestContext.DestType == DMLibDataType.Local) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false); } else { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); } } finally { #if DEBUG TestHookCallbacks.GetFileAttributesCallback = null; TestHookCallbacks.SetFileAttributesCallback = null; #endif } } } }
public static void VerifySingleObjectResumeResult(TestResult<DMLibDataInfo> result, DMLibDataInfo expectedDataInfo) { Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result."); }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions<DMLibDataInfo>(); options.LimitSpeed = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); secondCheckpoint = transferContext.LastCheckpoint; Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled."); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; ProgressChecker firstProgressChecker = null, secondProgressChecker = null; // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.IsAsync)) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // first progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { // The destination is already created, will cause a transfer skip firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob)) { firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else { firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024); } // second progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream) { // The destination is already created, will cause a transfer skip secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream) { secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync) { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024); } else { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024); } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); TransferContext resumeContext = new TransferContext( IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file"); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); resumeContext = new TransferContext( IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = secondProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file"); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync) { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } }