private static List <TransferItem> GetTransformItemsForAllSingleTransferDirections(bool resume) { List <TransferItem> allItems = new List <TransferItem>(); foreach (DMLibTransferDirection direction in GetAllValidDirections()) { if (resume && (direction.SourceType == DMLibDataType.Stream || direction.DestType == DMLibDataType.Stream)) { continue; } string fileName = GetTransferFileName(direction); DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); FileNode fileNode = singleObjectNodes[fileName]; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = direction.SourceType, DestType = direction.DestType, CopyMethod = direction.CopyMethod, TransferContext = new SingleTransferContext() { SetAttributesCallbackAsync = AllTransferDirectionTest.SetAttributesCallbackMethodAsync } }; allItems.Add(item); } return(allItems); }
private static List <TransferItem> GetTransformItemsForAllDirTransferDirections(bool resume) { List <TransferItem> allItems = new List <TransferItem>(); foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections()) { string dirName = GetTransferDirName(direction); DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); DirNode dirNode = directoryNodes[dirName]; dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType); options.Recursive = true; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode), DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode), SourceType = direction.SourceType, DestType = direction.DestType, CopyMethod = direction.CopyMethod, IsDirectoryTransfer = true, Options = options, TransferContext = new DirectoryTransferContext() { SetAttributesCallbackAsync = AllTransferDirectionTest.SetAttributesCallbackMethodAsync } }; allItems.Add(item); } return(allItems); }
protected override Task DoTransferImp(TransferItem item) { if (item.IsDirectoryTransfer) { return(this.DownloadDirectory(item.SourceObject, item)); } else { return(this.Download(item.SourceObject, item)); } }
protected override Task DoTransferImp(TransferItem item) { if (item.IsDirectoryTransfer) { return this.DownloadDirectory(item.SourceObject, item); } else { return this.Download(item.SourceObject, item); } }
private static List <TransferItem> GetTransformItemsForAllDirections(bool resume) { List <TransferItem> allItems = new List <TransferItem>(); foreach (DMLibTransferDirection direction in GetAllValidDirections()) { if (resume && (direction.SourceType == DMLibDataType.Stream || direction.DestType == DMLibDataType.Stream)) { continue; } string fileName = GetTransferFileName(direction); DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); FileNode fileNode = singleObjectNodes[fileName]; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = direction.SourceType, DestType = direction.DestType, IsServiceCopy = direction.IsAsync, }; allItems.Add(item); } foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections()) { string dirName = GetTransferDirName(direction); DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); DirNode dirNode = directoryNodes[dirName]; dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType); options.Recursive = true; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode), DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode), SourceType = direction.SourceType, DestType = direction.DestType, IsServiceCopy = direction.IsAsync, IsDirectoryTransfer = true, Options = options, }; allItems.Add(item); } return(allItems); }
protected override Task DoTransferImp(TransferItem item) { if (item.IsDirectoryTransfer) { return this.CopyDirectory(item.SourceObject, item.DestObject, item); } else { return this.Copy(item.SourceObject, item.DestObject, item); } }
protected override async Task <TransferStatus> DoTransferImp(TransferItem item) { if (item.IsDirectoryTransfer) { return(await this.UploadDirectory(item.DestObject, item).ConfigureAwait(false)); } else { await this.Upload(item.DestObject, item).ConfigureAwait(false); return(null); } }
protected override async Task <TransferStatus> DoTransferImp(TransferItem item) { if (item.IsDirectoryTransfer) { return(await this.CopyDirectory(item.SourceObject, item.DestObject, item)); } else { await this.Copy(item.SourceObject, item.DestObject, item); return(null); } }
public TransferItem Clone() { TransferItem newTransferItem = new TransferItem() { SourceObject = NewLocationObject(this.SourceObject), DestObject = NewLocationObject(this.DestObject), SourceType = this.SourceType, DestType = this.DestType, IsServiceCopy = this.IsServiceCopy, Options = this.Options, }; return(newTransferItem); }
public void Upload_Symlinked_RootDir() { #if DNXCORE50 if (!CrossPlatformHelpers.IsLinux) { return; } DMLibDataInfo sourceDataInfo = new DMLibDataInfo(""); DMLibDataInfo targetDataInfo = new DMLibDataInfo("target"); UnicodeFileName = "TempTestName"; DirNode dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}"); dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}") { SizeInByte = 1024 }); targetDataInfo.RootNode.AddDirNode(dirNode); SourceAdaptor.CreateIfNotExists(); DestAdaptor.CreateIfNotExists(); SourceAdaptor.GenerateData(targetDataInfo); sourceDataInfo.RootNode = DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", "target", targetDataInfo.RootNode); SourceAdaptor.GenerateData(sourceDataInfo); TransferItem item = new TransferItem() { SourceObject = Path.Combine(SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as string, sourceDataInfo.RootNode.Name), DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode), IsDirectoryTransfer = true, SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(), TransferContext = new DirectoryTransferContext(), Options = DefaultTransferDirectoryOptions }; (item.Options as UploadDirectoryOptions).Recursive = true; var result = this.RunTransferItems(new List <TransferItem>() { item }, new TestExecutionOptions <DMLibDataInfo>()); Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown."); Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result."); #endif }
public TransferItem Clone() { TransferItem newTransferItem = new TransferItem() { SourceObject = NewLocationObject(this.SourceObject), DestObject = NewLocationObject(this.DestObject), SourceType = this.SourceType, DestType = this.DestType, IsDirectoryTransfer = this.IsDirectoryTransfer, CopyMethod = this.CopyMethod, Options = this.Options, }; return(newTransferItem); }
private Task <TransferStatus> DownloadDirectory(dynamic sourceObject, TransferItem item) { DownloadDirectoryOptions downloadDirectoryOptions = item.Options as DownloadDirectoryOptions; DirectoryTransferContext transferContext = item.TransferContext as DirectoryTransferContext; CancellationToken cancellationToken = item.CancellationToken; string destPath = item.DestObject as string; if (cancellationToken == null || cancellationToken == CancellationToken.None) { return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext)); } else { return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext, cancellationToken)); } }
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options) { this.CleanupData(); SourceAdaptor.CreateIfNotExists(); DestAdaptor.CreateIfNotExists(); if (sourceDataInfo != null) { SourceAdaptor.GenerateData(sourceDataInfo); } if (options.DestTransferDataInfo != null) { DestAdaptor.GenerateData(options.DestTransferDataInfo); } if (options.AfterDataPrepared != null) { options.AfterDataPrepared(); } List <TransferItem> allItems = new List <TransferItem>(); foreach (var fileNode in sourceDataInfo.EnumerateFileNodes()) { TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(fileNode), DestObject = DestAdaptor.GetTransferObject(fileNode), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, }; if (options.TransferItemModifier != null) { options.TransferItemModifier(fileNode, item); } allItems.Add(item); } return(this.RunTransferItems(allItems, options)); }
private Task CopyDirectory(dynamic sourceObject, dynamic destObject, TransferItem item) { CopyDirectoryOptions copyDirectoryOptions = item.Options as CopyDirectoryOptions; TransferContext transferContext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; if (cancellationToken != null && cancellationToken != CancellationToken.None) { return TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext, cancellationToken); } else if (transferContext != null || copyDirectoryOptions != null) { return TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext); } else { return TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy); } }
private Task Copy(dynamic sourceObject, dynamic destObject, TransferItem item) { CopyOptions copyOptions = item.Options as CopyOptions; SingleTransferContext transferContext = item.TransferContext as SingleTransferContext; CancellationToken cancellationToken = item.CancellationToken; if (cancellationToken != null && cancellationToken != CancellationToken.None) { return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy, copyOptions, transferContext, cancellationToken)); } else if (transferContext != null || copyOptions != null) { return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy, copyOptions, transferContext)); } else { return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy)); } }
private Task Download(dynamic sourceObject, TransferItem item) { DownloadOptions downloadOptions = item.Options as DownloadOptions; SingleTransferContext transferContext = item.TransferContext as SingleTransferContext; CancellationToken cancellationToken = item.CancellationToken; string destPath = item.DestObject as string; Stream destStream = item.DestObject as Stream; if (cancellationToken != null && cancellationToken != CancellationToken.None) { if (destPath != null) { return(TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext, cancellationToken)); } else { return(TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext, cancellationToken)); } } else if (transferContext != null || downloadOptions != null) { if (destPath != null) { return(TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext)); } else { return(TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext)); } } else { if (destPath != null) { return(TransferManager.DownloadAsync(sourceObject, destPath)); } else { return(TransferManager.DownloadAsync(sourceObject, destStream)); } } }
private Task Upload(dynamic destObject, TransferItem item) { UploadOptions uploadOptions = item.Options as UploadOptions; TransferContext transferContext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; string sourcePath = item.SourceObject as string; Stream sourceStream = item.SourceObject as Stream; if (cancellationToken != null && cancellationToken != CancellationToken.None) { if (sourcePath != null) { return TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext, cancellationToken); } else { return TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext, cancellationToken); } } else if (transferContext != null || uploadOptions != null) { if (sourcePath != null) { return TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext); } else { return TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext); } } else { if (sourcePath != null) { return TransferManager.UploadAsync(sourcePath, destObject); } else { return TransferManager.UploadAsync(sourceStream, destObject); } } }
private Task Upload(dynamic destObject, TransferItem item) { UploadOptions uploadOptions = item.Options as UploadOptions; TransferContext transferContext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; string sourcePath = item.SourceObject as string; Stream sourceStream = item.SourceObject as Stream; if (cancellationToken != null && cancellationToken != CancellationToken.None) { if (sourcePath != null) { return(TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext, cancellationToken)); } else { return(TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext, cancellationToken)); } } else if (transferContext != null || uploadOptions != null) { if (sourcePath != null) { return(TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext)); } else { return(TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext)); } } else { if (sourcePath != null) { return(TransferManager.UploadAsync(sourcePath, destObject)); } else { return(TransferManager.UploadAsync(sourceStream, destObject)); } } }
private Task Download(dynamic sourceObject, TransferItem item) { DownloadOptions downloadOptions = item.Options as DownloadOptions; TransferContext transferContext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; string destPath = item.DestObject as string; Stream destStream = item.DestObject as Stream; if (cancellationToken != null && cancellationToken != CancellationToken.None) { if (destPath != null) { return TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext, cancellationToken); } else { return TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext, cancellationToken); } } else if (transferContext != null || downloadOptions != null) { if (destPath != null) { return TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext); } else { return TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext); } } else { if (destPath != null) { return TransferManager.DownloadAsync(sourceObject, destPath); } else { return TransferManager.DownloadAsync(sourceObject, destStream); } } }
private Task <TransferStatus> UploadDirectory(dynamic destObject, TransferItem item) { UploadDirectoryOptions uploadDirectoryOptions = item.Options as UploadDirectoryOptions; TransferContext transferContrext = item.TransferContext; CancellationToken cancellationToken = item.CancellationToken; string sourcePath = item.SourceObject as string; if (cancellationToken != null && cancellationToken != CancellationToken.None) { return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext, cancellationToken)); } else if (transferContrext != null || uploadDirectoryOptions != null) { return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext)); } else { return(TransferManager.UploadDirectoryAsync(sourcePath, destObject)); } }
private List <TransferItem> GetTransformItemsForAllDirections(Dictionary <string, FileNode> fileNodes) { List <TransferItem> allItems = new List <TransferItem>(); foreach (DMLibTransferDirection direction in GetAllValidDirections()) { string fileName = GetTransferFileName(direction); DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); FileNode fileNode = fileNodes[fileName]; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(fileNode), DestObject = destAdaptor.GetTransferObject(fileNode), SourceType = direction.SourceType, DestType = direction.DestType, IsServiceCopy = direction.IsAsync, }; allItems.Add(item); } return(allItems); }
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options) { if (options.DisableSourceCleaner) { this.CleanupData(false, true); } else { this.CleanupData(); } SourceAdaptor.CreateIfNotExists(); DestAdaptor.CreateIfNotExists(); string sourceRootPath = string.Empty; DirNode sourceRootNode = new DirNode(string.Empty); if (sourceDataInfo != null) { sourceRootPath = sourceDataInfo.RootPath; sourceRootNode = sourceDataInfo.RootNode; if (!options.DisableSourceGenerator) { SourceAdaptor.GenerateData(sourceDataInfo); } } string destRootPath = string.Empty; if (options.DestTransferDataInfo != null) { destRootPath = options.DestTransferDataInfo.RootPath; DestAdaptor.GenerateData(options.DestTransferDataInfo); } if (options.AfterDataPrepared != null) { options.AfterDataPrepared(); } List <TransferItem> allItems = new List <TransferItem>(); if (options.IsDirectoryTransfer) { TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceRootPath, sourceRootNode, options.SourceCredentials), DestObject = DestAdaptor.GetTransferObject(destRootPath, sourceRootNode, options.DestCredentials), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, IsDirectoryTransfer = true, }; if (options.TransferItemModifier != null) { options.TransferItemModifier(null, item); } allItems.Add(item); } else { foreach (var fileNode in sourceDataInfo.EnumerateFileNodes()) { TransferItem item = new TransferItem() { SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, fileNode, options.SourceCredentials), DestObject = DestAdaptor.GetTransferObject(destRootPath, fileNode, options.DestCredentials), SourceType = DMLibTestContext.SourceType, DestType = DMLibTestContext.DestType, IsServiceCopy = DMLibTestContext.IsAsync, }; if (options.TransferItemModifier != null) { options.TransferItemModifier(fileNode, item); } allItems.Add(item); } } return(this.RunTransferItems(allItems, options)); }
public TransferItem Clone() { TransferItem newTransferItem = new TransferItem() { SourceObject = NewLocationObject(this.SourceObject), DestObject = NewLocationObject(this.DestObject), SourceType = this.SourceType, DestType = this.DestType, IsDirectoryTransfer = this.IsDirectoryTransfer, IsServiceCopy = this.IsServiceCopy, Options = this.Options, }; return newTransferItem; }
protected abstract Task DoTransferImp(TransferItem item);
public Task DoTransfer(TransferItem item) { Test.Info("Do transfer: {0}", item.ToString()); return(this.DoTransferImp(item)); }
private List<TransferItem> GetTransformItemsForAllDirections(Dictionary<string, FileNode> fileNodes) { List<TransferItem> allItems = new List<TransferItem>(); foreach (DMLibTransferDirection direction in GetAllValidDirections()) { string fileName = GetTransferFileName(direction); DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); FileNode fileNode = fileNodes[fileName]; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(fileNode), DestObject = destAdaptor.GetTransferObject(fileNode), SourceType = direction.SourceType, DestType = direction.DestType, IsServiceCopy = direction.IsAsync, }; allItems.Add(item); } return allItems; }
protected override Task DoTransferImp(TransferItem item) { return(this.Download(item.SourceObject, item)); }
private static List<TransferItem> GetTransformItemsForAllDirections(bool resume) { List<TransferItem> allItems = new List<TransferItem>(); foreach (DMLibTransferDirection direction in GetAllValidDirections()) { if (resume && (direction.SourceType == DMLibDataType.Stream || direction.DestType == DMLibDataType.Stream)) { continue; } string fileName = GetTransferFileName(direction); DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); FileNode fileNode = singleObjectNodes[fileName]; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode), DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode), SourceType = direction.SourceType, DestType = direction.DestType, IsServiceCopy = direction.IsAsync, }; allItems.Add(item); } foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections()) { string dirName = GetTransferDirName(direction); DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType); DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType); DirNode dirNode = directoryNodes[dirName]; dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType); options.Recursive = true; TransferItem item = new TransferItem() { SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode), DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode), SourceType = direction.SourceType, DestType = direction.DestType, IsServiceCopy = direction.IsAsync, IsDirectoryTransfer = true, Options = options, }; allItems.Add(item); } return allItems; }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); secondCheckpoint = transferContext.LastCheckpoint; Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled."); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; ProgressChecker firstProgressChecker = null, secondProgressChecker = null; // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.IsAsync)) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // first progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { // The destination is already created, will cause a transfer skip firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob)) { firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else { firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024); } // second progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream) { // The destination is already created, will cause a transfer skip secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream) { secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync) { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024); } else { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024); } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); TransferContext resumeContext = new TransferContext( IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file"); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); resumeContext = new TransferContext( IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = secondProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file"); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync) { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } }
public void TestResume() { int fileSizeInKB = 100 * 1024; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; bool IsStreamJournal = random.Next(0, 2) == 0; using (Stream journalStream = new MemoryStream()) { TransferContext transferContext = IsStreamJournal ? new SingleTransferContext(journalStream) : new SingleTransferContext(); var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); options.TransferItemModifier = (fileName, item) => { dynamic transferOptions = DefaultTransferOptions; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { transferOptions.PreserveSMBAttributes = true; transferOptions.PreserveSMBPermissions = true; } item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; item.DisableStreamDispose = true; item.Options = transferOptions; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { if (IsStreamJournal && (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)) { return; } // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint if (!IsStreamJournal) { firstCheckpoint = transferContext.LastCheckpoint; } Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; if ((DMLibTestContext.SourceType == DMLibDataType.Stream) || (DMLibTestContext.DestType == DMLibDataType.Stream)) { options.DisableDestinationFetch = true; } // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); if (!IsStreamJournal) { secondCheckpoint = transferContext.LastCheckpoint; } else { if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify job is failed"); Exception jobException = result.Exceptions[0]; Test.Info("{0}", jobException); VerificationHelper.VerifyExceptionErrorMessage(jobException, "Cannot deserialize to TransferLocation when its TransferLocationType is Stream."); transferItem.DisableStreamDispose = false; transferItem.CloseStreamIfNecessary(); return; } } Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled"); Exception exception = result.Exceptions[0]; Helper.VerifyCancelException(exception); TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; ProgressChecker firstProgressChecker = null, secondProgressChecker = null; if (!IsStreamJournal) { // DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with // the actual transfer progress when the destination is an append blob. if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || (DMLibTestContext.CopyMethod == DMLibCopyMethod.ServiceSideAsyncCopy))) { Test.Info("Resume with the first checkpoint first."); firstResumeCheckpoint = firstCheckpoint; secondResumeCheckpoint = secondCheckpoint; } else { Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } } // first progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob) { // The destination is already created, will cause a transfer skip firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob)) { firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else { firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024); } // second progress checker if (DMLibTestContext.SourceType == DMLibDataType.Stream) { // The destination is already created, will cause a transfer skip secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.Stream) { secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024); } else { secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024); } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); TransferContext resumeContext = null; if (IsStreamJournal) { Exception deserializeEX = null; try { resumeContext = new SingleTransferContext(journalStream) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; } catch (Exception ex) { if ((DMLibTestContext.SourceType != DMLibDataType.Stream) && (DMLibTestContext.DestType != DMLibDataType.Stream)) { Test.Error("Should no exception in deserialization when no target is stream."); } deserializeEX = ex; } } else { resumeContext = new SingleTransferContext(IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = firstProgressChecker.GetProgressHandler() }; } resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported"); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } if (!IsStreamJournal) { // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); resumeContext = new SingleTransferContext( IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = secondProgressChecker.GetProgressHandler() }; resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream) { Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream."); exception = result.Exceptions[0]; VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported"); } else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); exception = result.Exceptions[0]; Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob."); VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application."); } else { // For sync copy, recalculate md5 of destination by downloading the file to local. if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy)) { DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor); } VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo); } } if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( sourceDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } } }
public Task DoTransfer(TransferItem item) { Test.Info("Do transfer: {0}", item.ToString()); return this.DoTransferImp(item); }
private void ResumeInAllDirectionsHelper(bool directoryTransfer) { List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true) : AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true); int fileCount = expectedFileNodes.Keys.Count; // Execution and store checkpoints CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferContext transferContext = null; if (directoryTransfer) { transferContext = new DirectoryTransferContext(); } else { transferContext = new SingleTransferContext(); } var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); allItems.ForEach(item => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; }); var options = new TestExecutionOptions <DMLibDataInfo>(); options.DisableDestinationFetch = true; // Checkpoint names const string PartialStarted = "PartialStarted", AllStarted = "AllStarted", AllStartedAndWait = "AllStartedAndWait", BeforeCancel = "BeforeCancel", AfterCancel = "AfterCancel"; Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>(); TransferItem randomItem = allItems[random.Next(0, allItems.Count)]; randomItem.AfterStarted = () => { Test.Info("Store check point after transfer item: {0}.", randomItem.ToString()); checkpoints.Add(PartialStarted, transferContext.LastCheckpoint); }; options.AfterAllItemAdded = () => { if (!progressChecker.DataTransferred.WaitOne(30000)) { Test.Error("No progress in 30s."); } checkpoints.Add(AllStarted, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint); tokenSource.Cancel(); checkpoints.Add(AfterCancel, transferContext.LastCheckpoint); }; var result = this.RunTransferItems(allItems, options); // Resume with stored checkpoints in random order var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >(); checkpointList.AddRange(checkpoints); checkpointList.Shuffle(); foreach (var pair in checkpointList) { Test.Info("===Resume with checkpoint '{0}'===", pair.Key); options = new TestExecutionOptions <DMLibDataInfo>(); options.DisableDestinationFetch = true; progressChecker.Reset(); if (directoryTransfer) { transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value)) { ProgressHandler = progressChecker.GetProgressHandler(), // The checkpoint can be stored when DMLib doesn't check overwrite callback yet. // So it will case an skip file error if the desination file already exists and // We don't have overwrite callback here. ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY() }; } else { transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value)) { ProgressHandler = progressChecker.GetProgressHandler(), // The checkpoint can be stored when DMLib doesn't check overwrite callback yet. // So it will case an skip file error if the desination file already exists and // We don't have overwrite callback here. ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY() }; } int expectedFailureCount = 0; transferContext.FileFailed += (resource, eventArgs) => { TransferException exception = eventArgs.Exception as TransferException; if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId) { Interlocked.Increment(ref expectedFailureCount); } }; TransferEventChecker eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); List <TransferItem> itemsToResume = allItems.Select(item => { TransferItem itemToResume = item.Clone(); itemToResume.TransferContext = transferContext; return(itemToResume); }).ToList(); result = this.RunTransferItems(itemsToResume, options); foreach (DMLibDataType destDataType in DataTypes) { if (DMLibDataType.URI == destDataType) { continue; } DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType); DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { string fileName = destFileNode.Name; FileNode sourceFileNode = expectedFileNodes[fileName]; Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result."); } } if (!directoryTransfer) { Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count); } else { Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count); Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber); } } }
protected override Task DoTransferImp(TransferItem item) { return(this.Copy(item.SourceObject, item.DestObject, item)); }
public TestResult <DMLibDataInfo> RunTransferItems(List <TransferItem> items, TestExecutionOptions <DMLibDataInfo> options) { Dictionary <TransferItem, Task <TransferStatus> > allTasks = new Dictionary <TransferItem, Task <TransferStatus> >(); var testResult = new TestResult <DMLibDataInfo>(); testResult.TransferItems = items; try { foreach (TransferItem item in items) { DMLibWrapper wrapper = GetDMLibWrapper(item.SourceType, item.DestType, DMLibTestContext.IsAsync); if (item.BeforeStarted != null) { item.BeforeStarted(); } try { if (options.LimitSpeed) { OperationContext.GlobalSendingRequest += this.LimitSpeed; TransferManager.Configurations.ParallelOperations = DMLibTestConstants.LimitedSpeedNC; } if (options.BlockSize.HasValue) { TransferManager.Configurations.BlockSize = options.BlockSize.Value; } allTasks.Add(item, wrapper.DoTransfer(item)); } catch (Exception e) { testResult.AddException(e); } if (item.AfterStarted != null) { item.AfterStarted(); } } if (options.AfterAllItemAdded != null) { options.AfterAllItemAdded(); } try { Task.WaitAll(allTasks.Values.ToArray(), options.TimeoutInMs); } catch (Exception e) { AggregateException ae = e as AggregateException; if (ae != null) { ae = ae.Flatten(); foreach (var innerE in ae.InnerExceptions) { testResult.AddException(innerE); } } else { testResult.AddException(e); } } } finally { if (options.LimitSpeed) { OperationContext.GlobalSendingRequest -= this.LimitSpeed; TransferManager.Configurations.ParallelOperations = DMLibTestConstants.DefaultNC; } TransferManager.Configurations.BlockSize = DMLibTestConstants.DefaultBlockSize; } Parallel.ForEach(allTasks, pair => { TransferItem transferItem = pair.Key; Task <TransferStatus> task = pair.Value; transferItem.CloseStreamIfNecessary(); try { transferItem.FinalStatus = task.Result; } catch (Exception e) { transferItem.Exception = e; } }); if (!options.DisableDestinationFetch) { testResult.DataInfo = DestAdaptor.GetTransferDataInfo(string.Empty); } foreach (var exception in testResult.Exceptions) { Test.Info("Exception from DMLib: {0}", exception.ToString()); } return(testResult); }
public void TestDirectoryResume() { int bigFileSizeInKB = 5 * 1024; // 5 MB int smallFileSizeInKB = 1; // 1 KB int bigFileNum = 5; int smallFileNum = 50; long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024; int totalFileNum = bigFileNum + smallFileNum; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode bigFileDirNode = new DirNode("big"); DirNode smallFileDirNode = new DirNode("small"); sourceDataInfo.RootNode.AddDirNode(bigFileDirNode); sourceDataInfo.RootNode.AddDirNode(smallFileDirNode); DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB); DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; options.IsDirectoryTransfer = true; using (Stream journalStream = new MemoryStream()) { bool IsStreamJournal = random.Next(0, 2) == 0; var transferContext = IsStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext(); var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); var eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); transferContext.FileFailed += (sender, e) => { Helper.VerifyCancelException(e.Exception); }; options.TransferItemModifier = (fileName, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { dirOptions.PreserveSMBAttributes = true; dirOptions.PreserveSMBPermissions = true; } item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); if (!IsStreamJournal) { // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; } Thread.Sleep(1000); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); if (progressChecker.FailedFilesNumber <= 0) { Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber); } TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; if (!IsStreamJournal) { secondCheckpoint = transferContext.LastCheckpoint; Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; } // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = null; if (IsStreamJournal) { resumeContext = new DirectoryTransferContext(journalStream) { ProgressHandler = progressChecker.GetProgressHandler() }; } else { resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler() }; } eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); if (!IsStreamJournal) { // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler(), // Need this overwrite callback since some files is already transferred to destination ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(), }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); } if (DMLibTestContext.SourceType == DMLibDataType.CloudFile && DMLibTestContext.DestType == DMLibDataType.CloudFile) { Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true); Helper.CompareSMBPermissions( sourceDataInfo.RootNode, result.DataInfo.RootNode, PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL); } } }
public void TestDirectoryResume() { int bigFileSizeInKB = 5 * 1024; // 5 MB int smallFileSizeInKB = 1; // 1 KB int bigFileNum = 5; int smallFileNum = 50; long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024; int totalFileNum = bigFileNum + smallFileNum; DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); DirNode bigFileDirNode = new DirNode("big"); DirNode smallFileDirNode = new DirNode("small"); sourceDataInfo.RootNode.AddDirNode(bigFileDirNode); sourceDataInfo.RootNode.AddDirNode(smallFileDirNode); DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB); DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB); CancellationTokenSource tokenSource = new CancellationTokenSource(); TransferItem transferItem = null; var options = new TestExecutionOptions <DMLibDataInfo>(); options.LimitSpeed = true; options.IsDirectoryTransfer = true; var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); var eventChecker = new TransferEventChecker(); eventChecker.Apply(transferContext); transferContext.FileFailed += (sender, e) => { Test.Assert(e.Exception.Message.Contains("cancel"), "Verify task is canceled: {0}", e.Exception.Message); }; options.TransferItemModifier = (fileName, item) => { dynamic dirOptions = DefaultTransferDirectoryOptions; dirOptions.Recursive = true; item.Options = dirOptions; item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; transferItem = item; }; TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null; options.AfterAllItemAdded = () => { // Wait until there are data transferred progressChecker.DataTransferred.WaitOne(); // Store the first checkpoint firstCheckpoint = transferContext.LastCheckpoint; Thread.Sleep(100); // Cancel the transfer and store the second checkpoint tokenSource.Cancel(); }; // Cancel and store checkpoint for resume var result = this.ExecuteTestCase(sourceDataInfo, options); secondCheckpoint = transferContext.LastCheckpoint; if (progressChecker.FailedFilesNumber <= 0) { Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber); } TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null; Test.Info("Resume with the second checkpoint first."); firstResumeCheckpoint = secondCheckpoint; secondResumeCheckpoint = firstCheckpoint; // resume with firstResumeCheckpoint TransferItem resumeItem = transferItem.Clone(); progressChecker.Reset(); TransferContext resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler() }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); // resume with secondResumeCheckpoint resumeItem = transferItem.Clone(); progressChecker.Reset(); resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint)) { ProgressHandler = progressChecker.GetProgressHandler(), // Need this overwrite callback since some files is already transferred to destination OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(), }; eventChecker.Reset(); eventChecker.Apply(resumeContext); resumeItem.TransferContext = resumeContext; result = this.RunTransferItems(new List <TransferItem>() { resumeItem }, new TestExecutionOptions <DMLibDataInfo>()); VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0); VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes); VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo); }
protected override Task DoTransferImp(TransferItem item) { return this.Download(item.SourceObject, item); }
protected abstract Task <TransferStatus> DoTransferImp(TransferItem item);
public void ResumeInAllDirections() { long fileSizeInByte = 10 * 1024 * 1024; Dictionary <string, FileNode> sourceFileNodes = this.PrepareSourceData(fileSizeInByte); List <TransferItem> allItems = this.GetTransformItemsForAllDirections(sourceFileNodes); int fileCount = sourceFileNodes.Keys.Count; // Execution and store checkpoints CancellationTokenSource tokenSource = new CancellationTokenSource(); var transferContext = new TransferContext(); var progressChecker = new ProgressChecker(fileCount, fileSizeInByte * fileCount); transferContext.ProgressHandler = progressChecker.GetProgressHandler(); allItems.ForEach(item => { item.CancellationToken = tokenSource.Token; item.TransferContext = transferContext; }); var options = new TestExecutionOptions <DMLibDataInfo>(); options.DisableDestinationFetch = true; // Checkpoint names const string PartialStarted = "PartialStarted", AllStarted = "AllStarted", AllStartedAndWait = "AllStartedAndWait", BeforeCancel = "BeforeCancel", AfterCancel = "AfterCancel"; Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>(); TransferItem randomItem = allItems[random.Next(0, allItems.Count)]; randomItem.AfterStarted = () => { Test.Info("Store check point after transfer item: {0}.", randomItem.ToString()); checkpoints.Add(PartialStarted, transferContext.LastCheckpoint); }; options.AfterAllItemAdded = () => { progressChecker.DataTransferred.WaitOne(); checkpoints.Add(AllStarted, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint); Thread.Sleep(1000); checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint); tokenSource.Cancel(); checkpoints.Add(AfterCancel, transferContext.LastCheckpoint); }; var result = this.RunTransferItems(allItems, options); // Resume with stored checkpoints in random order var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >(); checkpointList.AddRange(checkpoints); checkpointList.Shuffle(); foreach (var pair in checkpointList) { Test.Info("===Resume with checkpoint '{0}'===", pair.Key); options = new TestExecutionOptions <DMLibDataInfo>(); options.DisableDestinationFetch = true; progressChecker.Reset(); transferContext = new TransferContext(pair.Value) { ProgressHandler = progressChecker.GetProgressHandler(), // The checkpoint can be stored when DMLib doesn't check overwrite callback yet. // So it will case an skip file error if the desination file already exists and // We don't have overwrite callback here. OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY() }; List <TransferItem> itemsToResume = allItems.Select(item => { TransferItem itemToResume = item.Clone(); itemToResume.TransferContext = transferContext; return(itemToResume); }).ToList(); result = this.RunTransferItems(itemsToResume, options); int resumeFailCount = 0; foreach (DMLibDataType destDataType in DataTypes) { DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType); DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty); foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes()) { string fileName = destFileNode.Name; if (!fileName.Contains(DMLibDataType.Stream.ToString())) { FileNode sourceFileNode = sourceFileNodes[fileName]; Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result."); } else { resumeFailCount++; } } } Test.Assert(result.Exceptions.Count == resumeFailCount, "Verify resume failure count: expected {0}, actual {1}.", resumeFailCount, result.Exceptions.Count); foreach (var resumeException in result.Exceptions) { Test.Assert(resumeException is NotSupportedException, "Verify resume exception is NotSupportedException."); } } }
private Task <TransferStatus> CopyDirectory(dynamic sourceObject, dynamic destObject, TransferItem item) { CopyDirectoryOptions copyDirectoryOptions = item.Options as CopyDirectoryOptions; DirectoryTransferContext transferContext = item.TransferContext as DirectoryTransferContext; CancellationToken cancellationToken = item.CancellationToken; if (cancellationToken == null || cancellationToken == CancellationToken.None) { return(TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext)); } else { return(TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext, cancellationToken)); } }
protected override Task DoTransferImp(TransferItem item) { return this.Upload(item.DestObject, item); }