private static List <TransferItem> GetTransformItemsForAllSingleTransferDirections(bool resume)
{
List <TransferItem> allItems = new List <TransferItem>();
foreach (DMLibTransferDirection direction in GetAllValidDirections())
{
if (resume && (direction.SourceType == DMLibDataType.Stream || direction.DestType == DMLibDataType.Stream))
{
continue;
}
string fileName = GetTransferFileName(direction);
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
FileNode fileNode = singleObjectNodes[fileName];
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
CopyMethod = direction.CopyMethod,
TransferContext = new SingleTransferContext()
{
SetAttributesCallbackAsync = AllTransferDirectionTest.SetAttributesCallbackMethodAsync
}
};
allItems.Add(item);
}
return(allItems);
}
private static List <TransferItem> GetTransformItemsForAllDirTransferDirections(bool resume)
{
List <TransferItem> allItems = new List <TransferItem>();
foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections())
{
string dirName = GetTransferDirName(direction);
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
DirNode dirNode = directoryNodes[dirName];
dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType);
options.Recursive = true;
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
CopyMethod = direction.CopyMethod,
IsDirectoryTransfer = true,
Options = options,
TransferContext = new DirectoryTransferContext()
{
SetAttributesCallbackAsync = AllTransferDirectionTest.SetAttributesCallbackMethodAsync
}
};
allItems.Add(item);
}
return(allItems);
}
protected override Task DoTransferImp(TransferItem item)
{
if (item.IsDirectoryTransfer)
{
return(this.DownloadDirectory(item.SourceObject, item));
}
else
{
return(this.Download(item.SourceObject, item));
}
}
protected override Task DoTransferImp(TransferItem item)
{
if (item.IsDirectoryTransfer)
{
return this.DownloadDirectory(item.SourceObject, item);
}
else
{
return this.Download(item.SourceObject, item);
}
}
private static List <TransferItem> GetTransformItemsForAllDirections(bool resume)
{
List <TransferItem> allItems = new List <TransferItem>();
foreach (DMLibTransferDirection direction in GetAllValidDirections())
{
if (resume && (direction.SourceType == DMLibDataType.Stream || direction.DestType == DMLibDataType.Stream))
{
continue;
}
string fileName = GetTransferFileName(direction);
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
FileNode fileNode = singleObjectNodes[fileName];
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
IsServiceCopy = direction.IsAsync,
};
allItems.Add(item);
}
foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections())
{
string dirName = GetTransferDirName(direction);
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
DirNode dirNode = directoryNodes[dirName];
dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType);
options.Recursive = true;
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
IsServiceCopy = direction.IsAsync,
IsDirectoryTransfer = true,
Options = options,
};
allItems.Add(item);
}
return(allItems);
}
protected override Task DoTransferImp(TransferItem item)
{
if (item.IsDirectoryTransfer)
{
return this.CopyDirectory(item.SourceObject, item.DestObject, item);
}
else
{
return this.Copy(item.SourceObject, item.DestObject, item);
}
}
protected override async Task <TransferStatus> DoTransferImp(TransferItem item)
{
if (item.IsDirectoryTransfer)
{
return(await this.UploadDirectory(item.DestObject, item).ConfigureAwait(false));
}
else
{
await this.Upload(item.DestObject, item).ConfigureAwait(false);
return(null);
}
}
protected override async Task <TransferStatus> DoTransferImp(TransferItem item)
{
if (item.IsDirectoryTransfer)
{
return(await this.CopyDirectory(item.SourceObject, item.DestObject, item));
}
else
{
await this.Copy(item.SourceObject, item.DestObject, item);
return(null);
}
}
public TransferItem Clone()
{
TransferItem newTransferItem = new TransferItem()
{
SourceObject = NewLocationObject(this.SourceObject),
DestObject = NewLocationObject(this.DestObject),
SourceType = this.SourceType,
DestType = this.DestType,
IsServiceCopy = this.IsServiceCopy,
Options = this.Options,
};
return(newTransferItem);
}
public void Upload_Symlinked_RootDir()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
DMLibDataInfo targetDataInfo = new DMLibDataInfo("target");
UnicodeFileName = "TempTestName";
DirNode dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
targetDataInfo.RootNode.AddDirNode(dirNode);
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
SourceAdaptor.GenerateData(targetDataInfo);
sourceDataInfo.RootNode = DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", "target", targetDataInfo.RootNode);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferItem item = new TransferItem()
{
SourceObject = Path.Combine(SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as string, sourceDataInfo.RootNode.Name),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = new DirectoryTransferContext(),
Options = DefaultTransferDirectoryOptions
};
(item.Options as UploadDirectoryOptions).Recursive = true;
var result = this.RunTransferItems(new List <TransferItem>()
{
item
}, new TestExecutionOptions <DMLibDataInfo>());
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public TransferItem Clone()
{
TransferItem newTransferItem = new TransferItem()
{
SourceObject = NewLocationObject(this.SourceObject),
DestObject = NewLocationObject(this.DestObject),
SourceType = this.SourceType,
DestType = this.DestType,
IsDirectoryTransfer = this.IsDirectoryTransfer,
CopyMethod = this.CopyMethod,
Options = this.Options,
};
return(newTransferItem);
}
private Task <TransferStatus> DownloadDirectory(dynamic sourceObject, TransferItem item)
{
DownloadDirectoryOptions downloadDirectoryOptions = item.Options as DownloadDirectoryOptions;
DirectoryTransferContext transferContext = item.TransferContext as DirectoryTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string destPath = item.DestObject as string;
if (cancellationToken == null || cancellationToken == CancellationToken.None)
{
return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext));
}
else
{
return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext, cancellationToken));
}
}
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options)
{
this.CleanupData();
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
if (sourceDataInfo != null)
{
SourceAdaptor.GenerateData(sourceDataInfo);
}
if (options.DestTransferDataInfo != null)
{
DestAdaptor.GenerateData(options.DestTransferDataInfo);
}
if (options.AfterDataPrepared != null)
{
options.AfterDataPrepared();
}
List <TransferItem> allItems = new List <TransferItem>();
foreach (var fileNode in sourceDataInfo.EnumerateFileNodes())
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(fileNode),
DestObject = DestAdaptor.GetTransferObject(fileNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(fileNode, item);
}
allItems.Add(item);
}
return(this.RunTransferItems(allItems, options));
}
private Task CopyDirectory(dynamic sourceObject, dynamic destObject, TransferItem item)
{
CopyDirectoryOptions copyDirectoryOptions = item.Options as CopyDirectoryOptions;
TransferContext transferContext = item.TransferContext;
CancellationToken cancellationToken = item.CancellationToken;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
return TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext, cancellationToken);
}
else if (transferContext != null || copyDirectoryOptions != null)
{
return TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext);
}
else
{
return TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy);
}
}
private Task Copy(dynamic sourceObject, dynamic destObject, TransferItem item)
{
CopyOptions copyOptions = item.Options as CopyOptions;
SingleTransferContext transferContext = item.TransferContext as SingleTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy, copyOptions, transferContext, cancellationToken));
}
else if (transferContext != null || copyOptions != null)
{
return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy, copyOptions, transferContext));
}
else
{
return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy));
}
}
private Task Download(dynamic sourceObject, TransferItem item)
{
DownloadOptions downloadOptions = item.Options as DownloadOptions;
SingleTransferContext transferContext = item.TransferContext as SingleTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string destPath = item.DestObject as string;
Stream destStream = item.DestObject as Stream;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
if (destPath != null)
{
return(TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext, cancellationToken));
}
else
{
return(TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext, cancellationToken));
}
}
else if (transferContext != null || downloadOptions != null)
{
if (destPath != null)
{
return(TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext));
}
else
{
return(TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext));
}
}
else
{
if (destPath != null)
{
return(TransferManager.DownloadAsync(sourceObject, destPath));
}
else
{
return(TransferManager.DownloadAsync(sourceObject, destStream));
}
}
}
private Task Upload(dynamic destObject, TransferItem item)
{
UploadOptions uploadOptions = item.Options as UploadOptions;
TransferContext transferContext = item.TransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string sourcePath = item.SourceObject as string;
Stream sourceStream = item.SourceObject as Stream;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
if (sourcePath != null)
{
return TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext, cancellationToken);
}
else
{
return TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext, cancellationToken);
}
}
else if (transferContext != null || uploadOptions != null)
{
if (sourcePath != null)
{
return TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext);
}
else
{
return TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext);
}
}
else
{
if (sourcePath != null)
{
return TransferManager.UploadAsync(sourcePath, destObject);
}
else
{
return TransferManager.UploadAsync(sourceStream, destObject);
}
}
}
private Task Upload(dynamic destObject, TransferItem item)
{
UploadOptions uploadOptions = item.Options as UploadOptions;
TransferContext transferContext = item.TransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string sourcePath = item.SourceObject as string;
Stream sourceStream = item.SourceObject as Stream;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
if (sourcePath != null)
{
return(TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext, cancellationToken));
}
else
{
return(TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext, cancellationToken));
}
}
else if (transferContext != null || uploadOptions != null)
{
if (sourcePath != null)
{
return(TransferManager.UploadAsync(sourcePath, destObject, uploadOptions, transferContext));
}
else
{
return(TransferManager.UploadAsync(sourceStream, destObject, uploadOptions, transferContext));
}
}
else
{
if (sourcePath != null)
{
return(TransferManager.UploadAsync(sourcePath, destObject));
}
else
{
return(TransferManager.UploadAsync(sourceStream, destObject));
}
}
}
private Task Download(dynamic sourceObject, TransferItem item)
{
DownloadOptions downloadOptions = item.Options as DownloadOptions;
TransferContext transferContext = item.TransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string destPath = item.DestObject as string;
Stream destStream = item.DestObject as Stream;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
if (destPath != null)
{
return TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext, cancellationToken);
}
else
{
return TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext, cancellationToken);
}
}
else if (transferContext != null || downloadOptions != null)
{
if (destPath != null)
{
return TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext);
}
else
{
return TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext);
}
}
else
{
if (destPath != null)
{
return TransferManager.DownloadAsync(sourceObject, destPath);
}
else
{
return TransferManager.DownloadAsync(sourceObject, destStream);
}
}
}
private Task <TransferStatus> UploadDirectory(dynamic destObject, TransferItem item)
{
UploadDirectoryOptions uploadDirectoryOptions = item.Options as UploadDirectoryOptions;
TransferContext transferContrext = item.TransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string sourcePath = item.SourceObject as string;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext, cancellationToken));
}
else if (transferContrext != null || uploadDirectoryOptions != null)
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext));
}
else
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject));
}
}
private List <TransferItem> GetTransformItemsForAllDirections(Dictionary <string, FileNode> fileNodes)
{
List <TransferItem> allItems = new List <TransferItem>();
foreach (DMLibTransferDirection direction in GetAllValidDirections())
{
string fileName = GetTransferFileName(direction);
DataAdaptor <DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
FileNode fileNode = fileNodes[fileName];
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(fileNode),
DestObject = destAdaptor.GetTransferObject(fileNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
IsServiceCopy = direction.IsAsync,
};
allItems.Add(item);
}
return(allItems);
}
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options)
{
if (options.DisableSourceCleaner)
{
this.CleanupData(false, true);
}
else
{
this.CleanupData();
}
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
string sourceRootPath = string.Empty;
DirNode sourceRootNode = new DirNode(string.Empty);
if (sourceDataInfo != null)
{
sourceRootPath = sourceDataInfo.RootPath;
sourceRootNode = sourceDataInfo.RootNode;
if (!options.DisableSourceGenerator)
{
SourceAdaptor.GenerateData(sourceDataInfo);
}
}
string destRootPath = string.Empty;
if (options.DestTransferDataInfo != null)
{
destRootPath = options.DestTransferDataInfo.RootPath;
DestAdaptor.GenerateData(options.DestTransferDataInfo);
}
if (options.AfterDataPrepared != null)
{
options.AfterDataPrepared();
}
List <TransferItem> allItems = new List <TransferItem>();
if (options.IsDirectoryTransfer)
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceRootPath, sourceRootNode, options.SourceCredentials),
DestObject = DestAdaptor.GetTransferObject(destRootPath, sourceRootNode, options.DestCredentials),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
IsDirectoryTransfer = true,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(null, item);
}
allItems.Add(item);
}
else
{
foreach (var fileNode in sourceDataInfo.EnumerateFileNodes())
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, fileNode, options.SourceCredentials),
DestObject = DestAdaptor.GetTransferObject(destRootPath, fileNode, options.DestCredentials),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(fileNode, item);
}
allItems.Add(item);
}
}
return(this.RunTransferItems(allItems, options));
}
public TransferItem Clone()
{
TransferItem newTransferItem = new TransferItem()
{
SourceObject = NewLocationObject(this.SourceObject),
DestObject = NewLocationObject(this.DestObject),
SourceType = this.SourceType,
DestType = this.DestType,
IsDirectoryTransfer = this.IsDirectoryTransfer,
IsServiceCopy = this.IsServiceCopy,
Options = this.Options,
};
return newTransferItem;
}
protected abstract Task DoTransferImp(TransferItem item);
public Task DoTransfer(TransferItem item)
{
Test.Info("Do transfer: {0}", item.ToString());
return(this.DoTransferImp(item));
}
private List<TransferItem> GetTransformItemsForAllDirections(Dictionary<string, FileNode> fileNodes)
{
List<TransferItem> allItems = new List<TransferItem>();
foreach (DMLibTransferDirection direction in GetAllValidDirections())
{
string fileName = GetTransferFileName(direction);
DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
FileNode fileNode = fileNodes[fileName];
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(fileNode),
DestObject = destAdaptor.GetTransferObject(fileNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
IsServiceCopy = direction.IsAsync,
};
allItems.Add(item);
}
return allItems;
}
protected override Task DoTransferImp(TransferItem item)
{
return(this.Download(item.SourceObject, item));
}
private static List<TransferItem> GetTransformItemsForAllDirections(bool resume)
{
List<TransferItem> allItems = new List<TransferItem>();
foreach (DMLibTransferDirection direction in GetAllValidDirections())
{
if (resume && (direction.SourceType == DMLibDataType.Stream || direction.DestType == DMLibDataType.Stream))
{
continue;
}
string fileName = GetTransferFileName(direction);
DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
FileNode fileNode = singleObjectNodes[fileName];
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, fileNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, fileNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
IsServiceCopy = direction.IsAsync,
};
allItems.Add(item);
}
foreach (DMLibTransferDirection direction in GetAllDirectoryValidDirections())
{
string dirName = GetTransferDirName(direction);
DataAdaptor<DMLibDataInfo> sourceAdaptor = GetSourceAdaptor(direction.SourceType);
DataAdaptor<DMLibDataInfo> destAdaptor = GetDestAdaptor(direction.DestType);
DirNode dirNode = directoryNodes[dirName];
dynamic options = DMLibTestBase.GetDefaultTransferDirectoryOptions(direction.SourceType, direction.DestType);
options.Recursive = true;
TransferItem item = new TransferItem()
{
SourceObject = sourceAdaptor.GetTransferObject(string.Empty, dirNode),
DestObject = destAdaptor.GetTransferObject(string.Empty, dirNode),
SourceType = direction.SourceType,
DestType = direction.DestType,
IsServiceCopy = direction.IsAsync,
IsDirectoryTransfer = true,
Options = options,
};
allItems.Add(item);
}
return allItems;
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
secondCheckpoint = transferContext.LastCheckpoint;
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled.");
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
ProgressChecker firstProgressChecker = null, secondProgressChecker = null;
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.IsAsync))
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// first progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
// The destination is already created, will cause a transfer skip
firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob))
{
firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else
{
firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024);
}
// second progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
// The destination is already created, will cause a transfer skip
secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream)
{
secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync)
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024);
}
else
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024);
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
TransferContext resumeContext = new TransferContext(
IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
resumeContext = new TransferContext(
IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = secondProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file");
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync)
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
TransferContext transferContext = IsStreamJournal ? new SingleTransferContext(journalStream) : new SingleTransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
dynamic transferOptions = DefaultTransferOptions;
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
transferOptions.PreserveSMBAttributes = true;
transferOptions.PreserveSMBPermissions = true;
}
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
item.DisableStreamDispose = true;
item.Options = transferOptions;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
if (IsStreamJournal &&
(DMLibTestContext.SourceType == DMLibDataType.Stream ||
DMLibTestContext.DestType == DMLibDataType.Stream))
{
return;
}
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
if (!IsStreamJournal)
{
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
if ((DMLibTestContext.SourceType == DMLibDataType.Stream) || (DMLibTestContext.DestType == DMLibDataType.Stream))
{
options.DisableDestinationFetch = true;
}
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (!IsStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
}
else
{
if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify job is failed");
Exception jobException = result.Exceptions[0];
Test.Info("{0}", jobException);
VerificationHelper.VerifyExceptionErrorMessage(jobException, "Cannot deserialize to TransferLocation when its TransferLocationType is Stream.");
transferItem.DisableStreamDispose = false;
transferItem.CloseStreamIfNecessary();
return;
}
}
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
ProgressChecker firstProgressChecker = null, secondProgressChecker = null;
if (!IsStreamJournal)
{
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || (DMLibTestContext.CopyMethod == DMLibCopyMethod.ServiceSideAsyncCopy)))
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
}
// first progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
// The destination is already created, will cause a transfer skip
firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob))
{
firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else
{
firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024);
}
// second progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
// The destination is already created, will cause a transfer skip
secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream)
{
secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024);
}
else
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024);
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
TransferContext resumeContext = null;
if (IsStreamJournal)
{
Exception deserializeEX = null;
try
{
resumeContext = new SingleTransferContext(journalStream)
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
}
catch (Exception ex)
{
if ((DMLibTestContext.SourceType != DMLibDataType.Stream) &&
(DMLibTestContext.DestType != DMLibDataType.Stream))
{
Test.Error("Should no exception in deserialization when no target is stream.");
}
deserializeEX = ex;
}
}
else
{
resumeContext = new SingleTransferContext(IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
if (!IsStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
resumeContext = new SingleTransferContext(
IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = secondProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported");
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
}
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
Helper.CompareSMBPermissions(
sourceDataInfo.RootNode,
result.DataInfo.RootNode,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
}
}
public Task DoTransfer(TransferItem item)
{
Test.Info("Do transfer: {0}", item.ToString());
return this.DoTransferImp(item);
}
private void ResumeInAllDirectionsHelper(bool directoryTransfer)
{
List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true)
: AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferContext transferContext = null;
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext();
}
else
{
transferContext = new SingleTransferContext();
}
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
else
{
transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
int expectedFailureCount = 0;
transferContext.FileFailed += (resource, eventArgs) =>
{
TransferException exception = eventArgs.Exception as TransferException;
if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId)
{
Interlocked.Increment(ref expectedFailureCount);
}
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
if (DMLibDataType.URI == destDataType)
{
continue;
}
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
if (!directoryTransfer)
{
Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count);
}
else
{
Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count);
Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber);
}
}
}
protected override Task DoTransferImp(TransferItem item)
{
return(this.Copy(item.SourceObject, item.DestObject, item));
}
public TestResult <DMLibDataInfo> RunTransferItems(List <TransferItem> items, TestExecutionOptions <DMLibDataInfo> options)
{
Dictionary <TransferItem, Task <TransferStatus> > allTasks = new Dictionary <TransferItem, Task <TransferStatus> >();
var testResult = new TestResult <DMLibDataInfo>();
testResult.TransferItems = items;
try
{
foreach (TransferItem item in items)
{
DMLibWrapper wrapper = GetDMLibWrapper(item.SourceType, item.DestType, DMLibTestContext.IsAsync);
if (item.BeforeStarted != null)
{
item.BeforeStarted();
}
try
{
if (options.LimitSpeed)
{
OperationContext.GlobalSendingRequest += this.LimitSpeed;
TransferManager.Configurations.ParallelOperations = DMLibTestConstants.LimitedSpeedNC;
}
if (options.BlockSize.HasValue)
{
TransferManager.Configurations.BlockSize = options.BlockSize.Value;
}
allTasks.Add(item, wrapper.DoTransfer(item));
}
catch (Exception e)
{
testResult.AddException(e);
}
if (item.AfterStarted != null)
{
item.AfterStarted();
}
}
if (options.AfterAllItemAdded != null)
{
options.AfterAllItemAdded();
}
try
{
Task.WaitAll(allTasks.Values.ToArray(), options.TimeoutInMs);
}
catch (Exception e)
{
AggregateException ae = e as AggregateException;
if (ae != null)
{
ae = ae.Flatten();
foreach (var innerE in ae.InnerExceptions)
{
testResult.AddException(innerE);
}
}
else
{
testResult.AddException(e);
}
}
}
finally
{
if (options.LimitSpeed)
{
OperationContext.GlobalSendingRequest -= this.LimitSpeed;
TransferManager.Configurations.ParallelOperations = DMLibTestConstants.DefaultNC;
}
TransferManager.Configurations.BlockSize = DMLibTestConstants.DefaultBlockSize;
}
Parallel.ForEach(allTasks, pair =>
{
TransferItem transferItem = pair.Key;
Task <TransferStatus> task = pair.Value;
transferItem.CloseStreamIfNecessary();
try
{
transferItem.FinalStatus = task.Result;
}
catch (Exception e)
{
transferItem.Exception = e;
}
});
if (!options.DisableDestinationFetch)
{
testResult.DataInfo = DestAdaptor.GetTransferDataInfo(string.Empty);
}
foreach (var exception in testResult.Exceptions)
{
Test.Info("Exception from DMLib: {0}", exception.ToString());
}
return(testResult);
}
public void TestDirectoryResume()
{
int bigFileSizeInKB = 5 * 1024; // 5 MB
int smallFileSizeInKB = 1; // 1 KB
int bigFileNum = 5;
int smallFileNum = 50;
long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
int totalFileNum = bigFileNum + smallFileNum;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
options.IsDirectoryTransfer = true;
using (Stream journalStream = new MemoryStream())
{
bool IsStreamJournal = random.Next(0, 2) == 0;
var transferContext = IsStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext();
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Helper.VerifyCancelException(e.Exception);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
dirOptions.PreserveSMBAttributes = true;
dirOptions.PreserveSMBPermissions = true;
}
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
if (!IsStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = null;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (!IsStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
Helper.CompareSMBPermissions(
sourceDataInfo.RootNode,
result.DataInfo.RootNode,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
}
}
protected abstract Task DoTransferImp(TransferItem item);
public void TestDirectoryResume()
{
int bigFileSizeInKB = 5 * 1024; // 5 MB
int smallFileSizeInKB = 1; // 1 KB
int bigFileNum = 5;
int smallFileNum = 50;
long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
int totalFileNum = bigFileNum + smallFileNum;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
options.IsDirectoryTransfer = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Test.Assert(e.Exception.Message.Contains("cancel"), "Verify task is canceled: {0}", e.Exception.Message);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(100);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
secondCheckpoint = transferContext.LastCheckpoint;
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
protected override Task DoTransferImp(TransferItem item)
{
return this.Download(item.SourceObject, item);
}
protected abstract Task <TransferStatus> DoTransferImp(TransferItem item);
public void ResumeInAllDirections()
{
long fileSizeInByte = 10 * 1024 * 1024;
Dictionary <string, FileNode> sourceFileNodes = this.PrepareSourceData(fileSizeInByte);
List <TransferItem> allItems = this.GetTransformItemsForAllDirections(sourceFileNodes);
int fileCount = sourceFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(fileCount, fileSizeInByte * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
progressChecker.DataTransferred.WaitOne();
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
transferContext = new TransferContext(pair.Value)
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
int resumeFailCount = 0;
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
if (!fileName.Contains(DMLibDataType.Stream.ToString()))
{
FileNode sourceFileNode = sourceFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
else
{
resumeFailCount++;
}
}
}
Test.Assert(result.Exceptions.Count == resumeFailCount, "Verify resume failure count: expected {0}, actual {1}.", resumeFailCount, result.Exceptions.Count);
foreach (var resumeException in result.Exceptions)
{
Test.Assert(resumeException is NotSupportedException, "Verify resume exception is NotSupportedException.");
}
}
}
private Task <TransferStatus> CopyDirectory(dynamic sourceObject, dynamic destObject, TransferItem item)
{
CopyDirectoryOptions copyDirectoryOptions = item.Options as CopyDirectoryOptions;
DirectoryTransferContext transferContext = item.TransferContext as DirectoryTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
if (cancellationToken == null || cancellationToken == CancellationToken.None)
{
return(TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext));
}
else
{
return(TransferManager.CopyDirectoryAsync(sourceObject, destObject, item.IsServiceCopy, copyDirectoryOptions, transferContext, cancellationToken));
}
}
protected override Task DoTransferImp(TransferItem item)
{
return this.Upload(item.DestObject, item);
}
