public void TestProgressHandlerTest()
{
long fileSize = 10 * 1024 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize);
var options = new TestExecutionOptions<DMLibDataInfo>();
options.TransferItemModifier = (fileNode, transferItem) =>
{
TransferContext transferContext = new TransferContext();
ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
transferItem.TransferContext = transferContext;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public void TestProgressHandlerTest()
{
long fileSize = 10 * 1024 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSize);
var options = new TestExecutionOptions <DMLibDataInfo>();
options.TransferItemModifier = (fileNode, transferItem) =>
{
TransferContext transferContext = new SingleTransferContext();
ProgressChecker progressChecker = new ProgressChecker(1, fileNode.SizeInByte);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
transferItem.TransferContext = transferContext;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public void ResumeInAllDirections()
{
List<TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions<DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary<string, TransferCheckpoint> checkpoints = new Dictionary<string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
progressChecker.DataTransferred.WaitOne();
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List<KeyValuePair<string,TransferCheckpoint>>();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach(var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions<DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
transferContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List<TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return itemToResume;
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor<DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
Test.Assert(result.Exceptions.Count == 0, "Verify no error happens. Actual: {0}", result.Exceptions.Count);
}
}
public void TestDirectoryPreserveSMBPropertiesResume()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
CloudFileNtfsAttributes[] SMBFileAttributes =
{
CloudFileNtfsAttributes.ReadOnly,
CloudFileNtfsAttributes.Hidden,
#if DEBUG
CloudFileNtfsAttributes.System,
CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Normal,
CloudFileNtfsAttributes.Offline,
CloudFileNtfsAttributes.NotContentIndexed,
CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden,
CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly |
CloudFileNtfsAttributes.Hidden |
CloudFileNtfsAttributes.System |
CloudFileNtfsAttributes.Archive |
CloudFileNtfsAttributes.NotContentIndexed |
CloudFileNtfsAttributes.NoScrubData
#endif
};
for (int i = 0; i < SMBFileAttributes.Length; ++i)
{
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]);
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]);
DirectoryTransferContext dirTransferContext = null;
if (IsStreamJournal)
{
dirTransferContext = new DirectoryTransferContext(journalStream);
}
else
{
dirTransferContext = new DirectoryTransferContext();
}
var progressChecker = new ProgressChecker(14, 14 * 1024);
dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, item) =>
{
item.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.CancellationToken = tokenSource.Token;
transferItem = item;
};
TransferCheckpoint checkpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
checkpoint = dirTransferContext.LastCheckpoint;
}
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = (path) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
return(FileAttributes.Directory);
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory));
}
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i]));
}
};
TestHookCallbacks.SetFileAttributesCallback = (path, attributes) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot"))
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory),
"Directory attributes should be expected {0}", attributes);
}
}
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]),
"File attributes should be expected {0}", attributes);
}
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferItem resumeItem = transferItem.Clone();
DirectoryTransferContext resumeContext = null;
journalStream.Position = 0;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream);
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint));
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
IsDirectoryTransfer = true,
Options = DefaultTransferDirectoryOptions
};
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.TransferContext = new DirectoryTransferContext();
item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) =>
{
return(false);
};
List <TransferItem> items = new List <TransferItem>();
items.Add(item);
result = this.RunTransferItems(items, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.Local)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false);
}
else
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
}
}
finally
{
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = null;
TestHookCallbacks.SetFileAttributesCallback = null;
#endif
}
}
}
}
public void TestDirectoryPreserveSMBPermissionsResume()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
PreserveSMBPermissions preserveSMBPermissions =
PreserveSMBPermissions.Owner
| PreserveSMBPermissions.Group
| PreserveSMBPermissions.DACL;
string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)";
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithPermissions(sourceDataInfo.RootNode, 2, sampleSDDL);
DirectoryTransferContext dirTransferContext = null;
if (IsStreamJournal)
{
dirTransferContext = new DirectoryTransferContext(journalStream);
}
else
{
dirTransferContext = new DirectoryTransferContext();
}
var progressChecker = new ProgressChecker(14, 14 * 1024);
dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, item) =>
{
item.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBPermissions = preserveSMBPermissions;
item.Options = transferOptions;
item.CancellationToken = tokenSource.Token;
transferItem = item;
};
TransferCheckpoint checkpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
checkpoint = dirTransferContext.LastCheckpoint;
}
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
#if DEBUG
TestHookCallbacks.UnderTesting = true;
TestHookCallbacks.GetFilePermissionsCallback = (path, SMBPermissionType) =>
{
Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected.");
return(sampleSDDL);
};
TestHookCallbacks.SetFilePermissionsCallback = (path, portableSDDL, SMBPermissionType) =>
{
Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected.");
Test.Assert(portableSDDL.StartsWith(sampleSDDL),
"The SDDL value should be expected.");
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferItem resumeItem = transferItem.Clone();
DirectoryTransferContext resumeContext = null;
journalStream.Position = 0;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream);
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint));
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBPermissions(sourceDataInfo.RootNode, result.DataInfo.RootNode, preserveSMBPermissions);
}
}
finally
{
#if DEBUG
TestHookCallbacks.UnderTesting = false;
TestHookCallbacks.GetFilePermissionsCallback = null;
TestHookCallbacks.SetFilePermissionsCallback = null;
#endif
}
}
}
public void TestDirectoryCheckContentMD5()
{
long fileSize = 5 * 1024 * 1024;
long totalSize = fileSize * 4;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
TransferContext context = new TransferContext();
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
};
ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
context.FileFailed += (eventSource, eventArgs) =>
{
transferExceptions.Add(eventArgs.Exception);
};
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
};
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
DMLibDataInfo actualDataInfo = testResult.DataInfo;
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != 1)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5)
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string wrongMD5File = "wrongMD5File";
string correctMD5File = "correctMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
DestAdaptor.Cleanup();
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
using (var resumeStream = new MemoryStream())
{
TransferContext context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti,
totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = !checkMD5,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, executionOption);
if (null != testResult.Exceptions)
{
foreach (var exception in testResult.Exceptions)
{
Test.Info("Got exception during transferring. {0}", exception);
}
}
eventChecker = new TransferEventChecker();
resumeStream.Position = 0;
context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(wrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
checkMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected.");
VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0);
if (checkMD5)
{
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
else
{
Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5");
}
}
}
private void TestDirectorySetAttribute_Restart(
int bigFileSizeInKB,
int smallFileSizeInKB,
int bigFileNum,
int smallFileNum,
Action <DirNode> bigFileDirAddFileAction,
Action <DirNode> smallFileDirAddFileAction,
SetAttributesCallbackAsync setAttributesCallback = null,
Action <DMLibDataInfo> sourceDataInfoDecorator = null)
{
int totalFileNum = bigFileNum + smallFileNum;
long totalSizeInBytes = ((bigFileSizeInKB * bigFileNum) + (smallFileSizeInKB * smallFileNum)) * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
bigFileDirAddFileAction(bigFileDirNode);
smallFileDirAddFileAction(smallFileDirNode);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo> {
LimitSpeed = true, IsDirectoryTransfer = true
};
using (Stream journalStream = new MemoryStream())
{
bool isStreamJournal = random.Next(0, 2) == 0;
var transferContext = isStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext();
transferContext.SetAttributesCallbackAsync = setAttributesCallback;
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Helper.VerifyCancelException(e.Exception);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!isStreamJournal)
{
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
if (!isStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = null;
if (isStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
resumeContext.SetAttributesCallbackAsync = setAttributesCallback;
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
sourceDataInfoDecorator?.Invoke(sourceDataInfo);
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (!isStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
SetAttributesCallbackAsync = setAttributesCallback
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
}
}
public void TestDirectoryCheckContentMD5Resume()
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
// Prepare data for transfer items with disabling MD5 check
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
notCheckMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
TransferContext context = new DirectoryTransferContext();
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, executionOption);
eventChecker = new TransferEventChecker();
context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint));
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
notCheckMD5Item = notCheckMD5Item.Clone();
checkMD5Item.TransferContext = context;
notCheckMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions<DMLibDataInfo>();
options.LimitSpeed = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
secondCheckpoint = transferContext.LastCheckpoint;
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled.");
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && DMLibTestContext.DestType != DMLibDataType.AppendBlob)
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = new TransferContext(firstResumeCheckpoint)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new TransferContext(secondResumeCheckpoint)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
if (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.SourceType == DMLibDataType.Stream)
{
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
else
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
}
public void TestDirectoryCheckContentMD5()
{
long fileSize = 5 * 1024 * 1024;
long totalSize = fileSize * 4;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
TransferContext context = new TransferContext();
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
};
ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
};
var testResult = this.RunTransferItems(new List<TransferItem>() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions<DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
DMLibDataInfo actualDataInfo = testResult.DataInfo;
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1);
if (testResult.Exceptions.Count != 1)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
VerificationHelper.VerifyTransferException(testResult.Exceptions[0], TransferErrorCode.SubTransferFails, "1 sub transfer(s) failed.");
}
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions<DMLibDataInfo>();
options.LimitSpeed = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
secondCheckpoint = transferContext.LastCheckpoint;
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled.");
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
ProgressChecker firstProgressChecker = null, secondProgressChecker = null;
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.IsAsync))
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// first progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
// The destination is already created, will cause a transfer skip
firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob))
{
firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else
{
firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024);
}
// second progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
// The destination is already created, will cause a transfer skip
secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream)
{
secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync)
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024);
}
else
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024);
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
TransferContext resumeContext = new TransferContext(
IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
resumeContext = new TransferContext(
IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = secondProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file");
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync)
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
}
public void TestDirectoryResume()
{
int bigFileSizeInKB = 5 * 1024; // 5 MB
int smallFileSizeInKB = 1; // 1 KB
int bigFileNum = 5;
int smallFileNum = 50;
long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
int totalFileNum = bigFileNum + smallFileNum;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions<DMLibDataInfo>();
options.LimitSpeed = true;
options.IsDirectoryTransfer = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Test.Assert(e.Exception.Message.Contains("cancel"), "Verify task is canceled: {0}", e.Exception.Message);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
secondCheckpoint = transferContext.LastCheckpoint;
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new TransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List<TransferItem>() { resumeItem }, new TestExecutionOptions<DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}