public void TestFileShareSnapshotsCopyToBase()
{
int fileCount = 3;
string snapshotFile = "snapshotFile";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
sourceDataInfo.IsFileShareSnapshot = true;
DMLibDataHelper.AddMultipleFiles(sourceDataInfo.RootNode, snapshotFile, fileCount, 1024);
SourceAdaptor.Cleanup();
SourceAdaptor.CreateIfNotExists();
SourceAdaptor.GenerateData(sourceDataInfo);
CloudFileDataAdaptor fileAdaptor = (SourceAdaptor as CloudFileDataAdaptor);
CloudFileDirectory SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as CloudFileDirectory;
CloudFileDirectory DestObject = fileAdaptor.fileHelper.FileClient.GetShareReference(fileAdaptor.ShareName).GetRootDirectoryReference();
// transfer File and finished succssfully
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(
SourceObject,
DestObject,
DMLibTestContext.IsAsync,
new CopyDirectoryOptions()
{
Recursive = true
},
new DirectoryTransferContext()
{
ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite
});
Test.Assert(task.Wait(15 * 60 * 100), "Tansfer finished in time.");
Test.Assert(task.Result.NumberOfFilesFailed == 0, "No Failed File.");
Test.Assert(task.Result.NumberOfFilesSkipped == 0, "No Skipped File.");
Test.Assert(task.Result.NumberOfFilesTransferred == fileCount, string.Format("Transferred file :{0} == {1}", task.Result.NumberOfFilesTransferred, fileCount));
// verify that Files in Share Snapshot is transferred
IEnumerable <IListFileItem> sourceFiles = SourceObject.ListFilesAndDirectories(HelperConst.DefaultFileOptions);
foreach (IListFileItem item in sourceFiles)
{
if (item is CloudFile)
{
CloudFile srcFile = item as CloudFile;
CloudFile destFile = DestObject.GetFileReference(srcFile.Name);
srcFile.FetchAttributes();
destFile.FetchAttributes();
Test.Assert(srcFile.Properties.ContentMD5 == destFile.Properties.ContentMD5, string.Format("File {0} MD5 :{1} == {2}", srcFile.Name, srcFile.Properties.ContentMD5, destFile.Properties.ContentMD5));
}
}
}
public void FollowSymlink_DeadLoop_2()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
DMLibDataInfo targetDataInfo = new DMLibDataInfo("targetC");
DirNode dirNodeC = new DirNode("folder_C");
DirNode dirNodeB = new DirNode("folder_B");
dirNodeB.AddFileNode(new FileNode("File_B")
{
SizeInByte = 1024
});
dirNodeC.AddDirNode(dirNodeB);
dirNodeB.AddDirNode(DirNode.SymlinkedDir("symlinkToC", "../../folder_C", dirNodeC));
targetDataInfo.RootNode.AddDirNode(dirNodeC);
SourceAdaptor.CreateIfNotExists();
SourceAdaptor.GenerateData(targetDataInfo);
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir("symlinkToB", "../targetC/folder_C/folder_B", dirNodeB));
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
(transferOptions as UploadDirectoryOptions).FollowSymlink = true;
item.Options = transferOptions;
},
DisableSourceCleaner = true,
};
// This case may takes very long time.
options.TimeoutInMs = 60 * 60 * 1000;
var result = this.ExecuteTestCase(sourceDataInfo, options);
foreach (var exception in result.Exceptions)
{
Test.Info(exception.Message);
}
Test.Assert(result.Exceptions.Count == 1 && result.Exceptions[0] is TransferException && result.Exceptions[0].InnerException.Message.Contains("Too many levels of symbolic links"), "Verify expected exception is thrown.");
#endif
}
public void FollowSymlink_1_BrokenSymlink()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
var dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
sourceDataInfo.RootNode.AddDirNode(dirNode);
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", dirNode.Name, dirNode));
dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}_1");
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}_1", dirNode.Name, dirNode));
SourceAdaptor.GenerateData(sourceDataInfo);
sourceDataInfo.RootNode.DeleteDirNode($"{UnicodeFileName}{SymlinkSuffix}_1");
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
DisableSourceGenerator = true,
DisableSourceCleaner = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
(transferOptions as UploadDirectoryOptions).FollowSymlink = true;
item.Options = transferOptions;
},
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public void Upload_Symlinked_RootDir()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
DMLibDataInfo targetDataInfo = new DMLibDataInfo("target");
UnicodeFileName = "TempTestName";
DirNode dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
targetDataInfo.RootNode.AddDirNode(dirNode);
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
SourceAdaptor.GenerateData(targetDataInfo);
sourceDataInfo.RootNode = DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", "target", targetDataInfo.RootNode);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferItem item = new TransferItem()
{
SourceObject = Path.Combine(SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as string, sourceDataInfo.RootNode.Name),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = new DirectoryTransferContext(),
Options = DefaultTransferDirectoryOptions
};
(item.Options as UploadDirectoryOptions).Recursive = true;
var result = this.RunTransferItems(new List <TransferItem>()
{
item
}, new TestExecutionOptions <DMLibDataInfo>());
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public void TestFileShareSnapshotsDest()
{
string failError1 = "Failed to validate destination";
string failError2 = "Cannot perform this operation on a share representing a snapshot.";
//Prepare Data
string snapshotFile = "snapshotFile";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, snapshotFile, 1024);
SourceAdaptor.GenerateData(sourceDataInfo);
DMLibDataInfo DestDataInfo = new DMLibDataInfo(string.Empty);
DestDataInfo.IsFileShareSnapshot = true;
DestAdaptor.GenerateData(DestDataInfo);
CloudBlobDirectory SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as CloudBlobDirectory;
CloudFileDirectory DestObject = DestAdaptor.GetTransferObject(DestDataInfo.RootPath, DestDataInfo.RootNode) as CloudFileDirectory;
// transfer File and failed with expected Error
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(
SourceObject,
DestObject,
DMLibTestContext.IsAsync,
new CopyDirectoryOptions()
{
Recursive = true
},
new DirectoryTransferContext()
{
ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite
});
try
{
task.Wait(15 * 60 * 100);
}
catch (Exception e)
{
Test.Assert(e.InnerException.Message.Contains(failError1), "Tansfer Exception should contain:" + failError1);
Test.Assert(e.InnerException.InnerException.Message.Contains(failError2), "Tansfer Exception should contain:" + failError2);
}
}
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options)
{
this.CleanupData();
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
if (sourceDataInfo != null)
{
SourceAdaptor.GenerateData(sourceDataInfo);
}
if (options.DestTransferDataInfo != null)
{
DestAdaptor.GenerateData(options.DestTransferDataInfo);
}
if (options.AfterDataPrepared != null)
{
options.AfterDataPrepared();
}
List <TransferItem> allItems = new List <TransferItem>();
foreach (var fileNode in sourceDataInfo.EnumerateFileNodes())
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(fileNode),
DestObject = DestAdaptor.GetTransferObject(fileNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(fileNode, item);
}
allItems.Add(item);
}
return(this.RunTransferItems(allItems, options));
}
private void TestSASTokenOfEachVersion(string targetSASVersion, bool isDirectoryTransfer)
{
Test.Info("Testing version of {0}", targetSASVersion);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, DMLibTestBase.FileName, 1024);
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = isDirectoryTransfer;
options.TransferItemModifier = (fileNode, transferItem) =>
{
dynamic transferOptions = isDirectoryTransfer ? DefaultTransferDirectoryOptions : DefaultTransferOptions;
transferItem.Options = transferOptions;
if (isDirectoryTransfer)
{
transferItem.TransferContext = new DirectoryTransferContext();
transferItem.TransferContext.FileFailed += (source, e) =>
{
Test.Error(e.Exception.ToString());
};
DirectoryOptions dirOptions = transferItem.Options as DirectoryOptions;
dirOptions.Recursive = true;
}
else
{
transferItem.TransferContext = new SingleTransferContext();
}
transferItem.TransferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite;
};
string sourceSAS = null;
string destSAS = null;
switch (DMLibTestContext.SourceType)
{
case DMLibDataType.CloudBlob:
case DMLibDataType.AppendBlob:
case DMLibDataType.BlockBlob:
case DMLibDataType.PageBlob:
if ((DMLibTestContext.SourceType == DMLibDataType.AppendBlob) &&
(string.CompareOrdinal(targetSASVersion, "2015-04-05") < 0))
{
break;
}
SourceAdaptor.CreateIfNotExists();
CloudBlobDataAdaptor blobAdaptor = SourceAdaptor as CloudBlobDataAdaptor;
sourceSAS = Util.SASGenerator.GetSharedAccessSignature(blobAdaptor.GetBaseContainer(),
new SharedAccessBlobPolicy
{
Permissions = isDirectoryTransfer ? SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List : SharedAccessBlobPermissions.Read,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1)
},
null,
null,
null,
targetSASVersion);
break;
case DMLibDataType.CloudFile:
if (string.CompareOrdinal(targetSASVersion, "2015-02-21") < 0)
{
break;
}
SourceAdaptor.CreateIfNotExists();
CloudFileDataAdaptor fileAdaptor = SourceAdaptor as CloudFileDataAdaptor;
sourceSAS = Util.SASGenerator.GetSharedAccessSignature(
fileAdaptor.GetBaseShare(),
new SharedAccessFilePolicy
{
Permissions = isDirectoryTransfer ? SharedAccessFilePermissions.List | SharedAccessFilePermissions.Read : SharedAccessFilePermissions.Read,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1)
},
null,
null,
null,
targetSASVersion);
break;
default:
break;
}
if (!DMLibTestContext.IsAsync ||
(string.CompareOrdinal(targetSASVersion, "2014-02-14") >= 0))
{
switch (DMLibTestContext.DestType)
{
case DMLibDataType.CloudBlob:
case DMLibDataType.AppendBlob:
case DMLibDataType.BlockBlob:
case DMLibDataType.PageBlob:
if ((DMLibTestContext.DestType == DMLibDataType.AppendBlob) &&
(string.CompareOrdinal(targetSASVersion, "2015-04-05") < 0))
{
break;
}
DestAdaptor.CreateIfNotExists();
CloudBlobDataAdaptor blobAdaptor = DestAdaptor as CloudBlobDataAdaptor;
destSAS = Util.SASGenerator.GetSharedAccessSignature(blobAdaptor.GetBaseContainer(),
new SharedAccessBlobPolicy
{
Permissions = DMLibTestContext.IsAsync ? SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.Read : SharedAccessBlobPermissions.Write,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1)
},
null,
null,
null,
targetSASVersion);
break;
case DMLibDataType.CloudFile:
if (string.CompareOrdinal(targetSASVersion, "2015-02-21") < 0)
{
break;
}
DestAdaptor.CreateIfNotExists();
CloudFileDataAdaptor fileAdaptor = DestAdaptor as CloudFileDataAdaptor;
destSAS = Util.SASGenerator.GetSharedAccessSignature(
fileAdaptor.GetBaseShare(),
new SharedAccessFilePolicy
{
Permissions = DMLibTestContext.IsAsync ? SharedAccessFilePermissions.Write | SharedAccessFilePermissions.Read : SharedAccessFilePermissions.Write,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1)
},
null,
null,
null,
targetSASVersion);
break;
default:
break;
}
}
if (null != sourceSAS)
{
options.SourceCredentials = new StorageCredentials(sourceSAS);
}
if (null != destSAS)
{
options.DestCredentials = new StorageCredentials(destSAS);
}
var result = this.ExecuteTestCase(sourceDataInfo, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
public void TestDirectoryPreserveSMBProperties()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
CloudFileNtfsAttributes[] SMBFileAttributes =
{
CloudFileNtfsAttributes.ReadOnly,
CloudFileNtfsAttributes.Hidden,
#if DEBUG
CloudFileNtfsAttributes.System,
CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Normal,
CloudFileNtfsAttributes.Offline,
CloudFileNtfsAttributes.NotContentIndexed,
CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden,
CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly |
CloudFileNtfsAttributes.Hidden |
CloudFileNtfsAttributes.System |
CloudFileNtfsAttributes.Archive |
CloudFileNtfsAttributes.NotContentIndexed |
CloudFileNtfsAttributes.NoScrubData
#endif
};
for (int i = 0; i < SMBFileAttributes.Length; ++i)
{
Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]);
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]);
DirectoryTransferContext dirTransferContext = new DirectoryTransferContext();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
transferItem.Options = transferOptions;
};
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = (path) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
return(FileAttributes.Directory);
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory));
}
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i]));
}
};
TestHookCallbacks.SetFileAttributesCallback = (path, attributes) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot"))
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory),
"Directory attributes should be expected {0}", attributes);
}
}
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]),
"File attributes should be expected {0}", attributes);
}
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
IsDirectoryTransfer = true,
Options = DefaultTransferDirectoryOptions
};
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.TransferContext = new DirectoryTransferContext();
item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) =>
{
return(false);
};
List <TransferItem> items = new List <TransferItem>();
items.Add(item);
result = this.RunTransferItems(items, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.Local)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false);
}
else
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
}
}
finally
{
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = null;
TestHookCallbacks.SetFileAttributesCallback = null;
#endif
}
}
}
public void TestDirectoryPreserveSMBPropertiesResume()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
CloudFileNtfsAttributes[] SMBFileAttributes =
{
CloudFileNtfsAttributes.ReadOnly,
CloudFileNtfsAttributes.Hidden,
#if DEBUG
CloudFileNtfsAttributes.System,
CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Normal,
CloudFileNtfsAttributes.Offline,
CloudFileNtfsAttributes.NotContentIndexed,
CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden,
CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly |
CloudFileNtfsAttributes.Hidden |
CloudFileNtfsAttributes.System |
CloudFileNtfsAttributes.Archive |
CloudFileNtfsAttributes.NotContentIndexed |
CloudFileNtfsAttributes.NoScrubData
#endif
};
for (int i = 0; i < SMBFileAttributes.Length; ++i)
{
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]);
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]);
DirectoryTransferContext dirTransferContext = null;
if (IsStreamJournal)
{
dirTransferContext = new DirectoryTransferContext(journalStream);
}
else
{
dirTransferContext = new DirectoryTransferContext();
}
var progressChecker = new ProgressChecker(14, 14 * 1024);
dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, item) =>
{
item.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.CancellationToken = tokenSource.Token;
transferItem = item;
};
TransferCheckpoint checkpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
checkpoint = dirTransferContext.LastCheckpoint;
}
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = (path) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
return(FileAttributes.Directory);
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory));
}
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i]));
}
};
TestHookCallbacks.SetFileAttributesCallback = (path, attributes) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot"))
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory),
"Directory attributes should be expected {0}", attributes);
}
}
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]),
"File attributes should be expected {0}", attributes);
}
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferItem resumeItem = transferItem.Clone();
DirectoryTransferContext resumeContext = null;
journalStream.Position = 0;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream);
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint));
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
IsDirectoryTransfer = true,
Options = DefaultTransferDirectoryOptions
};
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.TransferContext = new DirectoryTransferContext();
item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) =>
{
return(false);
};
List <TransferItem> items = new List <TransferItem>();
items.Add(item);
result = this.RunTransferItems(items, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.Local)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false);
}
else
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
}
}
finally
{
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = null;
TestHookCallbacks.SetFileAttributesCallback = null;
#endif
}
}
}
}
public void TestDirectoryCheckContentMD5()
{
long fileSize = 5 * 1024 * 1024;
long totalSize = fileSize * 4;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
TransferContext context = new TransferContext();
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
};
ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
context.FileFailed += (eventSource, eventArgs) =>
{
transferExceptions.Add(eventArgs.Exception);
};
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
};
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
DMLibDataInfo actualDataInfo = testResult.DataInfo;
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != 1)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5)
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string wrongMD5File = "wrongMD5File";
string correctMD5File = "correctMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
DestAdaptor.Cleanup();
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
using (var resumeStream = new MemoryStream())
{
TransferContext context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti,
totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = !checkMD5,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, executionOption);
if (null != testResult.Exceptions)
{
foreach (var exception in testResult.Exceptions)
{
Test.Info("Got exception during transferring. {0}", exception);
}
}
eventChecker = new TransferEventChecker();
resumeStream.Position = 0;
context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(wrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
checkMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected.");
VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0);
if (checkMD5)
{
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
else
{
Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5");
}
}
}
public void TestDirectoryCheckContentMD5Resume()
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
// Prepare data for transfer items with disabling MD5 check
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
notCheckMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
TransferContext context = new DirectoryTransferContext();
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, executionOption);
eventChecker = new TransferEventChecker();
context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint));
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
notCheckMD5Item = notCheckMD5Item.Clone();
checkMD5Item.TransferContext = context;
notCheckMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
public TestResult <DMLibDataInfo> ExecuteTestCase(DMLibDataInfo sourceDataInfo, TestExecutionOptions <DMLibDataInfo> options)
{
if (options.DisableSourceCleaner)
{
this.CleanupData(false, true);
}
else
{
this.CleanupData();
}
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
string sourceRootPath = string.Empty;
DirNode sourceRootNode = new DirNode(string.Empty);
if (sourceDataInfo != null)
{
sourceRootPath = sourceDataInfo.RootPath;
sourceRootNode = sourceDataInfo.RootNode;
if (!options.DisableSourceGenerator)
{
SourceAdaptor.GenerateData(sourceDataInfo);
}
}
string destRootPath = string.Empty;
if (options.DestTransferDataInfo != null)
{
destRootPath = options.DestTransferDataInfo.RootPath;
DestAdaptor.GenerateData(options.DestTransferDataInfo);
}
if (options.AfterDataPrepared != null)
{
options.AfterDataPrepared();
}
List <TransferItem> allItems = new List <TransferItem>();
if (options.IsDirectoryTransfer)
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceRootPath, sourceRootNode, options.SourceCredentials),
DestObject = DestAdaptor.GetTransferObject(destRootPath, sourceRootNode, options.DestCredentials),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
IsDirectoryTransfer = true,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(null, item);
}
allItems.Add(item);
}
else
{
foreach (var fileNode in sourceDataInfo.EnumerateFileNodes())
{
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, fileNode, options.SourceCredentials),
DestObject = DestAdaptor.GetTransferObject(destRootPath, fileNode, options.DestCredentials),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
};
if (options.TransferItemModifier != null)
{
options.TransferItemModifier(fileNode, item);
}
allItems.Add(item);
}
}
return(this.RunTransferItems(allItems, options));
}
