public void TransferDirectoryDifferentSizeObject()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
DMLibDataHelper.AddMultipleFilesNormalSize(sourceDataInfo.RootNode, BVT.UnicodeFileName);
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
item.Options = transferOptions;
},
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public void DummySingleBlobDownload()
{
// Single dummy blob should behave the same as empty blob.
Dictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add(Constants.DirectoryBlobMetadataKey, "true");
Test.Info("Metadata is =====================");
foreach (var keyValue in metadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode dummyFolderNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = 0L,
Metadata = metadata
};
sourceDataInfo.RootNode.AddFileNode(dummyFolderNode);
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = false;
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
dummyFolderNode.Metadata = new Dictionary <string, string>();
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public void TestMetadata()
{
Dictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
Test.Info("Metadata is =====================");
foreach (var keyValue in metadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
Metadata = metadata
};
sourceDataInfo.RootNode.AddFileNode(fileNode);
var result = this.ExecuteTestCase(sourceDataInfo, new TestExecutionOptions <DMLibDataInfo>());
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public void TransferInAllDirections()
{
List <TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: false);
// Execution
var result = this.RunTransferItems(allItems, new TestExecutionOptions <DMLibDataInfo>());
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
foreach (DMLibDataType destDataType in DataTypes)
{
if (DMLibDataType.Stream == destDataType || DMLibDataType.URI == destDataType)
{
continue;
}
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
FileNode sourceFileNode = expectedFileNodes[destFileNode.Name];
if (IsCloudService(destDataType))
{
IDictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add("aa", "bb");
sourceFileNode.ContentLanguage = "EN";
sourceFileNode.Metadata = metadata;
}
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
}
public void TransferBigSizeObject()
{
int originParallel = TransferManager.Configurations.ParallelOperations;
TransferManager.Configurations.ParallelOperations = 4;
try
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddMultipleFilesBigSize(sourceDataInfo.RootNode, DMLibTestBase.FileName);
var option = new TestExecutionOptions <DMLibDataInfo>();
var result = this.ExecuteTestCase(sourceDataInfo, option);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
if (!(DMLibTestContext.DestType == DMLibDataType.Local || DMLibTestContext.DestType == DMLibDataType.Stream))
{
this.ValidateDestinationMD5ByDownloading(result.DataInfo, option);
}
}
finally
{
TransferManager.Configurations.ParallelOperations = originParallel;
}
}
public void TransferBigSizeObject()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddMultipleFilesBigSize(sourceDataInfo.RootNode, DMLibTestBase.FileName);
var result = this.ExecuteTestCase(sourceDataInfo, new TestExecutionOptions<DMLibDataInfo>());
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public void DummyDirectoryBlobDownload()
{
Dictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add(Constants.DirectoryBlobMetadataKey, "true");
Test.Info("Metadata is =====================");
foreach (var keyValue in metadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode dummyFolderNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = 0L,
Metadata = metadata
};
DirNode dirNode = new DirNode(DMLibTestBase.FileName);
FileNode actualFile = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
};
dirNode.AddFileNode(actualFile);
sourceDataInfo.RootNode.AddFileNode(dummyFolderNode);
sourceDataInfo.RootNode.AddDirNode(dirNode);
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DestTransferDataInfo = destDataInfo;
options.DisableDestinationFetch = true;
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, transferItem) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferItem.Options = transferOptions;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(DMLibDataType.Local);
destDataInfo = destAdaptor.GetTransferDataInfo(destDataInfo.RootPath);
sourceDataInfo.RootNode.DeleteFileNode(DMLibTestBase.FileName);
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, destDataInfo), "Verify transfer result.");
}
public void FollowSymlink_1_BrokenSymlink()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
var dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
sourceDataInfo.RootNode.AddDirNode(dirNode);
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", dirNode.Name, dirNode));
dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}_1");
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}_1", dirNode.Name, dirNode));
SourceAdaptor.GenerateData(sourceDataInfo);
sourceDataInfo.RootNode.DeleteDirNode($"{UnicodeFileName}{SymlinkSuffix}_1");
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
DisableSourceGenerator = true,
DisableSourceCleaner = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
(transferOptions as UploadDirectoryOptions).FollowSymlink = true;
item.Options = transferOptions;
},
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public static void VerifySingleObjectResumeResult(TestResult <DMLibDataInfo> result, DMLibDataInfo expectedDataInfo)
{
if (DMLibTestContext.SourceType != DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result.");
}
else
{
Test.Assert(result.Exceptions.Count == 1, "Verify stream resume is not supported");
Exception exception = result.Exceptions[0];
Test.Assert(exception is NotSupportedException, "Verify stream resume is not supported");
}
}
public void Upload_Symlinked_RootDir()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
DMLibDataInfo targetDataInfo = new DMLibDataInfo("target");
UnicodeFileName = "TempTestName";
DirNode dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
targetDataInfo.RootNode.AddDirNode(dirNode);
SourceAdaptor.CreateIfNotExists();
DestAdaptor.CreateIfNotExists();
SourceAdaptor.GenerateData(targetDataInfo);
sourceDataInfo.RootNode = DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}", "target", targetDataInfo.RootNode);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferItem item = new TransferItem()
{
SourceObject = Path.Combine(SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as string, sourceDataInfo.RootNode.Name),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = new DirectoryTransferContext(),
Options = DefaultTransferDirectoryOptions
};
(item.Options as UploadDirectoryOptions).Recursive = true;
var result = this.RunTransferItems(new List <TransferItem>()
{
item
}, new TestExecutionOptions <DMLibDataInfo>());
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public void TransferDifferentSizeObject()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddMultipleFilesNormalSize(sourceDataInfo.RootNode, BVT.UnicodeFileName);
var options = new TestExecutionOptions <DMLibDataInfo>();
options.AfterDataPrepared = () =>
{
if ((DMLibTestContext.SourceType == DMLibDataType.CloudFile || DMLibTestContext.SourceType == DMLibDataType.PageBlob) &&
!DMLibTestContext.IsAsync)
{
string sparseFileName = "SparseFile";
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, sparseFileName, 1);
FileNode sparseFileNode = sourceDataInfo.RootNode.GetFileNode(sparseFileName);
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor cloudFileDataAdaptor = SourceAdaptor as CloudFileDataAdaptor;
CloudFile sparseCloudFile = cloudFileDataAdaptor.GetCloudFileReference(sourceDataInfo.RootPath, sparseFileNode);
this.PrepareCloudFileWithDifferentSizeRange(sparseCloudFile);
sparseFileNode.MD5 = sparseCloudFile.Properties.ContentMD5;
sparseFileNode.Metadata = sparseCloudFile.Metadata;
}
else if (DMLibTestContext.SourceType == DMLibDataType.PageBlob)
{
CloudBlobDataAdaptor cloudBlobDataAdaptor = SourceAdaptor as CloudBlobDataAdaptor;
CloudPageBlob sparsePageBlob = cloudBlobDataAdaptor.GetCloudBlobReference(sourceDataInfo.RootPath, sparseFileNode) as CloudPageBlob;
this.PreparePageBlobWithDifferenSizePage(sparsePageBlob);
sparseFileNode.MD5 = sparsePageBlob.Properties.ContentMD5;
sparseFileNode.Metadata = sparsePageBlob.Metadata;
}
}
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public static bool Equals(DirNode dirNodeA, DirNode dirNodeB)
{
// The same node
if (dirNodeA == dirNodeB)
{
return(true);
}
// Empty node equals to null
if ((dirNodeA == null || dirNodeA.IsEmpty) &&
(dirNodeB == null || dirNodeB.IsEmpty))
{
return(true);
}
// Compare two nodes
if (null != dirNodeA && null != dirNodeB)
{
if (dirNodeA.FileNodeCount != dirNodeB.FileNodeCount ||
dirNodeA.NonEmptyDirNodeCount != dirNodeB.NonEmptyDirNodeCount)
{
return(false);
}
foreach (FileNode fileNodeA in dirNodeA.FileNodes)
{
FileNode fileNodeB = dirNodeB.GetFileNode(fileNodeA.Name);
if (!DMLibDataHelper.Equals(fileNodeA, fileNodeB))
{
return(false);
}
}
foreach (DirNode subDirNodeA in dirNodeA.DirNodes)
{
DirNode subDirNodeB = dirNodeB.GetDirNode(subDirNodeA.Name);
if (!DMLibDataHelper.Equals(subDirNodeA, subDirNodeB))
{
return(false);
}
}
return(true);
}
return(false);
}
public void FollowSymlink_Random_SymlinkDir_OnSameLevel()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
int level = random.Next(2, 40);
for (int i = 0; i < level; ++i)
{
var dirNode = new DirNode($"{UnicodeFileName}{FolderSuffix}_{i}");
dirNode.AddFileNode(new FileNode($"{UnicodeFileName}{FileSuffix}")
{
SizeInByte = 1024
});
sourceDataInfo.RootNode.AddDirNode(dirNode);
sourceDataInfo.RootNode.AddDirNode(DirNode.SymlinkedDir($"{UnicodeFileName}{SymlinkSuffix}_{i}", dirNode.Name, dirNode));
}
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
(transferOptions as UploadDirectoryOptions).FollowSymlink = true;
item.Options = transferOptions;
},
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public void FollowSymlink_Random_Levels_SymlinkDir()
{
#if DNXCORE50
if (!CrossPlatformHelpers.IsLinux)
{
return;
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("rootfolder");
int level = random.Next(2, 40);
Test.Info($"Testing {level} levels of symlinked dirs");
BuildDeepSymlinkDir(sourceDataInfo.RootNode, sourceDataInfo.RootNode, level);
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
(transferOptions as UploadDirectoryOptions).FollowSymlink = true;
item.Options = transferOptions;
},
};
// This case may takes very long time.
options.TimeoutInMs = 60 * 60 * 1000;
var result = this.ExecuteTestCase(sourceDataInfo, options);
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
#endif
}
public void ToLbb_100MBBlockSizeAndSmallFiles()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, 300 * 1024); // 300MB
for (int i = 0; i < 10; i++)
{
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName + "_" + i, i);
}
var options = new TestExecutionOptions <DMLibDataInfo>()
{
BlockSize = 100 * 1024 * 1024 // 100MB
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
this.ValidateDestinationMD5ByDownloading(result.DataInfo, options);
}
public void TransferInAllDirections()
{
List <TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: false);
// Execution
var result = this.RunTransferItems(allItems, new TestExecutionOptions <DMLibDataInfo>());
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
FileNode sourceFileNode = expectedFileNodes[destFileNode.Name];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
}
public void ToLbb_16MBBlockSizeAndSmallFiles()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName + "_" + 36 + "MB", 36 * 1024); // 36MB, should use 16MB block size, PutBlock and PutBlockList
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName + "_" + 16 + "MB", 16 * 1024); // 16MB, should use 16MB block size, PutBlob
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName + "_" + 14 + "MB", 14 * 1024); // 14MB, should use 16MB block size, PutBlob
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName + "_" + 8 + "MB", 8 * 1024); // 8MB, should use 8MB block size, PutBlob
for (int i = 0; i < 2; i++)
{
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName + "_" + i, i); // 0 and 1KB file
}
var options = new TestExecutionOptions <DMLibDataInfo>()
{
BlockSize = 16 * 1024 * 1024 // 16MB
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
this.ValidateDestinationMD5ByDownloading(result.DataInfo, options);
}
public void TestMetadataOverwrite()
{
Dictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
metadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
Test.Info("Metadata is =====================");
foreach (var keyValue in metadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
Metadata = metadata
};
sourceDataInfo.RootNode.AddFileNode(fileNode);
Dictionary <string, string> destMetadata = new Dictionary <string, string>();
destMetadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
destMetadata.Add(FileOp.NextCIdentifierString(random), FileOp.NextNormalString(random));
Test.Info("Destination metadata is =====================");
foreach (var keyValue in destMetadata)
{
Test.Info("name:{0} value:{1}", keyValue.Key, keyValue.Value);
}
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
Metadata = destMetadata
};
destDataInfo.RootNode.AddFileNode(fileNode);
var option = new TestExecutionOptions <DMLibDataInfo>()
{
DestTransferDataInfo = destDataInfo
};
SingleTransferContext transferContext = new SingleTransferContext();
transferContext.ShouldOverwriteCallbackAsync = async(source, dest) =>
{
return(true);
};
option.TransferItemModifier = (inputFileNode, transferItem) =>
{
transferItem.TransferContext = transferContext;
};
var result = this.ExecuteTestCase(sourceDataInfo, option);
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
}
public static void VerifySingleObjectResumeResult(TestResult <DMLibDataInfo> result, DMLibDataInfo expectedDataInfo)
{
Test.Assert(result.Exceptions.Count == 0, "Verify no exception is thrown.");
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result.");
}
public static bool Equals(DirNode dirNodeA, DirNode dirNodeB)
{
// The same node
if (dirNodeA == dirNodeB)
{
return(true);
}
// Empty node equals to null
if ((dirNodeA == null || dirNodeA.IsEmpty) &&
(dirNodeB == null || dirNodeB.IsEmpty))
{
return(true);
}
// Compare two nodes
if (null != dirNodeA && null != dirNodeB)
{
if (dirNodeA.FileNodeCount != dirNodeB.FileNodeCount ||
dirNodeA.NonEmptyDirNodeCount != dirNodeB.NonEmptyDirNodeCount)
{
return(false);
}
if ((null != dirNodeA.Metadata) && (dirNodeA.Metadata.Count > 0))
{
if (null == dirNodeB.Metadata)
{
return(false);
}
if (dirNodeA.Metadata.Count != dirNodeB.Metadata.Count)
{
return(false);
}
foreach (var keyValue in dirNodeA.Metadata)
{
if (!string.Equals(dirNodeB.Metadata[keyValue.Key], keyValue.Value))
{
return(false);
}
}
}
else
{
if ((null != dirNodeB.Metadata) && (dirNodeB.Metadata.Count > 0))
{
return(false);
}
}
foreach (FileNode fileNodeA in dirNodeA.FileNodes)
{
FileNode fileNodeB = dirNodeB.GetFileNode(fileNodeA.Name);
FileNode fileNodeAA = fileNodeA;
if (null == fileNodeB)
{
fileNodeB = dirNodeB.GetFileNode(DMLibTestHelper.EscapeInvalidCharacters(fileNodeA.Name));
if (null != fileNodeB)
{
fileNodeAA = fileNodeA.Clone(DMLibTestHelper.EscapeInvalidCharacters(fileNodeA.Name));
}
}
if (!DMLibDataHelper.Equals(fileNodeAA, fileNodeB))
{
return(false);
}
}
foreach (DirNode subDirNodeA in dirNodeA.DirNodes)
{
Test.Info("Verifying subfolder: {0} ", subDirNodeA.Name);
DirNode subDirNodeB = dirNodeB.GetDirNode(subDirNodeA.Name);
if (null == subDirNodeB)
{
subDirNodeB = dirNodeB.GetDirNode(DMLibTestHelper.EscapeInvalidCharacters(subDirNodeA.Name));
}
if (!DMLibDataHelper.Equals(subDirNodeA, subDirNodeB))
{
return(false);
}
}
return(true);
}
return(false);
}
private void ResumeInAllDirectionsHelper(bool directoryTransfer)
{
List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true)
: AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferContext transferContext = null;
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext();
}
else
{
transferContext = new SingleTransferContext();
}
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
else
{
transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
int expectedFailureCount = 0;
transferContext.FileFailed += (resource, eventArgs) =>
{
TransferException exception = eventArgs.Exception as TransferException;
if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId)
{
Interlocked.Increment(ref expectedFailureCount);
}
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
if (DMLibDataType.URI == destDataType)
{
continue;
}
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
if (!directoryTransfer)
{
Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count);
}
else
{
Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count);
Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber);
}
}
}
public void ResumeInAllDirections()
{
long fileSizeInByte = 10 * 1024 * 1024;
Dictionary <string, FileNode> sourceFileNodes = this.PrepareSourceData(fileSizeInByte);
List <TransferItem> allItems = this.GetTransformItemsForAllDirections(sourceFileNodes);
int fileCount = sourceFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(fileCount, fileSizeInByte * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
progressChecker.DataTransferred.WaitOne();
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
transferContext = new TransferContext(pair.Value)
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
int resumeFailCount = 0;
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
if (!fileName.Contains(DMLibDataType.Stream.ToString()))
{
FileNode sourceFileNode = sourceFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
else
{
resumeFailCount++;
}
}
}
Test.Assert(result.Exceptions.Count == resumeFailCount, "Verify resume failure count: expected {0}, actual {1}.", resumeFailCount, result.Exceptions.Count);
foreach (var resumeException in result.Exceptions)
{
Test.Assert(resumeException is NotSupportedException, "Verify resume exception is NotSupportedException.");
}
}
}