public void TransferInAllDirections()
{
List <TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: false);
// Execution
var result = this.RunTransferItems(allItems, new TestExecutionOptions <DMLibDataInfo>());
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
foreach (DMLibDataType destDataType in DataTypes)
{
if (DMLibDataType.Stream == destDataType || DMLibDataType.URI == destDataType)
{
continue;
}
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
FileNode sourceFileNode = expectedFileNodes[destFileNode.Name];
if (IsCloudService(destDataType))
{
IDictionary <string, string> metadata = new Dictionary <string, string>();
metadata.Add("aa", "bb");
sourceFileNode.ContentLanguage = "EN";
sourceFileNode.Metadata = metadata;
}
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
}
public static string DetailedInfo(this DMLibDataInfo dataInfo)
{
StringBuilder builder = new StringBuilder();
builder.AppendLine(string.Format("TransferDataInfo root: {0}", dataInfo.RootPath));
foreach (FileNode fileNode in dataInfo.EnumerateFileNodes())
{
builder.AppendLine(fileNode.DetailedInfo());
}
return(builder.ToString());
}
public void TransferInAllDirections()
{
List <TransferItem> allItems = AllTransferDirectionTest.GetTransformItemsForAllDirections(resume: false);
// Execution
var result = this.RunTransferItems(allItems, new TestExecutionOptions <DMLibDataInfo>());
// Verify all files are transfered successfully
Test.Assert(result.Exceptions.Count == 0, "Verify no exception occurs.");
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
FileNode sourceFileNode = expectedFileNodes[destFileNode.Name];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
}
public static void SetCalculatedFileMD5(DMLibDataInfo dataInfo, DataAdaptor <DMLibDataInfo> destAdaptor, bool disableMD5Check = false)
{
foreach (FileNode fileNode in dataInfo.EnumerateFileNodes())
{
if (DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType))
{
CloudBlobDataAdaptor cloudBlobDataAdaptor = destAdaptor as CloudBlobDataAdaptor;
CloudBlob cloudBlob = cloudBlobDataAdaptor.GetCloudBlobReference(fileNode);
fileNode.MD5 = CloudBlobHelper.CalculateMD5ByDownloading(cloudBlob, disableMD5Check);
}
else if (DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor cloudFileDataAdaptor = destAdaptor as CloudFileDataAdaptor;
CloudFile cloudFile = cloudFileDataAdaptor.GetCloudFileReference(fileNode);
fileNode.MD5 = CloudFileHelper.CalculateMD5ByDownloading(cloudFile, disableMD5Check);
}
// No need to set md5 for local destination
}
}
private void ResumeInAllDirectionsHelper(bool directoryTransfer)
{
List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true)
: AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferContext transferContext = null;
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext();
}
else
{
transferContext = new SingleTransferContext();
}
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
else
{
transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
int expectedFailureCount = 0;
transferContext.FileFailed += (resource, eventArgs) =>
{
TransferException exception = eventArgs.Exception as TransferException;
if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId)
{
Interlocked.Increment(ref expectedFailureCount);
}
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
if (DMLibDataType.URI == destDataType)
{
continue;
}
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
if (!directoryTransfer)
{
Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count);
}
else
{
Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count);
Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber);
}
}
}
public void ResumeInAllDirections()
{
long fileSizeInByte = 10 * 1024 * 1024;
Dictionary <string, FileNode> sourceFileNodes = this.PrepareSourceData(fileSizeInByte);
List <TransferItem> allItems = this.GetTransformItemsForAllDirections(sourceFileNodes);
int fileCount = sourceFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(fileCount, fileSizeInByte * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
progressChecker.DataTransferred.WaitOne();
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
transferContext = new TransferContext(pair.Value)
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
OverwriteCallback = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
int resumeFailCount = 0;
foreach (DMLibDataType destDataType in DataTypes)
{
DataAdaptor <DMLibDataInfo> destAdaptor = GetSourceAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
if (!fileName.Contains(DMLibDataType.Stream.ToString()))
{
FileNode sourceFileNode = sourceFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
else
{
resumeFailCount++;
}
}
}
Test.Assert(result.Exceptions.Count == resumeFailCount, "Verify resume failure count: expected {0}, actual {1}.", resumeFailCount, result.Exceptions.Count);
foreach (var resumeException in result.Exceptions)
{
Test.Assert(resumeException is NotSupportedException, "Verify resume exception is NotSupportedException.");
}
}
}
