public static void VerifyStorageException(Exception exception, int expectedHttpStatusCode, params string[] expectedMessages)
{
StorageException storageException = exception as StorageException;
if (storageException == null)
{
Test.Error("Verify exception is a storage exception.");
return;
}
Test.Assert(storageException.RequestInformation.HttpStatusCode == expectedHttpStatusCode, "Verify http status code: {0}, expected: {1}", storageException.RequestInformation.HttpStatusCode, expectedHttpStatusCode);
VerificationHelper.VerifyExceptionErrorMessage(exception, expectedMessages);
}
public static void VerifyTransferException(Exception exception, TransferErrorCode expectedErrorCode, params string[] expectedMessages)
{
TransferException transferException = exception as TransferException;
if (transferException == null)
{
Test.Error("Verify exception is a transfer exception.");
return;
}
Test.Assert(transferException.ErrorCode == expectedErrorCode, "Verify error code: {0}, expected: {1}", transferException.ErrorCode, expectedErrorCode);
VerificationHelper.VerifyExceptionErrorMessage(exception, expectedMessages);
}
private void TestSearchPatternError(bool recursive, string searchPattern, string expectedErrorMessage)
{
var options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (notUsed, item) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = recursive;
transferOptions.SearchPattern = searchPattern;
item.Options = transferOptions;
},
};
var testResult = this.ExecuteTestCase(null, options);
if (testResult.Exceptions.Count != 1)
{
Test.Error("Should be exactly one exception, actual: {0}", testResult.Exceptions.Count);
}
VerificationHelper.VerifyExceptionErrorMessage(testResult.Exceptions[0], expectedErrorMessage);
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
secondCheckpoint = transferContext.LastCheckpoint;
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled.");
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
ProgressChecker firstProgressChecker = null, secondProgressChecker = null;
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.IsAsync))
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// first progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
// The destination is already created, will cause a transfer skip
firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob))
{
firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else
{
firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024);
}
// second progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
// The destination is already created, will cause a transfer skip
secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream)
{
secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync)
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024);
}
else
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024);
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
TransferContext resumeContext = new TransferContext(
IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
resumeContext = new TransferContext(
IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = secondProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify transfer is skipped when source is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyTransferException(result.Exceptions[0], TransferErrorCode.NotOverwriteExistingDestination, "Skiped file");
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && !DMLibTestContext.IsAsync)
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && !DMLibTestContext.IsAsync)
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
TransferContext transferContext = IsStreamJournal ? new SingleTransferContext(journalStream) : new SingleTransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1, 0, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
dynamic transferOptions = DefaultTransferOptions;
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
transferOptions.PreserveSMBAttributes = true;
transferOptions.PreserveSMBPermissions = true;
}
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
item.DisableStreamDispose = true;
item.Options = transferOptions;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
if (IsStreamJournal &&
(DMLibTestContext.SourceType == DMLibDataType.Stream ||
DMLibTestContext.DestType == DMLibDataType.Stream))
{
return;
}
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
if (!IsStreamJournal)
{
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
if ((DMLibTestContext.SourceType == DMLibDataType.Stream) || (DMLibTestContext.DestType == DMLibDataType.Stream))
{
options.DisableDestinationFetch = true;
}
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (!IsStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
}
else
{
if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify job is failed");
Exception jobException = result.Exceptions[0];
Test.Info("{0}", jobException);
VerificationHelper.VerifyExceptionErrorMessage(jobException, "Cannot deserialize to TransferLocation when its TransferLocationType is Stream.");
transferItem.DisableStreamDispose = false;
transferItem.CloseStreamIfNecessary();
return;
}
}
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
ProgressChecker firstProgressChecker = null, secondProgressChecker = null;
if (!IsStreamJournal)
{
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && (DMLibTestContext.DestType != DMLibDataType.AppendBlob || (DMLibTestContext.CopyMethod == DMLibCopyMethod.ServiceSideAsyncCopy)))
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
}
// first progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType != DMLibDataType.BlockBlob)
{
// The destination is already created, will cause a transfer skip
firstProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream || (DMLibTestContext.SourceType == DMLibDataType.Stream && DMLibTestContext.DestType == DMLibDataType.BlockBlob))
{
firstProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else
{
firstProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1, 0, 0, fileSizeInKB * 1024);
}
// second progress checker
if (DMLibTestContext.SourceType == DMLibDataType.Stream)
{
// The destination is already created, will cause a transfer skip
secondProgressChecker = new ProgressChecker(2, fileSizeInKB * 1024, 0, 1 /* failed */, 1 /* skipped */, fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.Stream)
{
secondProgressChecker = new ProgressChecker(2, 2 * fileSizeInKB * 1024, 1 /* transferred */, 1 /* failed */, 0, 2 * fileSizeInKB * 1024);
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 0, 1 /* failed */, 0, fileSizeInKB * 1024);
}
else
{
secondProgressChecker = new ProgressChecker(1, fileSizeInKB * 1024, 1 /* transferred */, 0, 0, fileSizeInKB * 1024);
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
TransferContext resumeContext = null;
if (IsStreamJournal)
{
Exception deserializeEX = null;
try
{
resumeContext = new SingleTransferContext(journalStream)
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
}
catch (Exception ex)
{
if ((DMLibTestContext.SourceType != DMLibDataType.Stream) &&
(DMLibTestContext.DestType != DMLibDataType.Stream))
{
Test.Error("Should no exception in deserialization when no target is stream.");
}
deserializeEX = ex;
}
}
else
{
resumeContext = new SingleTransferContext(IsStreamDirection() ? firstResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = firstProgressChecker.GetProgressHandler()
};
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
if (!IsStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
resumeContext = new SingleTransferContext(
IsStreamDirection() ? secondResumeCheckpoint : DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = secondProgressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.SourceType == DMLibDataType.Stream || DMLibTestContext.DestType == DMLibDataType.Stream)
{
Test.Assert(result.Exceptions.Count == 1, "Verify error reported when source/destination is stream.");
exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(result.Exceptions[0], "Resuming transfer from or to a stream is not supported");
}
else if (DMLibTestContext.DestType == DMLibDataType.AppendBlob && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
else
{
// For sync copy, recalculate md5 of destination by downloading the file to local.
if (IsCloudService(DMLibTestContext.DestType) && (DMLibTestContext.CopyMethod != DMLibCopyMethod.ServiceSideAsyncCopy))
{
DMLibDataHelper.SetCalculatedFileMD5(result.DataInfo, DestAdaptor);
}
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
}
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
Helper.CompareSMBPermissions(
sourceDataInfo.RootNode,
result.DataInfo.RootNode,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
}
}
public void TestResume()
{
int fileSizeInKB = 100 * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, DMLibTestBase.FileName, fileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
var transferContext = new TransferContext();
var progressChecker = new ProgressChecker(1, fileSizeInKB * 1024);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
options.TransferItemModifier = (fileName, item) =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
secondCheckpoint = transferContext.LastCheckpoint;
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
VerificationHelper.VerifyExceptionErrorMessage(exception, "A task was canceled.");
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
// DMLib doesn't support to resume transfer from a checkpoint which is inconsistent with
// the actual transfer progress when the destination is an append blob.
if (Helper.RandomBoolean() && DMLibTestContext.DestType != DMLibDataType.AppendBlob)
{
Test.Info("Resume with the first checkpoint first.");
firstResumeCheckpoint = firstCheckpoint;
secondResumeCheckpoint = secondCheckpoint;
}
else
{
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = new TransferContext(firstResumeCheckpoint)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new TransferContext(secondResumeCheckpoint)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
if (DMLibTestContext.DestType != DMLibDataType.AppendBlob || DMLibTestContext.SourceType == DMLibDataType.Stream)
{
VerificationHelper.VerifySingleObjectResumeResult(result, sourceDataInfo);
}
else
{
Test.Assert(result.Exceptions.Count == 1, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
exception = result.Exceptions[0];
Test.Assert(exception is InvalidOperationException, "Verify reumse fails when checkpoint is inconsistent with the actual progress when destination is append blob.");
VerificationHelper.VerifyExceptionErrorMessage(exception, "Destination might be changed by other process or application.");
}
}