/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory
/// 2. Set their content type to "image/png".
/// </summary>
private static async Task BlobDirectoryUploadSample()
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob
};
// Register for transfer event.
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallback = (destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
context.ShouldTransferCallback = (source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
// Start the upload
var trasferStatus = await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString());
}
private void ConfigureContext(DirectoryTransferContext context)
{
var progressRateLimit = TimeSpan.FromSeconds(5);
context.ProgressHandler = new RateLimitedProgress <TransferStatus>(
progressRateLimit,
new Progress <TransferStatus>(progress =>
{
var bytesTransferred = progress.BytesTransferred;
var filesTransferred = progress.NumberOfFilesTransferred;
LogTransferProgress(bytesTransferred, filesTransferred);
}));
context.FileFailed += (s, e) => this.logger.LogWarning("Failed to download {0}", e.Destination);
context.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite;
}
private void TestDelimiter(char delimiter)
{
Test.Info("Test delimiter: {0}", delimiter);
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
string fileName = DMLibTestBase.FolderName + delimiter + DMLibTestBase.FolderName + delimiter + DMLibTestBase.FileName;
DMLibDataHelper.AddOneFile(sourceDataInfo.RootNode, fileName, 1);
TransferContext context = new DirectoryTransferContext();
context.FileFailed += (sender, e) =>
{
Test.Info(e.Exception.StackTrace);
};
TestExecutionOptions <DMLibDataInfo> options = new TestExecutionOptions <DMLibDataInfo>()
{
IsDirectoryTransfer = true,
TransferItemModifier = (node, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
dirOptions.Delimiter = delimiter;
item.Options = dirOptions;
item.TransferContext = context;
}
};
TestResult <DMLibDataInfo> result = this.ExecuteTestCase(sourceDataInfo, options);
DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty);
DirNode dirNode1 = new DirNode(FolderName);
DirNode dirNode2 = new DirNode(FolderName);
FileNode fileNode = sourceDataInfo.RootNode.GetFileNode(fileName).Clone(DMLibTestBase.FileName);
dirNode2.AddFileNode(fileNode);
dirNode1.AddDirNode(dirNode2);
expectedDataInfo.RootNode.AddDirNode(dirNode1);
VerificationHelper.VerifySingleTransferStatus(result, 1, 0, 0, null);
VerificationHelper.VerifyTransferSucceed(result, expectedDataInfo);
}
private Task <TransferStatus> UploadDirectory(dynamic destObject, TransferItem item)
{
UploadDirectoryOptions uploadDirectoryOptions = item.Options as UploadDirectoryOptions;
DirectoryTransferContext transferContrext = item.TransferContext as DirectoryTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string sourcePath = item.SourceObject as string;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext, cancellationToken));
}
else if (transferContrext != null || uploadDirectoryOptions != null)
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext));
}
else
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject));
}
}
public async Task DownloadLibraryAsync(string key)
{
try
{
var setting = CloudConfigurationManager.GetSetting(AppSettings.StorageConnectionString);
var account = CreateStorageAccountFromConnectionString(setting);
var client = account.CreateCloudFileClient();
var share = client.GetShareReference("library");
var root = share.GetRootDirectoryReference();
if (!await root.ExistsAsync())
{
return;
}
var dirName = $"Library/{key}";
var directory = root.GetDirectoryReference(dirName);
if (!await directory.ExistsAsync())
{
throw new DirectoryNotFoundException($"Directory does not exist {dirName}");
}
TransferManager.Configurations.ParallelOperations = 64;
var context = new DirectoryTransferContext();
var libraryPath = Path.Combine(_settings.LocalLibraryPath, key);
await TransferManager.DownloadDirectoryAsync(directory, libraryPath, new DownloadDirectoryOptions { Recursive = true }, context, CancellationToken.None);
}
catch (Exception ex)
{
_logger.Log(ex);
}
}
public async Task <TransferStatus?> ExecuteAsync(
CloudBlobDirectory blobDirectory,
string outputDirectoryPath,
DownloadDirectoryOptions options,
DirectoryTransferContext context,
CancellationToken cancellationToken)
{
try
{
return(await TransferManager.DownloadDirectoryAsync(
blobDirectory,
outputDirectoryPath,
options,
context,
cancellationToken));
}
catch (Exception t) when(ExceptionUtilities.IsFromCancellation(t))
{
return(null);
}
}
/// <summary>
/// Moves a folder with its contents to the path given
/// </summary>
/// <param name="folder"></param>
/// <param name="path"></param>
/// <returns></returns>
public async Task <BlobDto> MoveFolder(BlobDto folder, string path)
{
var container = await GetContainer();
if (path[path.Length - 1] != '/')
{
path = path + "/";
}
string newPath = $"{path}{folder.Name}";
var sourceBlobDir = container.GetDirectoryReference(folder.Path);
var destBlobDir = container.GetDirectoryReference(newPath);
TransferManager.Configurations.ParallelOperations = 64;
// Setup the transfer context and track the upoload progress
DirectoryTransferContext context = new DirectoryTransferContext
{
ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
})
};
var copyDirOptions = new CopyDirectoryOptions
{
Recursive = true,
IncludeSnapshots = true
};
await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, true, copyDirOptions, context);
await DeleteFile(folder.Path);
folder.Path = newPath;
folder.StoragePath = destBlobDir.Uri.ToString();
folder.DateModified = DateTime.UtcNow;
return(folder);
}
public async Task RunAsync(CancellationToken cancellationToken = default(CancellationToken))
{
var copyDirectoryOptions = new CopyDirectoryOptions
{
Recursive = true
};
var directoryTransferContext = new DirectoryTransferContext
{
//ProgressHandler = new Progress<TransferStatus>(progress => Log.Trace(() => $"Progress: transferred: {progress.NumberOfFilesTransferred}, failed: {progress.NumberOfFilesFailed}, skipped: {progress.NumberOfFilesSkipped}, bytes transferred: {progress.BytesTransferred}"))
};
directoryTransferContext.FileTransferred += (sender, args) => Log.Trace(() => $"Transferred {(args.Source as CloudFile)?.Name} -> {(args.Destination as CloudFile)?.Name}");
directoryTransferContext.FileSkipped += (sender, args) => Log.Trace(() => $"Skipped {(args.Source as CloudFile)?.Name} -> {(args.Destination as CloudFile)?.Name}");
directoryTransferContext.FileFailed += (sender, args) => Log.Error(() => $"Failed {(args.Source as CloudFile)?.Name} -> {(args.Destination as CloudFile)?.Name}");
Log.Info(() => $"Starting server side copy from {this.sourceDirectory.Uri} to {this.targetDirectory.Uri}");
var status = await TransferManager.CopyDirectoryAsync(sourceDirectory, targetDirectory, true, copyDirectoryOptions, directoryTransferContext, cancellationToken);
Log.Info(() => $"Finished server side copy, transferred: {status.NumberOfFilesTransferred}, failed: {status.NumberOfFilesFailed}, skipped: {status.NumberOfFilesSkipped}, bytes transferred: {status.BytesTransferred}");
}
public async Task TransferLocalDirectoryToAzureBlobDirectory(string LocalSourceDirPath, string ContainerName)
{
CloudBlobDirectory blobDirectory = GetBlobDirectory(ContainerName);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Task task;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(LocalSourceDirPath, blobDirectory, options, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
if (_stopFlag)
{
return;
}
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetDirectoryTransferContext(checkpoint);
await TransferManager.UploadDirectoryAsync(LocalSourceDirPath, blobDirectory, options, context);
}
}
protected async Task <bool> CheckShouldTransfer()
{
DirectoryTransferContext directoryTransferContext = this.TransferContext as DirectoryTransferContext;
if ((null != directoryTransferContext) &&
(null != directoryTransferContext.ShouldTransferCallbackAsync))
{
await Task.Yield();
bool shouldTransfer = true;
try
{
shouldTransfer = await directoryTransferContext.ShouldTransferCallbackAsync(this.TransferJob.Source.Instance, this.TransferJob.Destination.Instance);
}
catch (Exception ex)
{
this.FinishCallbackHandler(new TransferException(TransferErrorCode.FailedCheckingShouldTransfer, string.Empty, ex));
return(true);
}
if (shouldTransfer)
{
this.TransferJob.Transfer.ShouldTransferChecked = true;
}
else
{
this.TransferJob.Status = TransferJobStatus.SkippedDueToShouldNotTransfer;
this.FinishCallbackHandler(null);
return(true);
}
}
else
{
this.TransferJob.Transfer.ShouldTransferChecked = true;
}
return(false);
}
public async Task doCopy(DirectoryInfo source, string storageConnectionString)
{
try
{
// Connect to Azure Storage to create a container for this backup
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = account.CreateCloudBlobClient();
// Unique container name with timestamp
string container = source.Name.ToLower() + DateTime.Now.ToString("MMddyyyy-HHmmss");
CloudBlobContainer blobContainer = blobClient.GetContainerReference(container);
await blobContainer.CreateIfNotExistsAsync();
Console.WriteLine("Container {0} has been created.", container);
// Get root directory reference for the container
CloudBlobDirectory destBlob = blobContainer.GetDirectoryReference("");
// Parallel Operations
TransferManager.Configurations.ParallelOperations = 32;
// Setup the transfer context and track the upload progress
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileFailed += Program.FileFailedCallback;
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.WriteLine("{0} MB uploaded", progress.BytesTransferred / (1024 * 1024));
});
// Upload recursively
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
// Start the counter
Stopwatch s = Stopwatch.StartNew();
// Initiate the Upload from DMLib
TransferStatus transferStatus = await TransferManager.UploadDirectoryAsync(source.FullName, destBlob, options, context);
s.Stop();
if (transferStatus.NumberOfFilesFailed > 0)
{
Console.WriteLine("{0} files failed to transfer", transferStatus.NumberOfFilesFailed);
}
// Log the result
Console.WriteLine("Upload has been completed in {0} seconds.", s.Elapsed.TotalSeconds);
}
catch (StorageException ex)
{
Console.WriteLine(ex.Message);
}
catch (Exception ex)
{
ex = (ex.InnerException != null) ? ex.InnerException.GetBaseException() : ex;
Console.WriteLine(ex.Message);
}
}
public void TestDirectoryPreserveSMBPropertiesResume()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
CloudFileNtfsAttributes[] SMBFileAttributes =
{
CloudFileNtfsAttributes.ReadOnly,
CloudFileNtfsAttributes.Hidden,
#if DEBUG
CloudFileNtfsAttributes.System,
CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Normal,
CloudFileNtfsAttributes.Offline,
CloudFileNtfsAttributes.NotContentIndexed,
CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden,
CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly |
CloudFileNtfsAttributes.Hidden |
CloudFileNtfsAttributes.System |
CloudFileNtfsAttributes.Archive |
CloudFileNtfsAttributes.NotContentIndexed |
CloudFileNtfsAttributes.NoScrubData
#endif
};
for (int i = 0; i < SMBFileAttributes.Length; ++i)
{
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]);
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]);
DirectoryTransferContext dirTransferContext = null;
if (IsStreamJournal)
{
dirTransferContext = new DirectoryTransferContext(journalStream);
}
else
{
dirTransferContext = new DirectoryTransferContext();
}
var progressChecker = new ProgressChecker(14, 14 * 1024);
dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, item) =>
{
item.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.CancellationToken = tokenSource.Token;
transferItem = item;
};
TransferCheckpoint checkpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
checkpoint = dirTransferContext.LastCheckpoint;
}
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = (path) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
return(FileAttributes.Directory);
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory));
}
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i]));
}
};
TestHookCallbacks.SetFileAttributesCallback = (path, attributes) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot"))
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory),
"Directory attributes should be expected {0}", attributes);
}
}
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]),
"File attributes should be expected {0}", attributes);
}
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferItem resumeItem = transferItem.Clone();
DirectoryTransferContext resumeContext = null;
journalStream.Position = 0;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream);
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint));
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
IsDirectoryTransfer = true,
Options = DefaultTransferDirectoryOptions
};
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.TransferContext = new DirectoryTransferContext();
item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) =>
{
return(false);
};
List <TransferItem> items = new List <TransferItem>();
items.Add(item);
result = this.RunTransferItems(items, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.Local)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false);
}
else
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
}
}
finally
{
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = null;
TestHookCallbacks.SetFileAttributesCallback = null;
#endif
}
}
}
}
public void TestDirectoryPreserveSMBProperties()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
CloudFileNtfsAttributes[] SMBFileAttributes =
{
CloudFileNtfsAttributes.ReadOnly,
CloudFileNtfsAttributes.Hidden,
#if DEBUG
CloudFileNtfsAttributes.System,
CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Normal,
CloudFileNtfsAttributes.Offline,
CloudFileNtfsAttributes.NotContentIndexed,
CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly | CloudFileNtfsAttributes.Hidden,
CloudFileNtfsAttributes.System | CloudFileNtfsAttributes.Archive,
CloudFileNtfsAttributes.Offline | CloudFileNtfsAttributes.NotContentIndexed | CloudFileNtfsAttributes.NoScrubData,
CloudFileNtfsAttributes.ReadOnly |
CloudFileNtfsAttributes.Hidden |
CloudFileNtfsAttributes.System |
CloudFileNtfsAttributes.Archive |
CloudFileNtfsAttributes.NotContentIndexed |
CloudFileNtfsAttributes.NoScrubData
#endif
};
for (int i = 0; i < SMBFileAttributes.Length; ++i)
{
Test.Info("Testing setting attributes {0} to directories", SMBFileAttributes[i]);
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithAttributes(sourceDataInfo.RootNode, 2, SMBFileAttributes[i]);
DirectoryTransferContext dirTransferContext = new DirectoryTransferContext();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
transferItem.Options = transferOptions;
};
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = (path) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
return(FileAttributes.Directory);
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory));
}
}
else
{
return(Helper.ToFileAttributes(SMBFileAttributes[i]));
}
};
TestHookCallbacks.SetFileAttributesCallback = (path, attributes) =>
{
if (path.EndsWith("\\") || path.Substring(0, path.Length - 2).EndsWith(DMLibTestBase.DirName))
{
if (path.Substring(0, path.Length - 7).EndsWith("destroot") || path.Substring(0, path.Length - 6).EndsWith("destroot"))
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
if (SMBFileAttributes[i] == CloudFileNtfsAttributes.Normal)
{
Test.Assert(attributes == FileAttributes.Directory, "Directory attributes should be expected {0}", attributes);
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i] | CloudFileNtfsAttributes.Directory),
"Directory attributes should be expected {0}", attributes);
}
}
}
else
{
Test.Assert(attributes == Helper.ToFileAttributes(SMBFileAttributes[i]),
"File attributes should be expected {0}", attributes);
}
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
TransferItem item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
DestObject = DestAdaptor.GetTransferObject(string.Empty, sourceDataInfo.RootNode),
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
IsDirectoryTransfer = true,
Options = DefaultTransferDirectoryOptions
};
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBAttributes = true;
item.Options = transferOptions;
item.TransferContext = new DirectoryTransferContext();
item.TransferContext.ShouldOverwriteCallbackAsync = async(s, d) =>
{
return(false);
};
List <TransferItem> items = new List <TransferItem>();
items.Add(item);
result = this.RunTransferItems(items, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.Local)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, false);
}
else
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
}
}
finally
{
#if DEBUG
TestHookCallbacks.GetFileAttributesCallback = null;
TestHookCallbacks.SetFileAttributesCallback = null;
#endif
}
}
}
public void TestDirectoryPreserveSMBPermissionsResume()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
PreserveSMBPermissions preserveSMBPermissions =
PreserveSMBPermissions.Owner
| PreserveSMBPermissions.Group
| PreserveSMBPermissions.DACL;
string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)";
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
bool IsStreamJournal = random.Next(0, 2) == 0;
using (Stream journalStream = new MemoryStream())
{
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
GenerateDirNodeWithPermissions(sourceDataInfo.RootNode, 2, sampleSDDL);
DirectoryTransferContext dirTransferContext = null;
if (IsStreamJournal)
{
dirTransferContext = new DirectoryTransferContext(journalStream);
}
else
{
dirTransferContext = new DirectoryTransferContext();
}
var progressChecker = new ProgressChecker(14, 14 * 1024);
dirTransferContext.ProgressHandler = progressChecker.GetProgressHandler();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, item) =>
{
item.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBPermissions = preserveSMBPermissions;
item.Options = transferOptions;
item.CancellationToken = tokenSource.Token;
transferItem = item;
};
TransferCheckpoint checkpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
checkpoint = dirTransferContext.LastCheckpoint;
}
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
#if DEBUG
TestHookCallbacks.UnderTesting = true;
TestHookCallbacks.GetFilePermissionsCallback = (path, SMBPermissionType) =>
{
Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected.");
return(sampleSDDL);
};
TestHookCallbacks.SetFilePermissionsCallback = (path, portableSDDL, SMBPermissionType) =>
{
Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected.");
Test.Assert(portableSDDL.StartsWith(sampleSDDL),
"The SDDL value should be expected.");
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
Test.Assert(result.Exceptions.Count == 1, "Verify job is cancelled");
Exception exception = result.Exceptions[0];
Helper.VerifyCancelException(exception);
TransferItem resumeItem = transferItem.Clone();
DirectoryTransferContext resumeContext = null;
journalStream.Position = 0;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream);
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(checkpoint));
}
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBPermissions(sourceDataInfo.RootNode, result.DataInfo.RootNode, preserveSMBPermissions);
}
}
finally
{
#if DEBUG
TestHookCallbacks.UnderTesting = false;
TestHookCallbacks.GetFilePermissionsCallback = null;
TestHookCallbacks.SetFilePermissionsCallback = null;
#endif
}
}
}
/// <summary>
/// Copies the content of one directory to another on the server side.
/// Avoids having to download all items and reupload them to somewhere else on the client side.
/// </summary>
/// <param name="sourceDirectoryPath">The path to the directory containing the files to be transferred</param>
/// <param name="destinationDirectoryPath">The path to the destination directory </param>
/// <param name="transferEvent">Action callback for tracking progress of each file when transferring.</param>
/// <returns>Result from the server side transfer.</returns>
public async Task <ITransferResult> CopyDirectory(string sourceDirectoryPath, string destinationDirectoryPath, Action <Core.TransferEventType, ITransferEvent> transferEvent = null)
{
var sourceContainerName = GetContainerFromPath(sourceDirectoryPath);
var destinationContainerName = GetContainerFromPath(destinationDirectoryPath);
// Ensure destination folder exists if we've configured to create automatically.
if (CreateFolderIfNotExists)
{
GetContainer(destinationContainerName, true);
}
var directoryTransferContext = new DirectoryTransferContext();
// Subscribe to the transfer events if an action method was passed.
if (transferEvent != null)
{
directoryTransferContext.FileTransferred += (fileTransferSender, fileTransferredEventArgs) =>
{
ITransferEvent i = (TransferEvent)fileTransferredEventArgs;
transferEvent(Core.TransferEventType.Transferred, i);
};
directoryTransferContext.FileFailed += (fileFailedSender, fileTransferredEventArgs) =>
{
ITransferEvent i = (TransferEvent)fileTransferredEventArgs;
transferEvent(Core.TransferEventType.Failed, i);
};
directoryTransferContext.FileSkipped += (fileSkippedSender, fileTransferredEventArgs) =>
{
ITransferEvent i = (TransferEvent)fileTransferredEventArgs;
transferEvent(Core.TransferEventType.Skipped, i);
};
}
directoryTransferContext.ShouldOverwriteCallbackAsync = (source, destination) => Task.FromResult(true);
var copyOptions = new CopyDirectoryOptions {
BlobType = BlobType.AppendBlob, Recursive = true
};
var sourceContainer = CloudBlobClient.GetContainerReference(sourceContainerName);
var sourceRelativeUrl = GetPathWithoutContainer(sourceDirectoryPath);
var sourceDirectory = sourceContainer.GetDirectoryReference(sourceRelativeUrl);
var destinationContainer = CloudBlobClient.GetContainerReference(destinationContainerName);
var destinationRelativeUrl = GetPathWithoutContainer(destinationDirectoryPath);
var destinationDirectory = destinationContainer.GetDirectoryReference(destinationRelativeUrl);
var transferTask = TransferManager.CopyDirectoryAsync(sourceDirectory, destinationDirectory, CopyMethod.ServiceSideSyncCopy, copyOptions, directoryTransferContext);
var result = await transferTask;
// Output the result from the transfer.
return(new TransferResult
{
BytesTransferred = result.BytesTransferred,
NumberOfFilesFailed = result.NumberOfFilesFailed,
NumberOfFilesSkipped = result.NumberOfFilesSkipped,
NumberOfFilesTransferred = result.NumberOfFilesTransferred
});
}
public void DirectoryShouldTransfer()
{
// Prepare data
int totaFileNumber = DMLibTestConstants.FlatFileCount;
int expectedTransferred = totaFileNumber, transferred = 0;
int expectedSkipped = 0, skipped = 0;
int expectedFailed = 0, failed = 0;
DMLibDataInfo sourceDataInfo = this.GenerateSourceDataInfo(FileNumOption.FlatFolder, 1024);
DirectoryTransferContext dirTransferContext = new DirectoryTransferContext();
List <String> notTransferredFileNames = new List <String>();
dirTransferContext.ShouldTransferCallbackAsync = async(source, dest) =>
{
if (Helper.RandomBoolean())
{
return(true);
}
else
{
Interlocked.Decrement(ref expectedTransferred);
string fullName = DMLibTestHelper.TransferInstanceToString(source);
string fileName = fullName.Substring(fullName.IndexOf(DMLibTestBase.FileName));
lock (notTransferredFileNames)
{
notTransferredFileNames.Add(fileName);
}
Test.Info("{0} is filterred in ShouldTransfer.", fileName);
return(false);
}
};
dirTransferContext.FileTransferred += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref transferred);
};
dirTransferContext.FileSkipped += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref skipped);
};
dirTransferContext.FileFailed += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref failed);
};
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferItem.Options = transferOptions;
};
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
// Verify result
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DirNode expectedRootNode = expectedDataInfo.RootNode;
foreach (string fileNames in notTransferredFileNames)
{
expectedRootNode.DeleteFileNode(fileNames);
}
VerificationHelper.VerifyTransferSucceed(result, expectedDataInfo);
Test.Assert(expectedTransferred == transferred, string.Format("Verify transferred number. Expected: {0}, Actual: {1}", expectedTransferred, transferred));
Test.Assert(expectedSkipped == skipped, string.Format("Verify skipped number. Expected: {0}, Actual: {1}", expectedSkipped, skipped));
Test.Assert(expectedFailed == failed, string.Format("Verify failed number. Expected: {0}, Actual: {1}", expectedFailed, failed));
}
public void TestDirectorySetAttributes()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
for (int i = 0; i < 3; ++i)
{
FileNode fileNode = new FileNode(DMLibTestBase.FileName + i)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L
};
if (DMLibTestContext.SourceType != DMLibDataType.Local)
{
fileNode.Metadata = new Dictionary <string, string>();
fileNode.Metadata.Add("foo", "bar");
fileNode.ContentLanguage = SetAttributesTest.TestContentLanguage;
}
sourceDataInfo.RootNode.AddFileNode(fileNode);
}
DirectoryTransferContext context = new DirectoryTransferContext()
{
SetAttributesCallbackAsync = async(destObj) =>
{
dynamic destCloudObj = destObj;
destCloudObj.Properties.ContentType = SetAttributesTest.TestContentType;
destCloudObj.Metadata.Add("aa", "bb");
}
};
var options = new TestExecutionOptions <DMLibDataInfo>();
options.TransferItemModifier = (node, transferItem) =>
{
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferItem.Options = transferOptions;
transferItem.TransferContext = context;
};
options.IsDirectoryTransfer = true;
var result = this.ExecuteTestCase(sourceDataInfo, options);
foreach (FileNode fileNode in sourceDataInfo.EnumerateFileNodes())
{
fileNode.Metadata.Add("aa", "bb");
}
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
VerificationHelper.VerifySingleTransferStatus(result, 3, 0, 0, 3 * DMLibTestBase.FileSizeInKB * 1024L);
foreach (FileNode destFileNode in result.DataInfo.EnumerateFileNodes())
{
Test.Assert(TestContentType.Equals(destFileNode.ContentType), "Verify content type: {0}, expected {1}", destFileNode.ContentType, TestContentType);
if (DMLibTestContext.SourceType != DMLibDataType.Local)
{
Test.Assert(SetAttributesTest.TestContentLanguage.Equals(destFileNode.ContentLanguage), "Verify ContentLanguage: {0}, expected {1}", destFileNode.ContentLanguage, SetAttributesTest.TestContentLanguage);
}
}
}
/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory
/// and store transfer context in a streamed journal.
/// 2. Set their content type to "image/png".
/// 3. Cancel the transfer before it finishes with a CancellationToken
/// 3. Reload the transfer context from the streamed journal.
/// 4. Resume the transfer with the loaded transfer context
/// </summary>
private static async Task BlobDirectoryUploadSample()
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob
};
using (MemoryStream journalStream = new MemoryStream())
{
// Store the transfer context in a streamed journal.
DirectoryTransferContext context = new DirectoryTransferContext(journalStream);
// Register for transfer event.
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallbackAsync = async(destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
context.ShouldTransferCallbackAsync = async(source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferStatus trasferStatus = null;
try
{
// Start the upload
Task <TransferStatus> task = TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
trasferStatus = await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
journalStream.Position = 0;
// Deserialize transfer context from the streamed journal.
DirectoryTransferContext resumeContext = new DirectoryTransferContext(journalStream);
resumeContext.FileTransferred += FileTransferredCallback;
resumeContext.FileFailed += FileFailedCallback;
resumeContext.FileSkipped += FileSkippedCallback;
resumeContext.SetAttributesCallbackAsync = async(destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
resumeContext.ShouldTransferCallbackAsync = async(source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
// Resume the upload
trasferStatus = await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, resumeContext);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString());
}
}
public static DirectoryTransferContext GetDirectoryTransferContext(TransferCheckpoint checkpoint)
{
DirectoryTransferContext context = new DirectoryTransferContext(checkpoint);
return(context);
}
private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5)
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string wrongMD5File = "wrongMD5File";
string correctMD5File = "correctMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
DestAdaptor.Cleanup();
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
using (var resumeStream = new MemoryStream())
{
TransferContext context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti,
totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = !checkMD5,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, executionOption);
if (null != testResult.Exceptions)
{
foreach (var exception in testResult.Exceptions)
{
Test.Info("Got exception during transferring. {0}", exception);
}
}
eventChecker = new TransferEventChecker();
resumeStream.Position = 0;
context = new DirectoryTransferContext(resumeStream);
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(wrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
checkMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected.");
VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0);
if (checkMD5)
{
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
else
{
Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5");
}
}
}
public void TestDirectoryCheckContentMD5()
{
long fileSize = 5 * 1024 * 1024;
long totalSize = fileSize * 4;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, checkCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckWrongMD5File, fileSize);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, notCheckCorrectMD5File, fileSize);
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
FileNode tmpFileNode = checkMD5Folder.GetFileNode(checkWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
tmpFileNode = notCheckMD5Folder.GetFileNode(notCheckWrongMD5File);
tmpFileNode.MD5 = wrongMD5;
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
TransferContext context = new DirectoryTransferContext();
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
};
ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
context.FileFailed += (eventSource, eventArgs) =>
{
transferExceptions.Add(eventArgs.Exception);
};
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
CopyMethod = DMLibTestContext.CopyMethod.ToCopyMethod(),
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
};
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
DMLibDataInfo actualDataInfo = testResult.DataInfo;
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode(checkWrongMD5File);
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode(notCheckWrongMD5File);
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3, 0, 1);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != 1)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
private void TestDirectorySetAttribute_Restart(
int bigFileSizeInKB,
int smallFileSizeInKB,
int bigFileNum,
int smallFileNum,
Action <DirNode> bigFileDirAddFileAction,
Action <DirNode> smallFileDirAddFileAction,
SetAttributesCallbackAsync setAttributesCallback = null,
Action <DMLibDataInfo> sourceDataInfoDecorator = null)
{
int totalFileNum = bigFileNum + smallFileNum;
long totalSizeInBytes = ((bigFileSizeInKB * bigFileNum) + (smallFileSizeInKB * smallFileNum)) * 1024;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
bigFileDirAddFileAction(bigFileDirNode);
smallFileDirAddFileAction(smallFileDirNode);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo> {
LimitSpeed = true, IsDirectoryTransfer = true
};
using (Stream journalStream = new MemoryStream())
{
bool isStreamJournal = random.Next(0, 2) == 0;
var transferContext = isStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext();
transferContext.SetAttributesCallbackAsync = setAttributesCallback;
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Helper.VerifyCancelException(e.Exception);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!isStreamJournal)
{
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
if (!isStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = null;
if (isStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
resumeContext.SetAttributesCallbackAsync = setAttributesCallback;
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
sourceDataInfoDecorator?.Invoke(sourceDataInfo);
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (!isStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
SetAttributesCallbackAsync = setAttributesCallback
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
}
}
private void ResumeInAllDirectionsHelper(bool directoryTransfer)
{
List <TransferItem> allItems = directoryTransfer ? AllTransferDirectionTest.GetTransformItemsForAllDirTransferDirections(resume: true)
: AllTransferDirectionTest.GetTransformItemsForAllSingleTransferDirections(true);
int fileCount = expectedFileNodes.Keys.Count;
// Execution and store checkpoints
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferContext transferContext = null;
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext();
}
else
{
transferContext = new SingleTransferContext();
}
var progressChecker = new ProgressChecker(fileCount, 1024 * fileCount);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
allItems.ForEach(item =>
{
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
});
var options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
// Checkpoint names
const string PartialStarted = "PartialStarted",
AllStarted = "AllStarted",
AllStartedAndWait = "AllStartedAndWait",
BeforeCancel = "BeforeCancel",
AfterCancel = "AfterCancel";
Dictionary <string, TransferCheckpoint> checkpoints = new Dictionary <string, TransferCheckpoint>();
TransferItem randomItem = allItems[random.Next(0, allItems.Count)];
randomItem.AfterStarted = () =>
{
Test.Info("Store check point after transfer item: {0}.", randomItem.ToString());
checkpoints.Add(PartialStarted, transferContext.LastCheckpoint);
};
options.AfterAllItemAdded = () =>
{
if (!progressChecker.DataTransferred.WaitOne(30000))
{
Test.Error("No progress in 30s.");
}
checkpoints.Add(AllStarted, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(AllStartedAndWait, transferContext.LastCheckpoint);
Thread.Sleep(1000);
checkpoints.Add(BeforeCancel, transferContext.LastCheckpoint);
tokenSource.Cancel();
checkpoints.Add(AfterCancel, transferContext.LastCheckpoint);
};
var result = this.RunTransferItems(allItems, options);
// Resume with stored checkpoints in random order
var checkpointList = new List <KeyValuePair <string, TransferCheckpoint> >();
checkpointList.AddRange(checkpoints);
checkpointList.Shuffle();
foreach (var pair in checkpointList)
{
Test.Info("===Resume with checkpoint '{0}'===", pair.Key);
options = new TestExecutionOptions <DMLibDataInfo>();
options.DisableDestinationFetch = true;
progressChecker.Reset();
if (directoryTransfer)
{
transferContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
else
{
transferContext = new SingleTransferContext(DMLibTestHelper.RandomReloadCheckpoint(pair.Value))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// The checkpoint can be stored when DMLib doesn't check overwrite callback yet.
// So it will case an skip file error if the desination file already exists and
// We don't have overwrite callback here.
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY()
};
}
int expectedFailureCount = 0;
transferContext.FileFailed += (resource, eventArgs) =>
{
TransferException exception = eventArgs.Exception as TransferException;
if (null != exception && exception.ErrorCode == TransferErrorCode.MismatchCopyId)
{
Interlocked.Increment(ref expectedFailureCount);
}
};
TransferEventChecker eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
List <TransferItem> itemsToResume = allItems.Select(item =>
{
TransferItem itemToResume = item.Clone();
itemToResume.TransferContext = transferContext;
return(itemToResume);
}).ToList();
result = this.RunTransferItems(itemsToResume, options);
foreach (DMLibDataType destDataType in DataTypes)
{
if (DMLibDataType.URI == destDataType)
{
continue;
}
DataAdaptor <DMLibDataInfo> destAdaptor = GetDestAdaptor(destDataType);
DMLibDataInfo destDataInfo = destAdaptor.GetTransferDataInfo(string.Empty);
foreach (FileNode destFileNode in destDataInfo.EnumerateFileNodes())
{
string fileName = destFileNode.Name;
FileNode sourceFileNode = expectedFileNodes[fileName];
Test.Assert(DMLibDataHelper.Equals(sourceFileNode, destFileNode), "Verify transfer result.");
}
}
if (!directoryTransfer)
{
Test.Assert(result.Exceptions.Count == expectedFailureCount, "Verify no error happens. Expect {0}, Actual: {1}", expectedFailureCount, result.Exceptions.Count);
}
else
{
Test.Assert(result.Exceptions.Count == 0, "Verify no exception happens. Actual: {0}", result.Exceptions.Count);
Test.Assert(eventChecker.FailedFilesNumber == expectedFailureCount, "Verify no unexpected error happens. Expect {0}, Actual: {1}", expectedFailureCount, eventChecker.FailedFilesNumber);
}
}
}
private async Task <bool> InternalUploadSessionAsync(Session session, IProgress <TransferStatus> progressHandler, CancellationToken cancellationToken)
{
var srcDirectoryPath = session.LocalRecordingFolderPath;
var dstBlob = await GetDirectoryBlobAsync(session.Id).ConfigureAwait(false);
var journalFile = this.GetJournalFileForSession(srcDirectoryPath);
using var journalStream = await this.GetJournalStream(session, journalFile).ConfigureAwait(false);
using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, this.disposeCancellation.Token);
using var reg = linkedCancellation.Token.Register(() =>
{
this.logger.LogDebug($"The transfer has been cancelled and the journal file for session {session.Id} will be saved");
journalStream.Position = 0;
using var journal = File.Create(journalFile);
journalStream.CopyTo(journal);
});
try
{
var options = new UploadDirectoryOptions()
{
SearchPattern = FileUploadSearchPattern,
Recursive = false,
BlobType = BlobType.BlockBlob
};
var context = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressHandler,
LogLevel = Microsoft.Azure.Storage.LogLevel.Informational
};
var transferTcs = new TaskCompletionSource <bool>();
context.FileFailed += (sender, e) =>
{
this.logger.LogInformation(e.Exception, $"Transfer fails. {e.Source} -> {e.Destination}.");
transferTcs.TrySetException(e.Exception);
if (!linkedCancellation.IsCancellationRequested)
{
linkedCancellation.Cancel();
}
};
context.FileSkipped += (sender, e) =>
{
this.logger.LogInformation($"Transfer skips. {e.Source} -> {e.Destination}.");
};
context.FileTransferred += (sender, e) =>
{
this.logger.LogInformation(e.Exception, $"Transfer succeds. {e.Source} -> {e.Destination}.");
};
// Start the upload
var trasferStatus = await TransferManager.UploadDirectoryAsync(srcDirectoryPath, dstBlob, options, context, linkedCancellation.Token).ConfigureAwait(false);
transferTcs.TrySetResult(true);
await transferTcs.Task.ConfigureAwait(false);
}
catch (OperationCanceledException e)
{
this.logger.LogInformation(e, $"The transfer {session.Id} has been cancelled");
}
catch (InvalidOperationException e) when(e.Message.Contains("Only one transfer is allowed with stream journal."))
{
this.logger.LogWarning(e, $"The transfer {session.Id} has failed");
File.Delete(journalFile);
throw;
}
catch (Exception e)
{
linkedCancellation.Cancel();
this.logger.LogWarning(e, $"The transfer {session.Id} has failed");
throw;
}
if (!linkedCancellation.IsCancellationRequested)
{
File.Delete(journalFile);
return(true);
}
return(false);
}
public void TestDirectoryResume()
{
int bigFileSizeInKB = 5 * 1024; // 5 MB
int smallFileSizeInKB = 1; // 1 KB
int bigFileNum = 5;
int smallFileNum = 50;
long totalSizeInBytes = (bigFileSizeInKB * bigFileNum + smallFileSizeInKB * smallFileNum) * 1024;
int totalFileNum = bigFileNum + smallFileNum;
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode bigFileDirNode = new DirNode("big");
DirNode smallFileDirNode = new DirNode("small");
sourceDataInfo.RootNode.AddDirNode(bigFileDirNode);
sourceDataInfo.RootNode.AddDirNode(smallFileDirNode);
DMLibDataHelper.AddMultipleFiles(bigFileDirNode, FileName, bigFileNum, bigFileSizeInKB);
DMLibDataHelper.AddMultipleFiles(smallFileDirNode, FileName, smallFileNum, smallFileSizeInKB);
CancellationTokenSource tokenSource = new CancellationTokenSource();
TransferItem transferItem = null;
var options = new TestExecutionOptions <DMLibDataInfo>();
options.LimitSpeed = true;
options.IsDirectoryTransfer = true;
using (Stream journalStream = new MemoryStream())
{
bool IsStreamJournal = random.Next(0, 2) == 0;
var transferContext = IsStreamJournal ? new DirectoryTransferContext(journalStream) : new DirectoryTransferContext();
var progressChecker = new ProgressChecker(totalFileNum, totalSizeInBytes, totalFileNum, null, 0, totalSizeInBytes);
transferContext.ProgressHandler = progressChecker.GetProgressHandler();
var eventChecker = new TransferEventChecker();
eventChecker.Apply(transferContext);
transferContext.FileFailed += (sender, e) =>
{
Helper.VerifyCancelException(e.Exception);
};
options.TransferItemModifier = (fileName, item) =>
{
dynamic dirOptions = DefaultTransferDirectoryOptions;
dirOptions.Recursive = true;
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
dirOptions.PreserveSMBAttributes = true;
dirOptions.PreserveSMBPermissions = true;
}
item.Options = dirOptions;
item.CancellationToken = tokenSource.Token;
item.TransferContext = transferContext;
transferItem = item;
};
TransferCheckpoint firstCheckpoint = null, secondCheckpoint = null;
options.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
if (!IsStreamJournal)
{
// Store the first checkpoint
firstCheckpoint = transferContext.LastCheckpoint;
}
Thread.Sleep(1000);
// Cancel the transfer and store the second checkpoint
tokenSource.Cancel();
};
// Cancel and store checkpoint for resume
var result = this.ExecuteTestCase(sourceDataInfo, options);
if (progressChecker.FailedFilesNumber <= 0)
{
Test.Error("Verify file number in progress. Failed: {0}", progressChecker.FailedFilesNumber);
}
TransferCheckpoint firstResumeCheckpoint = null, secondResumeCheckpoint = null;
if (!IsStreamJournal)
{
secondCheckpoint = transferContext.LastCheckpoint;
Test.Info("Resume with the second checkpoint first.");
firstResumeCheckpoint = secondCheckpoint;
secondResumeCheckpoint = firstCheckpoint;
}
// resume with firstResumeCheckpoint
TransferItem resumeItem = transferItem.Clone();
progressChecker.Reset();
TransferContext resumeContext = null;
if (IsStreamJournal)
{
resumeContext = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
else
{
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(firstResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler()
};
}
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (!IsStreamJournal)
{
// resume with secondResumeCheckpoint
resumeItem = transferItem.Clone();
progressChecker.Reset();
resumeContext = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(secondResumeCheckpoint))
{
ProgressHandler = progressChecker.GetProgressHandler(),
// Need this overwrite callback since some files is already transferred to destination
ShouldOverwriteCallbackAsync = DMLibInputHelper.GetDefaultOverwiteCallbackY(),
};
eventChecker.Reset();
eventChecker.Apply(resumeContext);
resumeItem.TransferContext = resumeContext;
result = this.RunTransferItems(new List <TransferItem>()
{
resumeItem
}, new TestExecutionOptions <DMLibDataInfo>());
VerificationHelper.VerifyFinalProgress(progressChecker, totalFileNum, 0, 0);
VerificationHelper.VerifySingleTransferStatus(result, totalFileNum, 0, 0, totalSizeInBytes);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
if (DMLibTestContext.SourceType == DMLibDataType.CloudFile &&
DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBProperties(sourceDataInfo.RootNode, result.DataInfo.RootNode, true);
Helper.CompareSMBPermissions(
sourceDataInfo.RootNode,
result.DataInfo.RootNode,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
}
}
public void TestDirectoryCheckContentMD5Resume()
{
long fileSize = 5 * 1024;
int fileCountMulti = 32;
long totalSize = fileSize * 4 * fileCountMulti;
string wrongMD5 = "wrongMD5";
string checkWrongMD5File = "checkWrongMD5File";
string checkCorrectMD5File = "checkCorrectMD5File";
string notCheckWrongMD5File = "notCheckWrongMD5File";
string notCheckCorrectMD5File = "notCheckCorrectMD5File";
// Prepare data for transfer items with checkMD5
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DirNode checkMD5Folder = new DirNode("checkMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
checkMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(checkMD5Folder);
// Prepare data for transfer items with disabling MD5 check
DirNode notCheckMD5Folder = new DirNode("notCheckMD5");
for (int i = 0; i < fileCountMulti; ++i)
{
var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}")
{
SizeInByte = fileSize,
MD5 = wrongMD5
};
notCheckMD5Folder.AddFileNode(wrongMD5FileNode);
DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize);
}
sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder);
SourceAdaptor.GenerateData(sourceDataInfo);
TransferEventChecker eventChecker = new TransferEventChecker();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
TransferContext context = new DirectoryTransferContext();
eventChecker.Apply(context);
ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize);
context.ProgressHandler = progressChecker.GetProgressHandler();
List <Exception> transferExceptions = new List <Exception>();
TransferItem checkMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = false,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token,
};
TransferItem notCheckMD5Item = new TransferItem()
{
SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder),
IsDirectoryTransfer = true,
SourceType = DMLibTestContext.SourceType,
DestType = DMLibTestContext.DestType,
IsServiceCopy = DMLibTestContext.IsAsync,
TransferContext = context,
Options = new DownloadDirectoryOptions()
{
DisableContentMD5Validation = true,
Recursive = true,
},
CancellationToken = cancellationTokenSource.Token
};
var executionOption = new TestExecutionOptions <DMLibDataInfo>();
executionOption.AfterAllItemAdded = () =>
{
// Wait until there are data transferred
progressChecker.DataTransferred.WaitOne();
// Cancel the transfer and store the second checkpoint
cancellationTokenSource.Cancel();
};
executionOption.LimitSpeed = true;
var testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, executionOption);
eventChecker = new TransferEventChecker();
context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint));
eventChecker.Apply(context);
bool failureReported = false;
context.FileFailed += (sender, args) =>
{
if (args.Exception != null)
{
failureReported = args.Exception.Message.Contains(checkWrongMD5File);
}
transferExceptions.Add(args.Exception);
};
progressChecker.Reset();
context.ProgressHandler = progressChecker.GetProgressHandler();
checkMD5Item = checkMD5Item.Clone();
notCheckMD5Item = notCheckMD5Item.Clone();
checkMD5Item.TransferContext = context;
notCheckMD5Item.TransferContext = context;
testResult = this.RunTransferItems(new List <TransferItem>()
{
checkMD5Item, notCheckMD5Item
}, new TestExecutionOptions <DMLibDataInfo>());
DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone();
DMLibDataInfo actualDataInfo = testResult.DataInfo;
for (int i = 0; i < fileCountMulti; ++i)
{
expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}");
actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}");
}
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result.");
Test.Assert(failureReported, "Verify md5 check failure is reported.");
VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti);
if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti)
{
Test.Error("Expect one exception but actually no exception is thrown.");
}
else
{
for (int i = 0; i < fileCountMulti; ++i)
{
VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" });
}
}
}
public void DirectoryOverwriteDestination()
{
string destExistYName = "destExistY";
string destExistNName = "destExistN";
string destNotExistYName = "destNotExistY";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistYName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistNName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistYName, 1024);
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistYName, 1024);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistNName, 1024);
TransferContext transferContext = new DirectoryTransferContext();
transferContext.ShouldOverwriteCallback = (source, destination) =>
{
if (DMLibTestHelper.TransferInstanceToString(source).EndsWith(destExistNName))
{
return(false);
}
else
{
return(true);
}
};
int skipCount = 0;
int successCount = 0;
transferContext.FileSkipped += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref skipCount);
TransferException transferException = args.Exception as TransferException;
Test.Assert(transferException != null, "Verify the exception is a TransferException");
VerificationHelper.VerifyTransferException(transferException, TransferErrorCode.NotOverwriteExistingDestination,
"Skiped file", destExistNName);
};
transferContext.FileTransferred += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref successCount);
};
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
options.DestTransferDataInfo = destDataInfo;
}
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = transferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferItem.Options = transferOptions;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
DMLibDataInfo expectedDataInfo = new DMLibDataInfo(string.Empty);
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destExistYName));
expectedDataInfo.RootNode.AddFileNode(destDataInfo.RootNode.GetFileNode(destExistNName));
expectedDataInfo.RootNode.AddFileNode(sourceDataInfo.RootNode.GetFileNode(destNotExistYName));
}
else
{
expectedDataInfo = sourceDataInfo;
}
// Verify transfer result
Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, result.DataInfo), "Verify transfer result.");
// Verify exception
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
VerificationHelper.VerifySingleTransferStatus(result, 2, 1, 0, 1024 * 2);
Test.Assert(successCount == 2, "Verify success transfers");
Test.Assert(skipCount == 1, "Verify skipped transfer");
}
else
{
VerificationHelper.VerifySingleTransferStatus(result, 3, 0, 0, 1024 * 3);
Test.Assert(successCount == 3, "Very all transfers are success");
Test.Assert(skipCount == 0, "Very no transfer is skipped");
}
}
public void TestDirectoryPreserveSMBPermissions()
{
if (!CrossPlatformHelpers.IsWindows)
{
return;
}
// Prepare data
DMLibDataInfo sourceDataInfo = new DMLibDataInfo("");
string sampleSDDL = "O:S-1-5-21-2146773085-903363285-719344707-1375029G:S-1-5-21-2146773085-903363285-719344707-513D:(A;ID;FA;;;BA)(A;OICIIOID;GA;;;BA)(A;ID;FA;;;SY)(A;OICIIOID;GA;;;SY)(A;ID;0x1301bf;;;AU)(A;OICIIOID;SDGXGWGR;;;AU)(A;ID;0x1200a9;;;BU)(A;OICIIOID;GXGR;;;BU)";
GenerateDirNodeWithPermissions(sourceDataInfo.RootNode, 2, sampleSDDL);
DirectoryTransferContext dirTransferContext = new DirectoryTransferContext();
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
PreserveSMBPermissions preserveSMBPermissions =
PreserveSMBPermissions.Owner
| PreserveSMBPermissions.Group
| PreserveSMBPermissions.DACL;
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = dirTransferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferOptions.PreserveSMBPermissions = preserveSMBPermissions;
transferItem.Options = transferOptions;
};
#if DEBUG
TestHookCallbacks.UnderTesting = true;
TestHookCallbacks.GetFilePermissionsCallback = (path, SMBPermissionType) =>
{
Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected.");
return(sampleSDDL);
};
TestHookCallbacks.SetFilePermissionsCallback = (path, portableSDDL, SMBPermissionType) =>
{
Test.Assert(SMBPermissionType == preserveSMBPermissions, "The SMB permission type should be expected.");
Test.Assert(portableSDDL.StartsWith(sampleSDDL),
"The SDDL value should be expected.");
};
#endif
try
{
// Execute test case
var result = this.ExecuteTestCase(sourceDataInfo, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
if (DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
Helper.CompareSMBPermissions(sourceDataInfo.RootNode, result.DataInfo.RootNode, preserveSMBPermissions);
}
}
finally
{
#if DEBUG
TestHookCallbacks.UnderTesting = false;
TestHookCallbacks.GetFilePermissionsCallback = null;
TestHookCallbacks.SetFilePermissionsCallback = null;
#endif
}
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlobDirectory
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint into a file after transfer being cancelled
/// 4. Reload checkpoint from the file
/// 4. Resume the transfer with the loaded checkpoint
/// </summary>
private static async Task BlobDirectoryCopySample()
{
CloudBlobDirectory sourceBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
CloudBlobDirectory destBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir2");
// When source is CloudBlobDirectory:
// 1. If recursive is set to true, data movement library matches the source blob name against SearchPattern as prefix.
// 2. Otherwise, data movement library matches the blob with the exact name specified by SearchPattern.
//
// You can also replace the source directory with a CloudFileDirectory instance to copy data from Azure File Storage. If so:
// 1. If recursive is set to true, SearchPattern is not supported. Data movement library simply transfer all azure files
// under the source CloudFileDirectory and its sub-directories.
// 2. Otherwise, data movement library matches the azure file with the exact name specified by SearchPattern.
//
// In the following case, data movement library will copy all blobs with the prefix "azure" in source blob directory.
CopyDirectoryOptions options = new CopyDirectoryOptions()
{
SearchPattern = "azure",
Recursive = true,
};
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
TransferStatus trasferStatus = null;
try
{
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
trasferStatus = await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Serialize the checkpoint into a file
#if DOTNET5_4
var formatter = new DataContractSerializer(typeof(TransferCheckpoint));
#else
IFormatter formatter = new BinaryFormatter();
#endif
string tempFileName = Guid.NewGuid().ToString();
using (var stream = new FileStream(tempFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
formatter.Serialize(stream, checkpoint);
}
// Deserialize the checkpoint from the file
using (var stream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
checkpoint = formatter.Deserialize(stream) as TransferCheckpoint;
}
File.Delete(tempFileName);
// Create a new TransferContext with the store checkpoint
DirectoryTransferContext resumeContext = new DirectoryTransferContext(checkpoint);
resumeContext.FileTransferred += FileTransferredCallback;
resumeContext.FileFailed += FileFailedCallback;
resumeContext.FileSkipped += FileSkippedCallback;
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
resumeContext.ProgressHandler = recorder;
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
trasferStatus = await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, resumeContext);
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
}
public void DirectoryForceOverwriteTest()
{
string destExistName = "destExist";
string destNotExistName = "destNotExist";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destExistName, 1024);
DMLibDataHelper.AddOneFileInBytes(sourceDataInfo.RootNode, destNotExistName, 1024);
DMLibDataInfo destDataInfo = new DMLibDataInfo(string.Empty);
DMLibDataHelper.AddOneFileInBytes(destDataInfo.RootNode, destExistName, 1024);
TransferContext transferContext = new DirectoryTransferContext();
transferContext.ShouldOverwriteCallback = TransferContext.ForceOverwrite;
int skipCount = 0;
int successCount = 0;
transferContext.FileSkipped += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref skipCount);
};
transferContext.FileTransferred += (object sender, TransferEventArgs args) =>
{
Interlocked.Increment(ref successCount);
};
var options = new TestExecutionOptions <DMLibDataInfo>();
options.IsDirectoryTransfer = true;
if (DMLibTestContext.DestType != DMLibDataType.Stream)
{
options.DestTransferDataInfo = destDataInfo;
}
if (IsCloudService(DMLibTestContext.DestType))
{
SharedAccessPermissions permissions;
if (DMLibTestContext.IsAsync)
{
permissions = SharedAccessPermissions.Write | SharedAccessPermissions.Read;
}
else
{
permissions = SharedAccessPermissions.Write;
}
StorageCredentials destSAS = new StorageCredentials(DestAdaptor.GenerateSAS(permissions, (int)new TimeSpan(1, 0, 0, 0).TotalSeconds));
options.DestCredentials = destSAS;
}
options.TransferItemModifier = (fileNode, transferItem) =>
{
transferItem.TransferContext = transferContext;
dynamic transferOptions = DefaultTransferDirectoryOptions;
transferOptions.Recursive = true;
transferItem.Options = transferOptions;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
// Verify transfer result
Test.Assert(DMLibDataHelper.Equals(sourceDataInfo, result.DataInfo), "Verify transfer result.");
VerificationHelper.VerifySingleTransferStatus(result, 2, 0, 0, 1024 * 2);
Test.Assert(successCount == 2, "Verify success transfers");
Test.Assert(skipCount == 0, "Verify skipped transfer");
}
