public async Task DownloadAzureBlobDirectoryToLocalDirectory(string LocalSourceDirPath, string ContainerName, bool RecursiveDownload = true)
{
CloudBlobDirectory blobDirectory = GetBlobDirectory(ContainerName);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
DownloadDirectoryOptions options = new DownloadDirectoryOptions()
{
Recursive = RecursiveDownload
};
try
{
task = TransferManager.DownloadDirectoryAsync(blobDirectory, LocalSourceDirPath, options, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
public async Task doDownload(string containerName, DirectoryInfo localDir, string storageConnectionString)
{
try
{
// Connect to Azure Storage to download a container
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = account.CreateCloudBlobClient();
// Get the container and root directory reference
CloudBlobContainer blobContainer = blobClient.GetContainerReference(containerName);
CloudBlobDirectory rootDir = blobContainer.GetDirectoryReference("");
// Log
Console.WriteLine("Directory to be downloaded is {0} and {1}", rootDir.Container.Name, rootDir.StorageUri);
// Parallel Operations
TransferManager.Configurations.ParallelOperations = 32;
// Setup the transfer context and track the upoload progress
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileFailed += Program.FileFailedCallback;
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.WriteLine("{0} MB downloaded", progress.BytesTransferred / (1024 * 1024));
});
// Download recursively
DownloadDirectoryOptions options = new DownloadDirectoryOptions()
{
Recursive = true
};
// Start the counter
Stopwatch s = Stopwatch.StartNew();
// Initiate the download from DMLib
TransferStatus transferStatus = await TransferManager.DownloadDirectoryAsync(rootDir, localDir.ToString(), options, context);
s.Stop();
if (transferStatus.NumberOfFilesFailed > 0)
{
Console.WriteLine("{0} files failed to transfer", transferStatus.NumberOfFilesFailed);
}
// Log the result
Console.WriteLine("Download has been completed in {0} seconds", s.Elapsed.TotalSeconds);
}
catch (StorageException ex)
{
Console.WriteLine(ex.Message);
}
catch (Exception ex)
{
ex = (ex.InnerException != null) ? ex.InnerException.GetBaseException() : ex;
Console.WriteLine(ex.Message);
}
}
private Task <TransferStatus> DownloadDirectory(dynamic sourceObject, TransferItem item)
{
DownloadDirectoryOptions downloadDirectoryOptions = item.Options as DownloadDirectoryOptions;
DirectoryTransferContext transferContext = item.TransferContext as DirectoryTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string destPath = item.DestObject as string;
if (cancellationToken == null || cancellationToken == CancellationToken.None)
{
return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext));
}
else
{
return(TransferManager.DownloadDirectoryAsync(sourceObject, destPath, downloadDirectoryOptions, transferContext, cancellationToken));
}
}
public async Task DownloadLibraryAsync(string key)
{
try
{
var setting = CloudConfigurationManager.GetSetting(AppSettings.StorageConnectionString);
var account = CreateStorageAccountFromConnectionString(setting);
var client = account.CreateCloudFileClient();
var share = client.GetShareReference("library");
var root = share.GetRootDirectoryReference();
if (!await root.ExistsAsync())
{
return;
}
var dirName = $"Library/{key}";
var directory = root.GetDirectoryReference(dirName);
if (!await directory.ExistsAsync())
{
throw new DirectoryNotFoundException($"Directory does not exist {dirName}");
}
TransferManager.Configurations.ParallelOperations = 64;
var context = new DirectoryTransferContext();
var libraryPath = Path.Combine(_settings.LocalLibraryPath, key);
await TransferManager.DownloadDirectoryAsync(directory, libraryPath, new DownloadDirectoryOptions { Recursive = true }, context, CancellationToken.None);
}
catch (Exception ex)
{
_logger.Log(ex);
}
}
public async Task <TransferStatus?> ExecuteAsync(
CloudBlobDirectory blobDirectory,
string outputDirectoryPath,
DownloadDirectoryOptions options,
DirectoryTransferContext context,
CancellationToken cancellationToken)
{
try
{
return(await TransferManager.DownloadDirectoryAsync(
blobDirectory,
outputDirectoryPath,
options,
context,
cancellationToken));
}
catch (Exception t) when(ExceptionUtilities.IsFromCancellation(t))
{
return(null);
}
}
/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory.
/// 2. Store source file's file attributes and permissions into destination blob's meta data.
/// 3. Download png files starting with "azure" in the source directory to a local directory, not including the sub-directory.
/// 4. Restore file attributes and permissions to destination local file.
/// </summary>
private static async Task BlobDirectoryPreserveFilePropertiesSampleAsync()
{
//Enable required privileges before getting/setting permissions from/to local file system.
FileSecurityOperations.EnableRequiredPrivileges(PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL, true);
try
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob
};
DirectoryTransferContext context = new DirectoryTransferContext();
// Register for transfer event.
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallbackAsync = async(sourceObj, destination) =>
{
string sourcePath = sourceObj as string;
DateTimeOffset?creationTime = null;
DateTimeOffset?lastWriteTime = null;
FileAttributes?fileAttributes = null;
FileOperations.GetFileProperties(sourcePath, out creationTime, out lastWriteTime, out fileAttributes);
string sourceFileSDDL = FileSecurityOperations.GetFilePortableSDDL(sourcePath,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
CloudBlob destBlob = destination as CloudBlob;
// Blob's original meta data has already been gotten from azure storage by DataMovement Library,
// Here only need to add new meta data key-value pairs, DataMovement Library will set these value to destination blob later.
destBlob.Metadata.Add(CreationTimeName, creationTime.Value.ToString());
destBlob.Metadata.Add(LastWriteTimeName, lastWriteTime.Value.ToString());
destBlob.Metadata.Add(FileAttributesName, fileAttributes.Value.ToString());
destBlob.Metadata.Add(FilePermissionsName, sourceFileSDDL);
};
context.ShouldTransferCallbackAsync = async(source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
TransferStatus trasferStatus =
await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString());
//Next the sample will show how to download a directory and restore file attributes to local file.
string destDirPath = ".";
CloudBlobDirectory sourceDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// In the following case, data movement library will download file named "azure.png" in the source directory,
// not including the sub-directory.
DownloadDirectoryOptions downloadDirectoryOptions = new DownloadDirectoryOptions()
{
SearchPattern = "azure.png",
Recursive = false
};
DirectoryTransferContext directoryTransferContext = new DirectoryTransferContext();
// Register for transfer event.
directoryTransferContext.FileFailed += FileFailedCallback;
directoryTransferContext.FileSkipped += FileSkippedCallback;
//Get stored file properties from source blob meta data and set to local file.
directoryTransferContext.FileTransferred += (object sender, TransferEventArgs e) =>
{
CloudBlob sourceBlob = e.Source as CloudBlob;
string destFilePath = e.Destination as string;
string metadataValue = null;
DateTimeOffset creationTime = default(DateTimeOffset);
DateTimeOffset lastWriteTime = default(DateTimeOffset);
FileAttributes fileAttributes = default(FileAttributes);
bool gotCreationTime = false;
bool gotLastWriteTime = false;
bool gotFileAttributes = false;
if (sourceBlob.Metadata.TryGetValue(CreationTimeName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
gotCreationTime = DateTimeOffset.TryParse(metadataValue, out creationTime);
}
if (sourceBlob.Metadata.TryGetValue(LastWriteTimeName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
gotLastWriteTime = DateTimeOffset.TryParse(metadataValue, out lastWriteTime);
}
if (sourceBlob.Metadata.TryGetValue(FileAttributesName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
gotFileAttributes = Enum.TryParse <FileAttributes>(metadataValue, out fileAttributes);
}
if (gotCreationTime && gotLastWriteTime && gotFileAttributes)
{
FileOperations.SetFileProperties(destFilePath, creationTime, lastWriteTime, fileAttributes);
}
if (sourceBlob.Metadata.TryGetValue(FilePermissionsName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
FileSecurityOperations.SetFileSecurity(destFilePath, metadataValue,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
};
// Always writes to destination no matter it exists or not.
directoryTransferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite;
trasferStatus =
await TransferManager.DownloadDirectoryAsync(sourceDir, destDirPath, downloadDirectoryOptions, directoryTransferContext);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} downloading to {1} is finished.", sourceDir.Uri.ToString(), destDirPath);
}
finally
{
//Restore privileges after getting/setting permissions from/to local file system.
FileSecurityOperations.RestorePrivileges(PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL, true);
}
}
