public async Task TransferLocalDirectoryToAzureBlobDirectory(string LocalSourceDirPath, string ContainerName)
{
CloudBlobDirectory blobDirectory = GetBlobDirectory(ContainerName);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(LocalSourceDirPath, blobDirectory, options, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
}
stopWatch.Stop();
}
public async Task Upload()
{
CloudBlobDirectory destDir = await GetCloudBlobDirectoryAsync(TargetFolder);
UploadDirectoryOptions options = new UploadDirectoryOptions {
SearchPattern = "*",
Recursive = true,
BlobType = BlobType.BlockBlob,
};
using (MemoryStream journalStream = new MemoryStream()) {
// Store the transfer context in a streamed journal.
DirectoryTransferContext context = new DirectoryTransferContext(journalStream);
// Register for transfer event.
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallback = (destination) => {
CloudBlob destBlob = destination as CloudBlob;
switch (Path.GetExtension(destBlob.Name).ToLowerInvariant())
{
case "html":
case ".html":
destBlob.Properties.ContentType = "text/html";
break;
default:
destBlob.Properties.ContentType = "text/plain";
break;
}
Console.WriteLine($"Setting attributes for {destination}");
};
context.ShouldTransferCallback = (source, destination) => {
// Can add more logic here to evaluate whether really need to transfer the target.
Console.WriteLine($"Should transfer from {source} to {destination}? YES");
return(true);
};
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferStatus transferStatus = null;
StartCancelThread(cancellationSource);
try {
// Start the upload
Task <TransferStatus> task = TransferManager.UploadDirectoryAsync(InputDirectory, destDir, options, context, cancellationSource.Token);
transferStatus = await task;
} catch (Exception e) {
Console.WriteLine("The transfer failed: {0}", e.Message);
}
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(transferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", InputDirectory, destDir.Uri);
}
}
public static object GetDefaultTransferDirectoryOptions(DMLibDataType sourceType, DMLibDataType destType)
{
if (DMLibTestBase.IsLocal(sourceType))
{
var result = new UploadDirectoryOptions();
if (IsCloudBlob(destType))
{
result.BlobType = MapBlobDataTypeToXSCLBlobType(destType);
}
return(result);
}
else if (DMLibTestBase.IsLocal(destType))
{
return(new DownloadDirectoryOptions());
}
else
{
var result = new CopyDirectoryOptions();
if (IsCloudBlob(destType))
{
result.BlobType = MapBlobDataTypeToXSCLBlobType(destType);
}
return(result);
}
}
public async Task TransferLocalDirectoryToAzureBlobDirectory(string localDirectoryPath)
{
NLogManager.LogInfo("Updating folder");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_connectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(_containerName);
container.CreateIfNotExistsAsync().Wait();
CloudBlobDirectory blobDirectory = container.GetDirectoryReference("");
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, context, cancellationSource.Token);
await task;
}
catch (Exception e)
{
NLogManager.LogInfo("Updating folder Error: " + e);
}
// stopWatch.Stop();
}
public void FollowSymlink_Set_True()
{
#if DNXCORE50
var uploadOption = new UploadDirectoryOptions();
bool gotException = false;
try
{
uploadOption.FollowSymlink = true;
}
catch (PlatformNotSupportedException)
{
gotException = true;
}
if (!CrossPlatformHelpers.IsLinux)
{
Test.Assert(gotException, "Should throw not supported exception on non-Linux platform.");
}
else
{
Test.Assert(uploadOption.FollowSymlink == true, "Follow symlink is set correctly");
}
#endif
}
/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory
/// 2. Set their content type to "image/png".
/// </summary>
private static async Task BlobDirectoryUploadSample()
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = Util.GetCloudBlobDirectory(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob,
ContentType = "image/png",
};
// Register for transfer event. It's optional and also applicable to single object transfer after v0.2.0.
TransferContext context = new TransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
// Start the upload
await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context);
Console.WriteLine("Files in directory {0} is uploaded to {1} successfully.", sourceDirPath, destDir.Uri.ToString());
}
static void Main(string[] args)
{
string storageConnectionString = "xxxx";
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudFileClient fileClient = account.CreateCloudFileClient();
CloudFileShare fileShare = fileClient.GetShareReference("t22");
fileShare.CreateIfNotExists();
CloudFileDirectory fileDirectory = fileShare.GetRootDirectoryReference();
//here, I want to upload all the files and subfolders in the follow path.
string source_path = @"F:\temp\1";
//if I want to upload the folder 1, then use the following code to create a file directory in azure.
CloudFileDirectory fileDirectory_2 = fileDirectory.GetDirectoryReference("1");
fileDirectory_2.CreateIfNotExists();
UploadDirectoryOptions directoryOptions = new UploadDirectoryOptions
{
Recursive = true
};
var task = TransferManager.UploadDirectoryAsync(source_path, fileDirectory_2, directoryOptions, null);
task.Wait();
Console.WriteLine("the upload is completed");
Console.ReadLine();
}
public async Task doCopy(DirectoryInfo source, string storageConnectionString)
{
try
{
// Connect to Azure Storage to create a container for this backup
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = account.CreateCloudBlobClient();
// Unique container name with timestamp
string container = source.Name + DateTime.Now.ToString("MMddyyyy-HHmmss");
CloudBlobContainer blobContainer = blobClient.GetContainerReference(container);
await blobContainer.CreateIfNotExistsAsync();
Console.WriteLine("Container {0} has been created.", container);
// Get root directory reference for the container
CloudBlobDirectory destBlob = blobContainer.GetDirectoryReference("");
// Parallel Operations
TransferManager.Configurations.ParallelOperations = 32;
// Setup the transfer context and track the upload progress
TransferContext context = new TransferContext();
context.FileFailed += Program.FileFailedCallback;
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.WriteLine("{0} MB uploaded", progress.BytesTransferred / (1024 * 1024));
});
// Upload recursively
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
// Start the counter
Stopwatch s = Stopwatch.StartNew();
// Initiate the Upload from DMLib
TransferStatus transferStatus = await TransferManager.UploadDirectoryAsync(source.FullName, destBlob, options, context);
s.Stop();
if (transferStatus.NumberOfFilesFailed > 0)
{
Console.WriteLine("{0} files failed to transfer", transferStatus.NumberOfFilesFailed);
}
// Log the result
Console.WriteLine("Upload has been completed in {0} seconds.", s.Elapsed.TotalSeconds);
}
catch (StorageException ex)
{
Console.WriteLine(ex.Message);
}
}
public static async Task TransferLocalDirectoryToAzureBlobDirectory(CloudStorageAccount account)
{
string localDirectoryPath = GetDirSourcePath();
CloudBlobDirectory blobDirectory = GetBlobDirectory(account);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Console.WriteLine("\nTransfer started...\nPress 'c' to temporarily cancel your transfer...\n");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
Console.WriteLine("\nThe transfer is canceled: {0}", e.Message);
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nTransfer will resume in 3 seconds...");
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetDirectoryTransferContext(checkpoint);
Console.WriteLine("\nResuming transfer...\n");
await TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, context);
}
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
//ExecuteChoice(account);
}
private async Task TransferLocalDirectoryToAzureBlobDirectory(string directory)
{
CloudBlobDirectory blobDirectory = await GetBlobDirectory();
if (blobDirectory == null)
{
return;
}
DirectoryTransferContext context = GetDirectoryTransferContext();
CancellationTokenSource cancellationSource = new CancellationTokenSource();
logger.Log($"Transfer started...");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(directory, blobDirectory, options, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
logger.Error(string.Format("The transfer is canceled: {0}", e.Message), null);
}
stopWatch.Stop();
Console.WriteLine();
logger.Log(string.Format("Transfer operation completed in {0} seconds.", stopWatch.Elapsed.TotalSeconds));
}
/// <summary>
/// Uploads the folder.
/// </summary>
/// <param name="localFolder">The local folder.</param>
/// <param name="containerName">Name of the container.</param>
/// <param name="remoteFolder">The remote folder.</param>
/// <param name="createContainerIfNotExists">if set to <c>true</c> [create container if not exists].</param>
/// <returns>The upload status.</returns>
public async Task <UploadStatus> UploadFolder(
string localFolder,
string containerName,
string remoteFolder,
bool createContainerIfNotExists = false)
{
if (string.IsNullOrWhiteSpace(localFolder))
{
throw new ArgumentNullException(nameof(localFolder));
}
if (string.IsNullOrWhiteSpace(containerName))
{
throw new ArgumentNullException(nameof(containerName));
}
if (string.IsNullOrWhiteSpace(remoteFolder))
{
throw new ArgumentNullException(nameof(remoteFolder));
}
var blobClient = this.storageAccount.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference(containerName);
if (createContainerIfNotExists)
{
await blobContainer.CreateIfNotExistsAsync();
}
var options = new UploadDirectoryOptions
{
Recursive = true,
};
var blobDirectory = blobContainer.GetDirectoryReference(remoteFolder);
var transferStatus = await TransferManager.UploadDirectoryAsync(
localFolder,
blobDirectory,
options,
new DirectoryTransferContext(checkpoint : null));
return(new UploadStatus(
bytesTransferred: transferStatus.BytesTransferred,
numberOfFilesTransferred: transferStatus.NumberOfFilesTransferred,
numberOfFilesFailed: transferStatus.NumberOfFilesFailed,
numberOfFilesSkipped: transferStatus.NumberOfFilesSkipped));
}
/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory
/// 2. Set their content type to "image/png".
/// </summary>
private static async Task BlobDirectoryUploadSample()
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob
};
// Register for transfer event.
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallback = (destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
context.ShouldTransferCallback = (source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
// Start the upload
var trasferStatus = await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString());
}
private Task <TransferStatus> UploadDirectory(dynamic destObject, TransferItem item)
{
UploadDirectoryOptions uploadDirectoryOptions = item.Options as UploadDirectoryOptions;
TransferContext transferContrext = item.TransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string sourcePath = item.SourceObject as string;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext, cancellationToken));
}
else if (transferContrext != null || uploadDirectoryOptions != null)
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject, uploadDirectoryOptions, transferContrext));
}
else
{
return(TransferManager.UploadDirectoryAsync(sourcePath, destObject));
}
}
private static async Task TransferLocalDirectoryToAzureBlobDirectory(CloudStorageAccount account)
{
// From・Toの情報を取得
var localDirectoryPath = GetSourcePath();
var blobDirectory = GetBlobDirectory(account);
// 再帰的にアップロードを実施する
var options = new UploadDirectoryOptions()
{
Recursive = true
};
// 転送処理の実施
Console.WriteLine("\nTransfer started...\n");
var stopWatch = Stopwatch.StartNew();
await TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, GetDirectoryTransferContext());
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
}
/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory
/// and store transfer context in a streamed journal.
/// 2. Set their content type to "image/png".
/// 3. Cancel the transfer before it finishes with a CancellationToken
/// 3. Reload the transfer context from the streamed journal.
/// 4. Resume the transfer with the loaded transfer context
/// </summary>
private static async Task BlobDirectoryUploadSample()
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob
};
using (MemoryStream journalStream = new MemoryStream())
{
// Store the transfer context in a streamed journal.
DirectoryTransferContext context = new DirectoryTransferContext(journalStream);
// Register for transfer event.
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallbackAsync = async(destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
context.ShouldTransferCallbackAsync = async(source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferStatus trasferStatus = null;
try
{
// Start the upload
Task <TransferStatus> task = TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
trasferStatus = await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
journalStream.Position = 0;
// Deserialize transfer context from the streamed journal.
DirectoryTransferContext resumeContext = new DirectoryTransferContext(journalStream);
resumeContext.FileTransferred += FileTransferredCallback;
resumeContext.FileFailed += FileFailedCallback;
resumeContext.FileSkipped += FileSkippedCallback;
resumeContext.SetAttributesCallbackAsync = async(destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
resumeContext.ShouldTransferCallbackAsync = async(source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
// Resume the upload
trasferStatus = await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, resumeContext);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString());
}
}
public async Task TransferLocalDirectoryToAzureStorage(Func <string, UploadDirectoryOptions, DirectoryTransferContext, CancellationToken, Task <TransferStatus> > UploadDirectoryAsync,
Func <TransferEventArgs, string> getSourceToDestinationInfo,
CancellationToken cancellationToken)
{
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = _repo.Recursive,
BlobType = BlobType.BlockBlob
//SearchPattern =
};
var internalTokenSource = new CancellationTokenSource();
using var linkedCts =
CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, internalTokenSource.Token);
using var watcher = new UploadFileSystemWatcher(_repo.UploadFolder);
try
{
var transferCheckpoint = AzCpCheckpoint.Read(_repo.TransferCheckpointFilename);
if (transferCheckpoint != null)
{
_logger.Information("Resuming upload...");
}
while (true)
{
Stopwatch stopWatch = new Stopwatch();// Stopwatch.StartNew();
var context = new MyDirectoryTransferContext(_logger, _feedback, _repo.UploadFolder, _repo.ArchiveFolder, _repo.TransferCheckpointFilename, stopWatch, transferCheckpoint, getSourceToDestinationInfo);
transferCheckpoint = null;
_feedback.WriteProgress("Establishing connection...");
watcher.ResetChangeCount();
stopWatch.Start();
// var transferStatus = await TransferManager.UploadDirectoryAsync(_repo.UploadFolder, blobDirectory, options, context, linkedCts.Token);
var transferStatus = await UploadDirectoryAsync(_repo.UploadFolder, options, context, linkedCts.Token);
stopWatch.Stop();
linkedCts.Token.ThrowIfCancellationRequested();
try
{
if (File.Exists(_repo.TransferCheckpointFilename))
{
File.Delete(_repo.TransferCheckpointFilename);
}
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (IOException)
{
// ignore any issues deleting the checkpoint file
// e.g. doesn't exist
}
#pragma warning restore CA1031 // Do not catch general exception types
_logger.Information($"{stopWatch.Elapsed}: {transferStatus.ToUserString(context.InitialBytesTransferred, stopWatch)}");
// wait until there are new files to upload
// NOTE: Will also be triggered if files are renamed or deleted
while (!watcher.AnyChangesInUploadFolder)
{
_feedback.WriteProgress("Waiting for new files to upload...");
if (linkedCts.Token.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(1500 - DateTime.UtcNow.Millisecond)))
{
linkedCts.Token.ThrowIfCancellationRequested();
}
}
}
}
catch (Exception ex) when(ex is TaskCanceledException || ex is OperationCanceledException)
{
_logger.Information($"The transfer was cancelled");
throw;
}
catch (Exception)
{
//_logger.Error(ex, $"UNEXPECTED ERROR: {ex.Message}");
throw;
}
}
/// <summary>
/// Upload local pictures to azure storage.
/// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory.
/// 2. Store source file's file attributes and permissions into destination blob's meta data.
/// 3. Download png files starting with "azure" in the source directory to a local directory, not including the sub-directory.
/// 4. Restore file attributes and permissions to destination local file.
/// </summary>
private static async Task BlobDirectoryPreserveFilePropertiesSampleAsync()
{
//Enable required privileges before getting/setting permissions from/to local file system.
FileSecurityOperations.EnableRequiredPrivileges(PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL, true);
try
{
string sourceDirPath = ".";
CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of
// SearchPattern and Recuresive varies for different source directory types.
// See https://azure.github.io/azure-storage-net-data-movement for more details.
//
// When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If
// recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the
// sub-directory will be matched as well.
//
// In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs,
// not including the sub-directory.
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
SearchPattern = "azure*.png",
Recursive = false,
BlobType = BlobType.BlockBlob
};
DirectoryTransferContext context = new DirectoryTransferContext();
// Register for transfer event.
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
context.SetAttributesCallbackAsync = async(sourceObj, destination) =>
{
string sourcePath = sourceObj as string;
DateTimeOffset?creationTime = null;
DateTimeOffset?lastWriteTime = null;
FileAttributes?fileAttributes = null;
FileOperations.GetFileProperties(sourcePath, out creationTime, out lastWriteTime, out fileAttributes);
string sourceFileSDDL = FileSecurityOperations.GetFilePortableSDDL(sourcePath,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
CloudBlob destBlob = destination as CloudBlob;
// Blob's original meta data has already been gotten from azure storage by DataMovement Library,
// Here only need to add new meta data key-value pairs, DataMovement Library will set these value to destination blob later.
destBlob.Metadata.Add(CreationTimeName, creationTime.Value.ToString());
destBlob.Metadata.Add(LastWriteTimeName, lastWriteTime.Value.ToString());
destBlob.Metadata.Add(FileAttributesName, fileAttributes.Value.ToString());
destBlob.Metadata.Add(FilePermissionsName, sourceFileSDDL);
};
context.ShouldTransferCallbackAsync = async(source, destination) =>
{
// Can add more logic here to evaluate whether really need to transfer the target.
return(true);
};
TransferStatus trasferStatus =
await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString());
//Next the sample will show how to download a directory and restore file attributes to local file.
string destDirPath = ".";
CloudBlobDirectory sourceDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
// In the following case, data movement library will download file named "azure.png" in the source directory,
// not including the sub-directory.
DownloadDirectoryOptions downloadDirectoryOptions = new DownloadDirectoryOptions()
{
SearchPattern = "azure.png",
Recursive = false
};
DirectoryTransferContext directoryTransferContext = new DirectoryTransferContext();
// Register for transfer event.
directoryTransferContext.FileFailed += FileFailedCallback;
directoryTransferContext.FileSkipped += FileSkippedCallback;
//Get stored file properties from source blob meta data and set to local file.
directoryTransferContext.FileTransferred += (object sender, TransferEventArgs e) =>
{
CloudBlob sourceBlob = e.Source as CloudBlob;
string destFilePath = e.Destination as string;
string metadataValue = null;
DateTimeOffset creationTime = default(DateTimeOffset);
DateTimeOffset lastWriteTime = default(DateTimeOffset);
FileAttributes fileAttributes = default(FileAttributes);
bool gotCreationTime = false;
bool gotLastWriteTime = false;
bool gotFileAttributes = false;
if (sourceBlob.Metadata.TryGetValue(CreationTimeName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
gotCreationTime = DateTimeOffset.TryParse(metadataValue, out creationTime);
}
if (sourceBlob.Metadata.TryGetValue(LastWriteTimeName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
gotLastWriteTime = DateTimeOffset.TryParse(metadataValue, out lastWriteTime);
}
if (sourceBlob.Metadata.TryGetValue(FileAttributesName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
gotFileAttributes = Enum.TryParse <FileAttributes>(metadataValue, out fileAttributes);
}
if (gotCreationTime && gotLastWriteTime && gotFileAttributes)
{
FileOperations.SetFileProperties(destFilePath, creationTime, lastWriteTime, fileAttributes);
}
if (sourceBlob.Metadata.TryGetValue(FilePermissionsName, out metadataValue) &&
!string.IsNullOrEmpty(metadataValue))
{
FileSecurityOperations.SetFileSecurity(destFilePath, metadataValue,
PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL);
}
};
// Always writes to destination no matter it exists or not.
directoryTransferContext.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite;
trasferStatus =
await TransferManager.DownloadDirectoryAsync(sourceDir, destDirPath, downloadDirectoryOptions, directoryTransferContext);
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
Console.WriteLine("Files in directory {0} downloading to {1} is finished.", sourceDir.Uri.ToString(), destDirPath);
}
finally
{
//Restore privileges after getting/setting permissions from/to local file system.
FileSecurityOperations.RestorePrivileges(PreserveSMBPermissions.Owner | PreserveSMBPermissions.Group | PreserveSMBPermissions.DACL | PreserveSMBPermissions.SACL, true);
}
}
private async Task <bool> InternalUploadSessionAsync(Session session, IProgress <TransferStatus> progressHandler, CancellationToken cancellationToken)
{
var srcDirectoryPath = session.LocalRecordingFolderPath;
var dstBlob = await GetDirectoryBlobAsync(session.Id).ConfigureAwait(false);
var journalFile = this.GetJournalFileForSession(srcDirectoryPath);
using var journalStream = await this.GetJournalStream(session, journalFile).ConfigureAwait(false);
using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, this.disposeCancellation.Token);
using var reg = linkedCancellation.Token.Register(() =>
{
this.logger.LogDebug($"The transfer has been cancelled and the journal file for session {session.Id} will be saved");
journalStream.Position = 0;
using var journal = File.Create(journalFile);
journalStream.CopyTo(journal);
});
try
{
var options = new UploadDirectoryOptions()
{
SearchPattern = FileUploadSearchPattern,
Recursive = false,
BlobType = BlobType.BlockBlob
};
var context = new DirectoryTransferContext(journalStream)
{
ProgressHandler = progressHandler,
LogLevel = Microsoft.Azure.Storage.LogLevel.Informational
};
var transferTcs = new TaskCompletionSource <bool>();
context.FileFailed += (sender, e) =>
{
this.logger.LogInformation(e.Exception, $"Transfer fails. {e.Source} -> {e.Destination}.");
transferTcs.TrySetException(e.Exception);
if (!linkedCancellation.IsCancellationRequested)
{
linkedCancellation.Cancel();
}
};
context.FileSkipped += (sender, e) =>
{
this.logger.LogInformation($"Transfer skips. {e.Source} -> {e.Destination}.");
};
context.FileTransferred += (sender, e) =>
{
this.logger.LogInformation(e.Exception, $"Transfer succeds. {e.Source} -> {e.Destination}.");
};
// Start the upload
var trasferStatus = await TransferManager.UploadDirectoryAsync(srcDirectoryPath, dstBlob, options, context, linkedCancellation.Token).ConfigureAwait(false);
transferTcs.TrySetResult(true);
await transferTcs.Task.ConfigureAwait(false);
}
catch (OperationCanceledException e)
{
this.logger.LogInformation(e, $"The transfer {session.Id} has been cancelled");
}
catch (InvalidOperationException e) when(e.Message.Contains("Only one transfer is allowed with stream journal."))
{
this.logger.LogWarning(e, $"The transfer {session.Id} has failed");
File.Delete(journalFile);
throw;
}
catch (Exception e)
{
linkedCancellation.Cancel();
this.logger.LogWarning(e, $"The transfer {session.Id} has failed");
throw;
}
if (!linkedCancellation.IsCancellationRequested)
{
File.Delete(journalFile);
return(true);
}
return(false);
}
public void TestNotsupportedPlatform()
{
Exception exception = null;
try
{
DownloadDirectoryOptions downloadDirectoryOptions = new DownloadDirectoryOptions()
{
PreserveSMBAttributes = true,
};
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
try
{
UploadDirectoryOptions uploadDirectoryOptions = new UploadDirectoryOptions()
{
PreserveSMBAttributes = true,
};
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
try
{
DownloadOptions downloadOptions = new DownloadOptions()
{
PreserveSMBAttributes = true,
};
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
try
{
UploadOptions uploadOptions = new UploadOptions()
{
PreserveSMBAttributes = true,
};
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
}
public void TestNotsupportedPlatform()
{
Exception exception = null;
DownloadDirectoryOptions downloadDirectoryOptions = new DownloadDirectoryOptions();
try
{
downloadDirectoryOptions.PreserveSMBAttributes = true;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
try
{
downloadDirectoryOptions.PreserveSMBPermissions = PreserveSMBPermissions.Owner;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
UploadDirectoryOptions uploadDirectoryOptions = new UploadDirectoryOptions();
try
{
uploadDirectoryOptions.PreserveSMBAttributes = true;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
try
{
uploadDirectoryOptions.PreserveSMBPermissions = PreserveSMBPermissions.Owner;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
DownloadOptions downloadOptions = new DownloadOptions();
try
{
downloadOptions.PreserveSMBAttributes = true;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
try
{
downloadOptions.PreserveSMBPermissions = PreserveSMBPermissions.Owner;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
exception = null;
UploadOptions uploadOptions = new UploadOptions();
try
{
uploadOptions.PreserveSMBAttributes = true;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
try
{
uploadOptions.PreserveSMBPermissions = PreserveSMBPermissions.Owner;
}
catch (Exception ex)
{
exception = ex;
}
ValidateExceptionForParemterSetting(exception);
}
