public async Task DownloadAzureBlobToLocalFile(string DestinationFullFilePath, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
try
{
task = TransferManager.DownloadAsync(blob, DestinationFullFilePath, null, context, cancellationSource.Token); //(LocalSourceFilePath, blob, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
/// <summary>
/// Upload a local picture to azure storage.
/// 1. Upload a local picture as a block blob.
/// 2. Set its content type to "image/png".
/// </summary>
private static async Task BlobUploadSample()
{
// When transfer large file to block blob, set TransferManager.Configurations.BlockSize to specify the size of the blocks.
// It must be between 4MB and 100MB and be multiple of 4MB. Default value is 4MB.
//
// Currently, the max block count of a block blob is limited to 50000.
// When transfering a big file and the BlockSize provided is smaller than the minimum value - (size/50000),
// it'll be reset to a value which is greater than the minimum value and multiple of 4MB for this file.
TransferManager.Configurations.BlockSize = 4 * 1024 * 1024; //4MB
string sourceFileName = "azure.png";
string destinationBlobName = "azure_blockblob.png";
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Use UploadOptions to set ContentType of destination CloudBlob
UploadOptions options = new UploadOptions();
SingleTransferContext context = new SingleTransferContext();
context.SetAttributesCallbackAsync = async(destination) =>
{
CloudBlob destBlob = destination as CloudBlob;
destBlob.Properties.ContentType = "image/png";
};
// Start the upload
await TransferManager.UploadAsync(sourceFileName, destinationBlob, options, context);
Console.WriteLine("File {0} is uploaded to {1} successfully.", sourceFileName, destinationBlob.Uri.ToString());
}
public static async Task Upload(FileInfo archive, CommandLineOptions options, ILogger log, CancellationToken cancellationToken)
{
log.Information($"Starting the upload to {options.StorageUrl}");
// following settings are recommended best practices https://github.com/Azure/azure-storage-net-data-movement#best-practice
ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;
ServicePointManager.Expect100Continue = false;
var storageConnectionString = GetConnectionString(options);
var account = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = account.CreateCloudBlobClient();
var containerName = options.ProjectName; // we use project name (CoreFX/CoreCLR etc) as a container name
var projectBlobContainer = blobClient.GetContainerReference(containerName);
TransferManager.Configurations.ParallelOperations = 64; // value taken from https://github.com/Azure/azure-storage-net-data-movement
var context = new SingleTransferContext
{
ProgressHandler = new Progress <TransferStatus>(progress => log.Information("Bytes uploaded: {0}", progress.BytesTransferred)),
};
var destinationBlob = projectBlobContainer.GetBlockBlobReference(GetDestinationBlobName(options, archive));
await TransferManager.UploadAsync(archive.FullName, destinationBlob, null, context, cancellationToken);
}
public async Task TransferLocalFileToAzureBlob(string connection, string region, string containerName, string sourcePath, string targetPath)
{
try
{
CloudStorageAccount account = CloudStorageAccount.Parse(connection);
CloudBlobClient blobClient = account.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(targetPath);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
Console.WriteLine("Transfer started..." + region + "");
Stopwatch stopWatch = Stopwatch.StartNew();
if (await blob.ExistsAsync())
{
//Log.WriteLog(region + " blob is exist.");
await blob.DeleteIfExistsAsync();
}
await TransferManager.UploadAsync(sourcePath, blob);
stopWatch.Stop();
Log.WriteLog(region + " transfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
}
catch (Exception e)
{
Log.WriteLog(e.Message, Log.Type.Exception);
throw e;
}
}
public async Task TransferAzureBlobToAzureBlob(string ContainerSourceName, string BlobSourceName, string ContainerDestinationName, string BlobDestinationName)
{
CloudBlockBlob sourceBlob = GetBlob(ContainerSourceName, BlobSourceName);
CloudBlockBlob destinationBlob = GetBlob(ContainerDestinationName, BlobDestinationName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(100000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
try
{
task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
protected SingleTransferContext GetTransferContext(ProgressRecord record, long totalTransferLength)
{
SingleTransferContext transferContext = new SingleTransferContext();
transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId;
if (this.Force)
{
transferContext.ShouldOverwriteCallback = TransferContext.ForceOverwrite;
}
else
{
transferContext.ShouldOverwriteCallback = ConfirmOverwrite;
}
transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) =>
{
if (record != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
record.PercentComplete = (totalTransferLength == 0) ? 100 : (int)(transferProgress.BytesTransferred * 100 / totalTransferLength);
record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, record.PercentComplete);
this.OutputStream.WriteProgress(record);
}
});
return(transferContext);
}
//Copies a blob between two azure containers.
public async Task CopyAzureBlobToAzureBlob(CloudStorageAccount account, CloudBlockBlob sourceBlob, CloudBlockBlob destinationBlob)
{
TransferCheckpoint Checkpoint = null;
SingleTransferContext Context = GetSingleTransferContext(Checkpoint);
CancellationTokenSource CancellationSource = new CancellationTokenSource();
Stopwatch StopWatch = Stopwatch.StartNew();
Task Task;
try
{
Task = TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null, Context, CancellationSource.Token);
await Task;
}
catch (AggregateException e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
catch (TransferException e)
{
_Log.LogInformation($"The Azure Blob {sourceBlob.Name} already exists in {destinationBlob.Parent.Container.Name} with message {e.Message}");
}
catch (Exception e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
StopWatch.Stop();
_Log.LogInformation($"The Azure Blob {sourceBlob.Name} transfer to {destinationBlob.Name} completed in: {StopWatch.Elapsed.TotalSeconds} seconds.");
}
public async Task <Uri> CopyBlobAsync(string fileName, string sourceContainerName, string targetContainerName)
{
var sourceContainer = await GetOrCreateContainerAsync(sourceContainerName);
var targetContainer = await GetOrCreateContainerAsync(targetContainerName);
var sourceBlob = sourceContainer.GetBlockBlobReference(fileName);
var targetBlob = targetContainer.GetBlockBlobReference(fileName);
var isSourceBlobExists = await sourceBlob.ExistsAsync();
if (!isSourceBlobExists)
{
throw new BlobNotFoundException($"Source blob doesn't found. FileName={fileName}, Container={sourceContainerName}");
}
var context = new SingleTransferContext
{
ShouldOverwriteCallbackAsync = (source, destination) => Task.FromResult(true)
};
var cancellationSource = new CancellationTokenSource();
await TransferManager.CopyAsync(sourceBlob, targetBlob, true, null, context, cancellationSource.Token);
return(targetBlob.Uri);
}
private void RunDataMovement(Uri sas, FileInfo file)
{
var container = new CloudBlobContainer(sas);
try
{
var blob = container.GetBlockBlobReference(file.Name);
var size = file.Length;
TransferManager.Configurations.ParallelOperations = 64;
var context = new SingleTransferContext
{
ProgressHandler = new Progress <TransferStatus>(p =>
{
var percentageComplete = (double)p.BytesTransferred / size;
var pc = Convert.ToInt32(percentageComplete * 100);
_worker.ReportProgress(pc, p.BytesTransferred);
})
};
var task = TransferManager.UploadAsync(file.FullName, blob, null, context, CancellationToken.None);
task.Wait();
}
catch (AggregateException e)
{
if (e.InnerExceptions.Any())
{
MessageBox.Show($@"{string.Join(",", e.InnerExceptions.Select(x => x.Message))}");
}
}
}
public async Task TransferUrlToAzureBlob(Uri BlobUrl, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Task task;
try
{
task = TransferManager.CopyAsync(BlobUrl, blob, true, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
if (_stopFlag)
{
return;
}
//autoretry
}
}
protected SingleTransferContext GetTransferContext(DataMovementUserData userData)
{
SingleTransferContext transferContext = new SingleTransferContext();
transferContext.ClientRequestId = CmdletOperationContext.ClientRequestId;
if (overwrite)
{
transferContext.ShouldOverwriteCallback = TransferContext.ForceOverwrite;
}
else
{
transferContext.ShouldOverwriteCallback = ConfirmOverwrite;
}
transferContext.ProgressHandler = new TransferProgressHandler((transferProgress) =>
{
if (userData.Record != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
userData.Record.PercentComplete = 0 == userData.TotalSize ? 100 : (int)(transferProgress.BytesTransferred * 100 / userData.TotalSize);
userData.Record.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, userData.Record.PercentComplete);
this.OutputStream.WriteProgress(userData.Record);
}
});
return(transferContext);
}
protected async Task <bool> GetSignatureBlobForTransfer()
{
DownloadOptions options = new DownloadOptions();
SingleTransferContext context = new SingleTransferContext();
context.LogLevel = LogLevel.Informational;
context.SetAttributesCallback = (destination) =>
{
SignatureBlob = destination as CloudBlockBlob;
};
context.FileTransferred += Context_BlobTransferred;
context.FileFailed += Context_BlobFailed;
try
{
ICloudBlob sig = await DestinationStorage.GetCloudBlobAsync(DestinationContainerName, SignatureBlobName);
L.Info("Azure Storage blob signature has size {size}.", sig.Properties.Length, sig.Name);
context.ProgressHandler = new SingleBlobTransferProgressReporter(sig);
if (sig.Properties.BlobType == BlobType.BlockBlob)
{
SignatureBlob = (CloudBlockBlob)sig;
}
using (Operation azOp = L.Begin("Download signature"))
using (MemoryStream ms = new MemoryStream())
{
DownloadSignatureTask = TransferManager.DownloadAsync(SignatureBlob, ms);
await DownloadSignatureTask;
if (DownloadSignatureTask.Status == TaskStatus.RanToCompletion)
{
IFormatter formatter = new BinaryFormatter();
Signature = (SingleFileSignature)formatter.Deserialize(ms);
ComputeSignatureTask = Task.CompletedTask;
L.Info("Using Azure Storage blob signature for synchroniztion built on {date}.", Signature.ComputedDateTime);
azOp.Complete();
return(true);
}
else
{
return(false);
}
}
}
catch (Exception e)
{
LogTransferException(e, "download signature");
if (e is AggregateException && e.InnerException != null && (e.InnerException is TaskCanceledException || e.InnerException is OperationCanceledException || e.InnerException is TransferSkippedException))
{
return(true);
}
else if (e is TaskCanceledException || e is OperationCanceledException || e is TransferSkippedException)
{
return(true);
}
else
{
return(false);
}
}
}
/// <summary>
/// Download data from Azure storage.
/// 1. Download a CloudBlob to a Stream instance
/// 2. Download the same CloudBlob with step #1 to a different Stream instance
/// 3. Download another CloudBlob to a different stream with content MD5 validation disabled
/// 4. Show the overall progress of all transfers
/// </summary>
private static async Task BlobDownloadToStreamSample()
{
string sourceBlobName1 = "azure_blockblob.png";
string sourceBlobName2 = "azure_blockblob2.png";
// Create the source CloudBlob instances
CloudBlob sourceBlob1 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName1, BlobType.BlockBlob);
CloudBlob sourceBlob2 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName2, BlobType.BlockBlob);
// Create a TransferContext shared by both transfers
SingleTransferContext sharedTransferContext = new SingleTransferContext();
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
sharedTransferContext.ProgressHandler = recorder;
MemoryStream memoryStream1_1 = new MemoryStream();
MemoryStream memoryStream1_2 = new MemoryStream();
MemoryStream memoryStream2 = new MemoryStream();
try
{
// Start the blob download
Task task1 = TransferManager.DownloadAsync(sourceBlob1, memoryStream1_1, null /* options */, sharedTransferContext);
// Start to download the same blob to another Stream
// Please note, DataMovement Library will download blob once for each of downloads.
// For example, if you start two downloads from the same source blob to two different Stream instance,
// DataMovement Library will download the blob content twice.
Task task2 = TransferManager.DownloadAsync(sourceBlob1, memoryStream1_2, null /* options */, sharedTransferContext);
// Create a DownloadOptions to disable md5 check after data is downloaded. Otherwise, data movement
// library will check the md5 checksum stored in the ContentMD5 property of the source CloudFile/CloudBlob
// You can uncomment following codes, enable ContentMD5Validation and have a try.
// sourceBlob2.Properties.ContentMD5 = "WrongMD5";
// sourceBlob2.SetProperties();
DownloadOptions options = new DownloadOptions();
options.DisableContentMD5Validation = true;
// Start the download
Task task3 = TransferManager.DownloadAsync(sourceBlob2, memoryStream2, options, sharedTransferContext);
// Wait for all transfers to finish
await task1;
await task2;
await task3;
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", recorder.ToString());
}
finally
{
memoryStream1_1.Dispose();
memoryStream1_2.Dispose();
memoryStream2.Dispose();
}
}
protected async Task <bool> UploadSignature()
{
UploadOptions options = new UploadOptions();
SingleTransferContext context = new SingleTransferContext();
bool transferred = false;
context.LogLevel = LogLevel.Informational;
context.ProgressHandler = new SingleFileTransferProgressReporter(SignatureFile);
context.SetAttributesCallback = (destination) =>
{
SignatureBlob = destination as CloudBlockBlob;
SignatureBlob.Properties.ContentType = "application/octet-stream";
};
context.ShouldOverwriteCallback = (source, destination) =>
{
return(true);
};
context.FileTransferred += (sender, e) =>
{
Context_FileTransferred(sender, e);
transferred = true;
};
context.FileFailed += (sender, e) =>
{
Context_FileFailed(sender, e);
transferred = false;
};
context.FileSkipped += (sender, e) =>
{
Context_FileSkipped(sender, e);
transferred = false;
};
try
{
SignatureBlob = await DestinationStorage.GetorCreateCloudBlobAsync(DestinationContainerName, DestinationBlob.Name + ".sig", BlobType.BlockBlob) as CloudBlockBlob;
if (await SignatureBlob.ExistsAsync())
{
L.Warn("The existing signature blob {blob} will be overwritten.", SignatureBlob.Name);
}
await TransferManager.UploadAsync(SignatureFile.FullName, SignatureBlob, options, context, CT);
transferred = true;
}
catch (Exception e)
{
LogTransferException(e, $"upload signature to cloud blob {SignatureBlob.Name}");
if (e is OperationCanceledException || e is TaskCanceledException || (e is TransferException && CT.IsCancellationRequested))
{
transferred = true;
}
else
{
transferred = false;
}
}
return(transferred);
}
protected override async Task <string> UploadFileCore(string id, string file)
{
var context = new SingleTransferContext();
var blob = _container.GetBlockBlobReference(id);
await TransferManager.UploadAsync(file, blob, null, context);
return(blob.Uri.AbsoluteUri);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "azure_blockblob.png";
string destinationBlobName = "azure_blockblob2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
SingleTransferContext context = new SingleTransferContext();
// Start the transfer
try
{
// With the CopyMethod parameter, you can indicate how the content would be copied to destination blob.
// SyncCopy is to download source blob content to local memory and then upload to destination blob.
// ServiceSideAsyncCopy is to send a start-copy request to Azure Storage Sever, and Azure Storage Server will do the actual copy.
// ServiceSideSyncCopy will leverage REST API of Put Block From URL, Append Block From URL and Put Page From URL in Azure Storage Server.
// Please see <c>https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url</c> for Put Block From URL,
// <c>https://docs.microsoft.com/en-us/rest/api/storageservices/append-block-from-url</c> for Append Block From URL,
// <c>https://docs.microsoft.com/en-us/rest/api/storageservices/put-page-from-url</c> for Put Page From URL.
// Following will use ServiceSideSyncCopy to copy a blob.
Task task = TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null /* options */, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will tranferred after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
SingleTransferContext resumeContext = new SingleTransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
private static SingleTransferContext GetSingleTransferContext()
{
SingleTransferContext context = new SingleTransferContext();
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.Write("\rBytes transferred: {0}", progress.BytesTransferred);
});
return(context);
}
private void TestSetAttributesToLocal(bool IsDirectoryTransfer)
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L,
};
sourceDataInfo.RootNode.AddFileNode(fileNode);
TransferContext context;
if (IsDirectoryTransfer)
{
context = new DirectoryTransferContext()
{
SetAttributesCallbackAsync = async(destObj) =>
{
Test.Error("SetAttributes callback should not be invoked when destination is local");
}
};
}
else
{
context = new SingleTransferContext()
{
SetAttributesCallbackAsync = async(destObj) =>
{
Test.Error("SetAttributes callback should not be invoked when destination is local");
}
};
}
var options = new TestExecutionOptions <DMLibDataInfo>();
options.TransferItemModifier = (node, transferItem) =>
{
dynamic transferOptions = IsDirectoryTransfer ? DefaultTransferDirectoryOptions : DefaultTransferOptions;
if (IsDirectoryTransfer)
{
transferOptions.Recursive = true;
}
transferItem.Options = transferOptions;
transferItem.TransferContext = context;
};
options.IsDirectoryTransfer = IsDirectoryTransfer;
var result = this.ExecuteTestCase(sourceDataInfo, options);
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
}
protected override async Task OnCopyAsync(
Uri sourceUri,
IStorage destinationStorage,
Uri destinationUri,
IReadOnlyDictionary <string, string> destinationProperties,
CancellationToken cancellationToken)
{
var azureDestinationStorage = destinationStorage as AzureStorage;
if (azureDestinationStorage == null)
{
throw new NotImplementedException("Copying is only supported from Azure storage to Azure storage.");
}
string sourceName = GetName(sourceUri);
string destinationName = azureDestinationStorage.GetName(destinationUri);
CloudBlockBlob sourceBlob = _directory.GetBlockBlobReference(sourceName);
CloudBlockBlob destinationBlob = azureDestinationStorage._directory.GetBlockBlobReference(destinationName);
var context = new SingleTransferContext();
if (destinationProperties?.Count > 0)
{
context.SetAttributesCallback = new SetAttributesCallback((destination) =>
{
var blob = (CloudBlockBlob)destination;
// The copy statement copied all properties from the source blob to the destination blob; however,
// there may be required properties on destination blob, all of which may have not already existed
// on the source blob at the time of copy.
foreach (var property in destinationProperties)
{
switch (property.Key)
{
case StorageConstants.CacheControl:
blob.Properties.CacheControl = property.Value;
break;
case StorageConstants.ContentType:
blob.Properties.ContentType = property.Value;
break;
default:
throw new NotImplementedException($"Storage property '{property.Value}' is not supported.");
}
}
});
}
context.ShouldOverwriteCallback = new ShouldOverwriteCallback((source, destination) => true);
await TransferManager.CopyAsync(sourceBlob, destinationBlob, _useServerSideCopy, options : null, context : context);
}
public static SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint, Stopwatch stopwatchTarea)
{
SingleTransferContext context = new SingleTransferContext(checkpoint);
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Log.Information("Mb descargados {0:F}\n", (progress.BytesTransferred / 1024) / 1024);
stopwatchTarea.Restart();
});
return(context);
}
public void TestSetAttributes()
{
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
FileNode fileNode = new FileNode(DMLibTestBase.FileName)
{
SizeInByte = DMLibTestBase.FileSizeInKB * 1024L
};
if (DMLibTestContext.SourceType != DMLibDataType.Local)
{
fileNode.Metadata = new Dictionary <string, string>();
fileNode.Metadata.Add("foo", "bar");
fileNode.ContentLanguage = SetAttributesTest.TestContentLanguage;
}
sourceDataInfo.RootNode.AddFileNode(fileNode);
TransferContext context = new SingleTransferContext()
{
SetAttributesCallbackAsync = async(destObj) =>
{
dynamic destCloudObj = destObj;
destCloudObj.Properties.ContentType = SetAttributesTest.TestContentType;
destCloudObj.Metadata.Add("aa", "bb");
}
};
var options = new TestExecutionOptions <DMLibDataInfo>();
options.TransferItemModifier = (node, transferItem) =>
{
dynamic transferOptions = DefaultTransferOptions;
transferItem.Options = transferOptions;
transferItem.TransferContext = context;
};
var result = this.ExecuteTestCase(sourceDataInfo, options);
fileNode.Metadata.Add("aa", "bb");
VerificationHelper.VerifyTransferSucceed(result, sourceDataInfo);
FileNode destFileNode = result.DataInfo.RootNode.GetFileNode(DMLibTestBase.FileName);
Test.Assert(TestContentType.Equals(destFileNode.ContentType), "Verify content type: {0}, expected {1}", destFileNode.ContentType, TestContentType);
if (DMLibTestContext.SourceType != DMLibDataType.Local)
{
Test.Assert(SetAttributesTest.TestContentLanguage.Equals(destFileNode.ContentLanguage), "Verify ContentLanguage: {0}, expected {1}", destFileNode.ContentLanguage, SetAttributesTest.TestContentLanguage);
}
}
private SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint, string blobName)
{
SingleTransferContext context = new SingleTransferContext(checkpoint);
context.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite;
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
_logger.LogDebug($"{blobName}: bytes transferred {progress.BytesTransferred}.");
});
return(context);
}
public SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint)
{
SingleTransferContext context = new SingleTransferContext(checkpoint);
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
if (BytesTransferred != null)
{
BytesTransferred(progress.BytesTransferred);
}
});
return(context);
}
public static async Task TransferLocalFileToAzureBlob(CloudStorageAccount account)
{
string localFilePath = GetSourcePath();
CloudBlockBlob blob = GetBlob(account);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Console.WriteLine("\nTransfer started...\nPress 'c' to temporarily cancel your transfer...\n");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
try
{
task = TransferManager.UploadAsync(localFilePath, blob, null, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
Console.WriteLine("\nThe transfer is canceled: {0}", e.Message);
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nTransfer will resume in 3 seconds...");
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetSingleTransferContext(checkpoint);
Console.WriteLine("\nResuming transfer...\n");
await TransferManager.UploadAsync(localFilePath, blob, null, context);
}
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
//ExecuteChoice(account);
}
SingleTransferContext PrepareTransfer()
{
TransferManager.Configurations.ParallelOperations = ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;
ServicePointManager.Expect100Continue = false;
var context = new SingleTransferContext
{
LogLevel = LogLevel.Warning,
ProgressHandler = new Progress <TransferStatus>((progress) =>
{
}),
ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite
};
return(context);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "application-user-photos/azure_blockblob.png";
string destinationBlobName = "application-user-photos/azure_blockblob-2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
SingleTransferContext context = new SingleTransferContext();
// Start the transfer
try
{
Task task = TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will tranferred after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
SingleTransferContext resumeContext = new SingleTransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
public async Task CopyAsync(
string blobName,
CancellationToken token = default(CancellationToken))
{
var sourceBlob = _sourceContainer.GetBlobReference(blobName);
var targetBlob = _targetContainer.GetBlobReference(blobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint, blobName);
await TransferManager.CopyAsync(
sourceBlob : sourceBlob,
destBlob : targetBlob,
copyMethod : CopyMethod.ServiceSideAsyncCopy,
options : null,
context : context,
cancellationToken : token).ConfigureAwait(false);
}
// The actual fun.. This is the function that uploads the data...
private async void ProcessDataAsync(file n)
{
// Get text from textBox regarding Blob container to add the files.
string container = containerinfo.Text;
// Start the Azure Blob client
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Not sure yet.....
CloudBlobContainer blobContainer = blobClient.GetContainerReference(container);
// Seems that the SDK create he blob/blob reference, which we somehow update with our actual data.. <= Have to check this out, what this do..
CloudBlockBlob destBlob = blobContainer.GetBlockBlobReference(n.name);
// Setup the number of the concurrent operations <= Not sure what this means.
TransferManager.Configurations.ParallelOperations = 64;
// Setup the transfer context and track the upoload progress <= Not sure what this means.
SingleTransferContext context = new SingleTransferContext();
// Some more .net black art. But using lets me cancel the upload. <= Not sure what this means.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
// Add the black art cancel stuff to the file item. Allows me to cancel the upload easly..
n.cancel = cancellationSource;
// Set the file item -status- to Downloading, gui is updated..
n.status = "Uploading";
// Event which is called when file transfer is complete.
//<= this has some fun stuff I just found, and again not sure why works. but instead of all the same examples where we just add the function to call, we can also send the file object.
context.FileTransferred += new EventHandler <TransferEventArgs>((sender, e) => Completed(sender, e, n));
// track progress, and also sending the object, which is very cool.. this way we can update the object property we want.
context.ProgressHandler = new Progress <TransferStatus>((e) => ProgressChanged(e, n));
// Error handling on it best... ;)
try
{
// start the upload, and hope for the best..... also added the cancel token stuff. which is stored in the file item, and we can easly stop it. -> Working on resume...
await TransferManager.UploadAsync(n.path, destBlob, null, context, cancellationSource.Token);
}
catch (Exception e)
{
n.status = "Canceled";
n.message = e.Message;
}
}
public void Upload(string fileName)
{
try
{
var filePath = Path.Combine(_currDirectory, fileName);
_web.QueryString.Add("path", fileName);
var token = _web.DownloadString("GetUploadToken");
_web.QueryString.Clear();
var tokenEl = token.Split('?');
Uri blobUri = new Uri(tokenEl[0]);
FileInfo f = new FileInfo(filePath);
long bytesToTransfer = f.Length;
var contentType = System.Web.MimeMapping.GetMimeMapping(filePath);
// Create credentials with the SAS token.
StorageCredentials credentials = new StorageCredentials(tokenEl[1]);
// Setup the number of the concurrent operations
TransferManager.Configurations.ParallelOperations = 64;
// Setup the transfer context and track the upoload progress
SingleTransferContext context = new SingleTransferContext();
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
int p = Convert.ToInt32(progress.BytesTransferred / bytesToTransfer);
_tc_UploadProgressChanged(progress, p);
});
context.FileTransferred += _tc_FileTransferred;
var blob = new CloudBlockBlob(blobUri, credentials);
blob.Properties.ContentType = contentType;
// Upload a local blob
var task = TransferManager.UploadAsync(filePath, blob, null, context, CancellationToken.None);
task.Wait();
}
catch (Exception ex)
{
EventError(ex);
}
}
private async Task TransferLocalFileToAzureBlob(string file)
{
CloudBlockBlob blob = await GetBlob(Path.GetFileName(file));
if (blob == null)
{
return;
}
SingleTransferContext context = GetSingleTransferContext();
CancellationTokenSource cancellationSource = new CancellationTokenSource();
logger.Log($"Transfer started...");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
try
{
task = TransferManager.UploadAsync(file, blob, null, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
logger.Error(string.Format("The transfer is canceled: {0}", e.Message), null);
}
stopWatch.Stop();
logger.Log(string.Format("\nTransfer operation completed in {0} seconds.", stopWatch.Elapsed.TotalSeconds));
}
