public async Task TransferLocalDirectoryToAzureBlobDirectory(string LocalSourceDirPath, string ContainerName)
{
CloudBlobDirectory blobDirectory = GetBlobDirectory(ContainerName);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(LocalSourceDirPath, blobDirectory, options, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
}
stopWatch.Stop();
}
public async Task TransferLocalFileToAzureBlob(string connection, string region, string containerName, string sourcePath, string targetPath)
{
try
{
CloudStorageAccount account = CloudStorageAccount.Parse(connection);
CloudBlobClient blobClient = account.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(targetPath);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
Console.WriteLine("Transfer started..." + region + "");
Stopwatch stopWatch = Stopwatch.StartNew();
if (await blob.ExistsAsync())
{
//Log.WriteLog(region + " blob is exist.");
await blob.DeleteIfExistsAsync();
}
await TransferManager.UploadAsync(sourcePath, blob);
stopWatch.Stop();
Log.WriteLog(region + " transfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
}
catch (Exception e)
{
Log.WriteLog(e.Message, Log.Type.Exception);
throw e;
}
}
public async Task TransferUrlToAzureBlob(Uri BlobUrl, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Task task;
try
{
task = TransferManager.CopyAsync(BlobUrl, blob, true, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
if (_stopFlag)
{
return;
}
//autoretry
}
}
private async static void SaveTransferCheckpoint(string jobId, TransferCheckpoint transferCheckpoint)
{
try
{
// Get reference to storage account we are using for Web Jobs Storage
CloudBlobDirectory directory = await GetCheckpointStorage();
CloudBlockBlob blob = directory.GetBlockBlobReference(jobId);
await blob.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, _blobRequestOptions, _opContext);
using (var stream = new MemoryStream())
{
IFormatter formatter = new BinaryFormatter();
formatter.Serialize(stream, transferCheckpoint);
stream.Position = 0;
await blob.UploadFromStreamAsync(stream, null, _blobRequestOptions, _opContext);
}
}
catch (Exception ex)
{
throw new Exception("Error in SaveTransferCheckpoint(): " + ex.Message);
}
}
//Copies a blob between two azure containers.
public async Task CopyAzureBlobToAzureBlob(CloudStorageAccount account, CloudBlockBlob sourceBlob, CloudBlockBlob destinationBlob)
{
TransferCheckpoint Checkpoint = null;
SingleTransferContext Context = GetSingleTransferContext(Checkpoint);
CancellationTokenSource CancellationSource = new CancellationTokenSource();
Stopwatch StopWatch = Stopwatch.StartNew();
Task Task;
try
{
Task = TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null, Context, CancellationSource.Token);
await Task;
}
catch (AggregateException e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
catch (TransferException e)
{
_Log.LogInformation($"The Azure Blob {sourceBlob.Name} already exists in {destinationBlob.Parent.Container.Name} with message {e.Message}");
}
catch (Exception e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
StopWatch.Stop();
_Log.LogInformation($"The Azure Blob {sourceBlob.Name} transfer to {destinationBlob.Name} completed in: {StopWatch.Elapsed.TotalSeconds} seconds.");
}
public async Task DownloadAzureBlobDirectoryToLocalDirectory(string LocalSourceDirPath, string ContainerName, bool RecursiveDownload = true)
{
CloudBlobDirectory blobDirectory = GetBlobDirectory(ContainerName);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Task task;
DownloadDirectoryOptions options = new DownloadDirectoryOptions()
{
Recursive = RecursiveDownload
};
try
{
task = TransferManager.DownloadDirectoryAsync(blobDirectory, LocalSourceDirPath, options, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
if (_stopFlag)
{
return;
}
//autoretry
}
}
//Copies a blob between two azure containers.
public static async Task CopyAzureBlobToAzureBlob(CloudStorageAccount account, CloudBlockBlob sourceBlob, CloudBlockBlob destinationBlob, ILogger log)
{
TransferCheckpoint Checkpoint = null;
var Context = GetSingleTransferContext(Checkpoint, log);
var CancellationSource = new CancellationTokenSource();
var StopWatch = Stopwatch.StartNew();
Task Task;
try
{
Task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, Context, CancellationSource.Token);
await Task;
}
catch (AggregateException e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
catch (Exception e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
StopWatch.Stop();
log.LogInformation("The Azure Blob " + sourceBlob + " transfer to " + destinationBlob + " completed in:" + StopWatch.Elapsed.TotalSeconds + " seconds.");
}
public async Task TransferLocalDirectoryToAzureBlobDirectory(string localDirectoryPath)
{
NLogManager.LogInfo("Updating folder");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(_connectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(_containerName);
container.CreateIfNotExistsAsync().Wait();
CloudBlobDirectory blobDirectory = container.GetDirectoryReference("");
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, context, cancellationSource.Token);
await task;
}
catch (Exception e)
{
NLogManager.LogInfo("Updating folder Error: " + e);
}
// stopWatch.Stop();
}
public static TransferCheckpoint SaveAndReloadCheckpoint(TransferCheckpoint checkpoint)
{
//return checkpoint;
Test.Info("Save and reload checkpoint");
#if BINARY_SERIALIZATION
IFormatter formatter = new BinaryFormatter();
#else
var formatter = new DataContractSerializer(typeof(TransferCheckpoint));
#endif
TransferCheckpoint reloadedCheckpoint;
string tempFileName = Guid.NewGuid().ToString();
using (var stream = new FileStream(tempFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
formatter.Serialize(stream, checkpoint);
}
using (var stream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
reloadedCheckpoint = formatter.Deserialize(stream) as TransferCheckpoint;
}
File.Delete(tempFileName);
return(reloadedCheckpoint);
}
public async Task DownloadAzureBlobToLocalFile(string DestinationFullFilePath, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
try
{
task = TransferManager.DownloadAsync(blob, DestinationFullFilePath, null, context, cancellationSource.Token); //(LocalSourceFilePath, blob, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
public async Task TransferAzureBlobToAzureBlob(string ContainerSourceName, string BlobSourceName, string ContainerDestinationName, string BlobDestinationName)
{
CloudBlockBlob sourceBlob = GetBlob(ContainerSourceName, BlobSourceName);
CloudBlockBlob destinationBlob = GetBlob(ContainerDestinationName, BlobDestinationName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(100000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
try
{
task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "azure_blockblob.png";
string destinationBlobName = "azure_blockblob2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
SingleTransferContext context = new SingleTransferContext();
// Start the transfer
try
{
// With the CopyMethod parameter, you can indicate how the content would be copied to destination blob.
// SyncCopy is to download source blob content to local memory and then upload to destination blob.
// ServiceSideAsyncCopy is to send a start-copy request to Azure Storage Sever, and Azure Storage Server will do the actual copy.
// ServiceSideSyncCopy will leverage REST API of Put Block From URL, Append Block From URL and Put Page From URL in Azure Storage Server.
// Please see <c>https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url</c> for Put Block From URL,
// <c>https://docs.microsoft.com/en-us/rest/api/storageservices/append-block-from-url</c> for Append Block From URL,
// <c>https://docs.microsoft.com/en-us/rest/api/storageservices/put-page-from-url</c> for Put Page From URL.
// Following will use ServiceSideSyncCopy to copy a blob.
Task task = TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null /* options */, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will tranferred after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
SingleTransferContext resumeContext = new SingleTransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
public static SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint)
{
SingleTransferContext context = new SingleTransferContext(checkpoint);
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Console.Write("\rBytes transferred: {0}", progress.BytesTransferred);
});
return(context);
}
public static TransferCheckpoint RandomReloadCheckpoint(TransferCheckpoint checkpoint)
{
if (Helper.RandomBoolean())
{
Test.Info("Save and reload checkpoint");
return(DMLibTestHelper.SaveAndReloadCheckpoint(checkpoint));
}
return(checkpoint);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "azure_blockblob.png";
string destinationBlobName = "azure_blockblob2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = Util.GetCloudBlob(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = Util.GetCloudBlob(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
TransferContext context = new TransferContext();
// Cancel the transfer after there's any progress reported
Progress <TransferProgress> progress = new Progress <TransferProgress>(
(transferProgress) => {
if (!cancellationSource.IsCancellationRequested)
{
Console.WriteLine("Cancel the transfer.");
// Cancel the transfer
cancellationSource.Cancel();
}
});
context.ProgressHandler = progress;
// Start the transfer
try
{
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, context, cancellationSource.Token);
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
TransferContext resumeContext = new TransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
public static async Task TransferLocalDirectoryToAzureBlobDirectory(CloudStorageAccount account)
{
string localDirectoryPath = GetDirSourcePath();
CloudBlobDirectory blobDirectory = GetBlobDirectory(account);
TransferCheckpoint checkpoint = null;
DirectoryTransferContext context = GetDirectoryTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Console.WriteLine("\nTransfer started...\nPress 'c' to temporarily cancel your transfer...\n");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
UploadDirectoryOptions options = new UploadDirectoryOptions()
{
Recursive = true
};
try
{
task = TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
Console.WriteLine("\nThe transfer is canceled: {0}", e.Message);
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nTransfer will resume in 3 seconds...");
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetDirectoryTransferContext(checkpoint);
Console.WriteLine("\nResuming transfer...\n");
await TransferManager.UploadDirectoryAsync(localDirectoryPath, blobDirectory, options, context);
}
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
//ExecuteChoice(account);
}
public static SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint, Stopwatch stopwatchTarea)
{
SingleTransferContext context = new SingleTransferContext(checkpoint);
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
Log.Information("Mb descargados {0:F}\n", (progress.BytesTransferred / 1024) / 1024);
stopwatchTarea.Restart();
});
return(context);
}
private SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint, string blobName)
{
SingleTransferContext context = new SingleTransferContext(checkpoint);
context.ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite;
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
_logger.LogDebug($"{blobName}: bytes transferred {progress.BytesTransferred}.");
});
return(context);
}
public DirectoryTransferContext GetDirectoryTransferContext(TransferCheckpoint checkpoint)
{
DirectoryTransferContext context = new DirectoryTransferContext(checkpoint);
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
if (BytesTransferred != null)
{
BytesTransferred(progress.BytesTransferred);
}
});
return(context);
}
public static async Task TransferAzureBlobToAzureBlob(CloudStorageAccount account)
{
CloudBlockBlob sourceBlob = GetBlob(account);
CloudBlockBlob destinationBlob = GetBlob(account);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Console.WriteLine("\nTransfer started...\nPress 'c' to temporarily cancel your transfer...\n");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
try
{
task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
Console.WriteLine("\nThe transfer is canceled: {0}", e.Message);
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nTransfer will resume in 3 seconds...");
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetSingleTransferContext(checkpoint);
Console.WriteLine("\nResuming transfer...\n");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false, null, context, cancellationSource.Token);
}
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
//ExecuteChoice(account);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "application-user-photos/azure_blockblob.png";
string destinationBlobName = "application-user-photos/azure_blockblob-2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
SingleTransferContext context = new SingleTransferContext();
// Start the transfer
try
{
Task task = TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will tranferred after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
SingleTransferContext resumeContext = new SingleTransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
public async Task CopyAsync(
string blobName,
CancellationToken token = default(CancellationToken))
{
var sourceBlob = _sourceContainer.GetBlobReference(blobName);
var targetBlob = _targetContainer.GetBlobReference(blobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint, blobName);
await TransferManager.CopyAsync(
sourceBlob : sourceBlob,
destBlob : targetBlob,
copyMethod : CopyMethod.ServiceSideAsyncCopy,
options : null,
context : context,
cancellationToken : token).ConfigureAwait(false);
}
//returns an Azure file transfer context for making a single file transfer.
public static SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint, ILogger log)
{
try
{
var Context = new SingleTransferContext(checkpoint);
Context.ProgressHandler = new Progress <TransferStatus>((Progress) =>
{
log.LogInformation("\rBytes transferred: {0}", Progress.BytesTransferred);
});
return(Context);
}
catch (Exception e)
{
log.LogInformation("\nGet transfer progress update fails.", e.Message);
return(null);
}
}
static void retryTransferencia(SingleTransferContext context, CloudBlockBlob cloudBlockBlob, string localFile)
{
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Stopwatch stopwatchTareaRetry = new Stopwatch();
stopwatchTareaRetry.Start();
TransferCheckpoint checkpoint = context.LastCheckpoint;
context = GetSingleTransferContext(checkpoint, stopwatchTareaRetry);
Console.WriteLine("\nReanudando la transferencia...\n");
try
{
DownloadOptions optionsWithDisableContentMD5Validation2 = new DownloadOptions()
{
DisableContentMD5Validation = true
};
var taskRetry = TransferManager.DownloadAsync(cloudBlockBlob, localFile, optionsWithDisableContentMD5Validation2, context, cancellationSource.Token);
evaluacionDeCaracteresDeErrorYretry(taskRetry, stopwatchTareaRetry);
taskRetry.Wait();
}
catch (Exception e)
{
Log.Error("\nLa transferencia fue cancelada : {0}", e.Message);
cancellationSource.Cancel();
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nLa transferencia será reanudada en 3 segundos...");
Thread.Sleep(3000);
retryTransferencia(context, cloudBlockBlob, localFile);
}
}
//returns an Azure file transfer context for making a single file transfer.
public SingleTransferContext GetSingleTransferContext(TransferCheckpoint checkpoint)
{
try
{
SingleTransferContext context = new SingleTransferContext(checkpoint)
{
ProgressHandler = new Progress <TransferStatus>((progress) =>
{
_Log.LogInformation("\rBytes transferred: {0}", progress.BytesTransferred);
})
};
return(context);
}
catch (Exception e)
{
_Log.LogInformation("\nGet transfer progress update fails.", e.Message);
return(null);
}
}
internal static void Write(TransferCheckpoint transferCheckpoint, string transferCheckpointFilename)
{
lock (_lock)
{
using var writer = new FileStream(transferCheckpointFilename, FileMode.Create, FileAccess.Write);
if (transferCheckpoint.GetType().IsSerializable)
{
var formatter = new BinaryFormatter();
formatter.Serialize(writer, transferCheckpoint);
}
else
{
var ser = new DataContractJsonSerializer(transferCheckpoint.GetType());
ser.WriteObject(writer, transferCheckpoint);
}
writer.Flush();
writer.Close();
}
}
public async Task TransferLocalFileToAzureBlob(string LocalSourceFilePath, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(1000000);
Task task;
try
{
task = TransferManager.UploadAsync(LocalSourceFilePath, blob, null, context, cancellationSource.Token);
if (ExposeTaskCancelation != null)
{
ExposeTaskCancelation(task, cancellationSource);
}
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
if (_stopFlag)
{
return;
}
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetSingleTransferContext(checkpoint);
await TransferManager.UploadAsync(LocalSourceFilePath, blob, null, context);
}
}
public async Task CopyAsync(string blobName, CancellationToken token = default(CancellationToken))
{
var sourceBlob = _sourceContainer.GetBlobReference(blobName);
var targetBlob = _targetContainer.GetBlobReference(blobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint, blobName);
try
{
await TransferManager.CopyAsync(
sourceBlob : sourceBlob,
destBlob : targetBlob,
copyMethod : CopyMethod.ServiceSideAsyncCopy,
options : null,
context : context,
cancellationToken : token);
}
catch (UnauthorizedAccessException)
{
_logger.LogInformation($"Failed to copy blob. You don't have permission to copy");
throw;
}
}
private async static Task <TransferCheckpoint> GetTransferCheckpoint(string jobId)
{
TransferCheckpoint transferCheckpoint = null;
try
{
// Get reference to storage account we are using for Web Jobs Storage
CloudBlobDirectory directory = await GetCheckpointStorage();
CloudBlockBlob blob = directory.GetBlockBlobReference(jobId);
if (await blob.ExistsAsync(_blobRequestOptions, _opContext))
{
using (var stream = new MemoryStream())
{
await blob.DownloadToStreamAsync(stream, null, _blobRequestOptions, _opContext);
stream.Position = 0;
// Deserialize
IFormatter formatter = new BinaryFormatter();
transferCheckpoint = formatter.Deserialize(stream) as TransferCheckpoint;
//_log.WriteLine("Resuming Copy. Job Id: " + jobId);
await Logger.JobInfoAsync("Resuming Copy. Job Id: " + jobId);
}
// Clean up the serialized CheckPoint
await blob.DeleteAsync(DeleteSnapshotsOption.None, null, _blobRequestOptions, _opContext);
}
}
catch (Exception ex)
{
throw new Exception("Error in GetTransferCheckpoint(): " + ex.Message);
}
return(transferCheckpoint);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlobDirectory
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint into a file after transfer being cancelled
/// 4. Reload checkpoint from the file
/// 4. Resume the transfer with the loaded checkpoint
/// </summary>
private static async Task BlobDirectoryCopySample()
{
CloudBlobDirectory sourceBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir");
CloudBlobDirectory destBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir2");
// When source is CloudBlobDirectory:
// 1. If recursive is set to true, data movement library matches the source blob name against SearchPattern as prefix.
// 2. Otherwise, data movement library matches the blob with the exact name specified by SearchPattern.
//
// You can also replace the source directory with a CloudFileDirectory instance to copy data from Azure File Storage. If so:
// 1. If recursive is set to true, SearchPattern is not supported. Data movement library simply transfer all azure files
// under the source CloudFileDirectory and its sub-directories.
// 2. Otherwise, data movement library matches the azure file with the exact name specified by SearchPattern.
//
// In the following case, data movement library will copy all blobs with the prefix "azure" in source blob directory.
CopyDirectoryOptions options = new CopyDirectoryOptions()
{
SearchPattern = "azure",
Recursive = true,
};
DirectoryTransferContext context = new DirectoryTransferContext();
context.FileTransferred += FileTransferredCallback;
context.FileFailed += FileFailedCallback;
context.FileSkipped += FileSkippedCallback;
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
TransferStatus trasferStatus = null;
try
{
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
trasferStatus = await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Serialize the checkpoint into a file
#if DOTNET5_4
var formatter = new DataContractSerializer(typeof(TransferCheckpoint));
#else
IFormatter formatter = new BinaryFormatter();
#endif
string tempFileName = Guid.NewGuid().ToString();
using (var stream = new FileStream(tempFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
formatter.Serialize(stream, checkpoint);
}
// Deserialize the checkpoint from the file
using (var stream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
checkpoint = formatter.Deserialize(stream) as TransferCheckpoint;
}
File.Delete(tempFileName);
// Create a new TransferContext with the store checkpoint
DirectoryTransferContext resumeContext = new DirectoryTransferContext(checkpoint);
resumeContext.FileTransferred += FileTransferredCallback;
resumeContext.FileFailed += FileFailedCallback;
resumeContext.FileSkipped += FileSkippedCallback;
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
resumeContext.ProgressHandler = recorder;
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
trasferStatus = await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, resumeContext);
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus));
}
