public void TestCopySourceNotExisted()
{
COSXMLCopyTask copyTask = new COSXMLCopyTask(notExistBucket, "remote_key", null);
Assert.ThrowsAsync <CosClientException>(async() => await transferManager.CopyAsync(copyTask));
CopySourceStruct notExistSource = new CopySourceStruct(QCloudServer.Instance().appid,
notExistBucket, QCloudServer.Instance().region, "example_key");
COSXMLCopyTask copyTask2 = new COSXMLCopyTask(notExistBucket, "remote_key", notExistSource);
Assert.ThrowsAsync <CosServerException>(async() => await transferManager.CopyAsync(copyTask2));
}
public async Task <Uri> CopyBlobAsync(string fileName, string sourceContainerName, string targetContainerName)
{
var sourceContainer = await GetOrCreateContainerAsync(sourceContainerName);
var targetContainer = await GetOrCreateContainerAsync(targetContainerName);
var sourceBlob = sourceContainer.GetBlockBlobReference(fileName);
var targetBlob = targetContainer.GetBlockBlobReference(fileName);
var isSourceBlobExists = await sourceBlob.ExistsAsync();
if (!isSourceBlobExists)
{
throw new BlobNotFoundException($"Source blob doesn't found. FileName={fileName}, Container={sourceContainerName}");
}
var context = new SingleTransferContext
{
ShouldOverwriteCallbackAsync = (source, destination) => Task.FromResult(true)
};
var cancellationSource = new CancellationTokenSource();
await TransferManager.CopyAsync(sourceBlob, targetBlob, true, null, context, cancellationSource.Token);
return(targetBlob.Uri);
}
private async static Task CopyBlobAsync(CloudBlob sourceBlob, CloudBlob destinationBlob, TransferContext transferContext, TransferCheckpoint transferCheckpoint, CancellationTokenSource cancellationTokenSource)
{
// Start the transfer
try
{
await TransferManager.CopyAsync(
sourceBlob : sourceBlob,
destBlob : destinationBlob,
isServiceCopy : true, //Async Server-Side Copy
options : null,
context : transferContext,
cancellationToken : cancellationTokenSource.Token);
// Store the transfer checkpoint.
// [longwen] How does the resume feature work here? Looks the checkpoint is assigned to a local variable and lost afterwards.
// FYI, the checkpoint is an serializable object. Thus, DMLib can resume a transfer cross process by presisting the
// checkpoint into a local file and then load it back.
transferCheckpoint = transferContext.LastCheckpoint;
}
catch (TransferException te)
{
// Swallow Exceptions from skipped files in Overwrite Callback
// Log any other Transfer Exceptions
if (te.ErrorCode != TransferErrorCode.SubTransferFails)
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Transfer Error: " + te.Message);
sb.AppendLine("Transfer Error Code: " + te.ErrorCode);
await _log.WriteLineAsync(sb.ToString());
}
}
}
public async Task TransferUrlToAzureBlob(Uri BlobUrl, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Task task;
try
{
task = TransferManager.CopyAsync(BlobUrl, blob, true, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
if (_stopFlag)
{
return;
}
//autoretry
}
}
//Copies a blob between two azure containers.
public static async Task CopyAzureBlobToAzureBlob(CloudStorageAccount account, CloudBlockBlob sourceBlob, CloudBlockBlob destinationBlob, ILogger log)
{
TransferCheckpoint Checkpoint = null;
var Context = GetSingleTransferContext(Checkpoint, log);
var CancellationSource = new CancellationTokenSource();
var StopWatch = Stopwatch.StartNew();
Task Task;
try
{
Task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, Context, CancellationSource.Token);
await Task;
}
catch (AggregateException e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
catch (Exception e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
StopWatch.Stop();
log.LogInformation("The Azure Blob " + sourceBlob + " transfer to " + destinationBlob + " completed in:" + StopWatch.Elapsed.TotalSeconds + " seconds.");
}
/// 高级接口拷贝对象
public async void TransferCopyObject()
{
TransferConfig transferConfig = new TransferConfig();
// 初始化 TransferManager
TransferManager transferManager = new TransferManager(cosXml, transferConfig);
//.cssg-snippet-body-start:[transfer-copy-object]
string sourceAppid = "1250000000"; //账号 appid
string sourceBucket = "sourcebucket-1250000000"; //"源对象所在的存储桶
string sourceRegion = "COS_REGION"; //源对象的存储桶所在的地域
string sourceKey = "sourceObject"; //源对象键
//构造源对象属性
CopySourceStruct copySource = new CopySourceStruct(sourceAppid, sourceBucket,
sourceRegion, sourceKey);
string bucket = "examplebucket-1250000000"; //目标存储桶，格式：BucketName-APPID
string key = "exampleobject"; //目标对象的对象键
COSXMLCopyTask copytask = new COSXMLCopyTask(bucket, key, copySource);
try {
COSXML.Transfer.COSXMLCopyTask.CopyTaskResult result = await
transferManager.CopyAsync(copytask);
Console.WriteLine(result.GetResultInfo());
string eTag = result.eTag;
} catch (Exception e) {
Console.WriteLine("CosException: " + e);
}
//.cssg-snippet-body-end
}
//Copies a blob between two azure containers.
public async Task CopyAzureBlobToAzureBlob(CloudStorageAccount account, CloudBlockBlob sourceBlob, CloudBlockBlob destinationBlob)
{
TransferCheckpoint Checkpoint = null;
SingleTransferContext Context = GetSingleTransferContext(Checkpoint);
CancellationTokenSource CancellationSource = new CancellationTokenSource();
Stopwatch StopWatch = Stopwatch.StartNew();
Task Task;
try
{
Task = TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null, Context, CancellationSource.Token);
await Task;
}
catch (AggregateException e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
catch (TransferException e)
{
_Log.LogInformation($"The Azure Blob {sourceBlob.Name} already exists in {destinationBlob.Parent.Container.Name} with message {e.Message}");
}
catch (Exception e)
{
e.Data.Add("sourceBlobName", sourceBlob);
e.Data.Add("destinationBlocName", destinationBlob);
throw;
}
StopWatch.Stop();
_Log.LogInformation($"The Azure Blob {sourceBlob.Name} transfer to {destinationBlob.Name} completed in: {StopWatch.Elapsed.TotalSeconds} seconds.");
}
static void Main(string[] args)
{
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("sourceStorageConnectionString"));
CloudStorageAccount targetStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("targetStorageConnectionString"));
CloudBlobClient sourceCloudBlobClient = sourceStorageAccount.CreateCloudBlobClient();
CloudBlobClient targetCloudBlobClient = targetStorageAccount.CreateCloudBlobClient();
CloudBlobContainer sourceContainer = sourceCloudBlobClient.GetContainerReference(CloudConfigurationManager.GetSetting("sourceContainer"));
CloudBlobContainer targetContainer = targetCloudBlobClient.GetContainerReference(CloudConfigurationManager.GetSetting("targetContainer"));
targetContainer.CreateIfNotExists();
// Copy each blob
foreach (IListBlobItem blob in sourceContainer.ListBlobs(useFlatBlobListing: true))
{
Uri thisBlobUri = blob.Uri;
var blobName = Path.GetFileName(thisBlobUri.ToString());
Console.WriteLine("Copying blob: " + blobName);
CloudBlockBlob sourceBlob = sourceContainer.GetBlockBlobReference(blobName);
CloudBlockBlob targetBlob = targetContainer.GetBlockBlobReference(blobName);
Task task = TransferManager.CopyAsync(sourceBlob, targetBlob, true /* isServiceCopy */);
}
Console.WriteLine("Press any key to continue:");
Console.Read();
}
public static async Task <string> CopyBlob(
[ActivityTrigger] string blobName,
TraceWriter log)
{
if (!string.IsNullOrEmpty(blobName))
{
CloudBlockBlob sourceBlob = await GetBlob("AzureWebJobsStorage", ConfigurationManager.AppSettings["SourceContainername"], blobName);
CloudBlockBlob destinationBlob = await GetBlob("StorageDestination", ConfigurationManager.AppSettings["DestinationsContainername"], blobName);
log.Info($"\nBlob: {blobName} Transfer started...\n");
try
{
if (!destinationBlob.Exists())
{
await TransferManager.CopyAsync(sourceBlob, destinationBlob, true);
}
log.Info($"\nBlob: {blobName} Transfer operation completed");
return(blobName);
}
catch (Exception ex)
{
Console.WriteLine($"\nBlob: {blobName} The transfer is canceled: {0}", ex.Message);
throw;
}
}
return(string.Empty);
}
public async Task TransferAzureBlobToAzureBlob(string ContainerSourceName, string BlobSourceName, string ContainerDestinationName, string BlobDestinationName)
{
CloudBlockBlob sourceBlob = GetBlob(ContainerSourceName, BlobSourceName);
CloudBlockBlob destinationBlob = GetBlob(ContainerDestinationName, BlobDestinationName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(100000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
try
{
task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "azure_blockblob.png";
string destinationBlobName = "azure_blockblob2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
SingleTransferContext context = new SingleTransferContext();
// Start the transfer
try
{
// With the CopyMethod parameter, you can indicate how the content would be copied to destination blob.
// SyncCopy is to download source blob content to local memory and then upload to destination blob.
// ServiceSideAsyncCopy is to send a start-copy request to Azure Storage Sever, and Azure Storage Server will do the actual copy.
// ServiceSideSyncCopy will leverage REST API of Put Block From URL, Append Block From URL and Put Page From URL in Azure Storage Server.
// Please see <c>https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url</c> for Put Block From URL,
// <c>https://docs.microsoft.com/en-us/rest/api/storageservices/append-block-from-url</c> for Append Block From URL,
// <c>https://docs.microsoft.com/en-us/rest/api/storageservices/put-page-from-url</c> for Put Page From URL.
// Following will use ServiceSideSyncCopy to copy a blob.
Task task = TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null /* options */, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will tranferred after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
SingleTransferContext resumeContext = new SingleTransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, CopyMethod.ServiceSideSyncCopy, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
protected override async Task OnCopyAsync(
Uri sourceUri,
IStorage destinationStorage,
Uri destinationUri,
IReadOnlyDictionary <string, string> destinationProperties,
CancellationToken cancellationToken)
{
var azureDestinationStorage = destinationStorage as AzureStorage;
if (azureDestinationStorage == null)
{
throw new NotImplementedException("Copying is only supported from Azure storage to Azure storage.");
}
string sourceName = GetName(sourceUri);
string destinationName = azureDestinationStorage.GetName(destinationUri);
CloudBlockBlob sourceBlob = _directory.GetBlockBlobReference(sourceName);
CloudBlockBlob destinationBlob = azureDestinationStorage._directory.GetBlockBlobReference(destinationName);
var context = new SingleTransferContext();
if (destinationProperties?.Count > 0)
{
context.SetAttributesCallback = new SetAttributesCallback((destination) =>
{
var blob = (CloudBlockBlob)destination;
// The copy statement copied all properties from the source blob to the destination blob; however,
// there may be required properties on destination blob, all of which may have not already existed
// on the source blob at the time of copy.
foreach (var property in destinationProperties)
{
switch (property.Key)
{
case StorageConstants.CacheControl:
blob.Properties.CacheControl = property.Value;
break;
case StorageConstants.ContentType:
blob.Properties.ContentType = property.Value;
break;
default:
throw new NotImplementedException($"Storage property '{property.Value}' is not supported.");
}
}
});
}
context.ShouldOverwriteCallback = new ShouldOverwriteCallback((source, destination) => true);
await TransferManager.CopyAsync(sourceBlob, destinationBlob, _useServerSideCopy, options : null, context : context);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "azure_blockblob.png";
string destinationBlobName = "azure_blockblob2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = Util.GetCloudBlob(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = Util.GetCloudBlob(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
TransferContext context = new TransferContext();
// Cancel the transfer after there's any progress reported
Progress <TransferProgress> progress = new Progress <TransferProgress>(
(transferProgress) => {
if (!cancellationSource.IsCancellationRequested)
{
Console.WriteLine("Cancel the transfer.");
// Cancel the transfer
cancellationSource.Cancel();
}
});
context.ProgressHandler = progress;
// Start the transfer
try
{
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, context, cancellationSource.Token);
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
TransferContext resumeContext = new TransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
public static async Task CopyFromAssetToBlob(IAsset asset, string sourceConn, string targetConn, string targetContainer, string targetDirectory,
Func <string, bool> action = null)
{
int skipSize = 0;
int batchSize = 1000;
int currentBatch = 0;
int i = 0;
int len = asset.AssetFiles.Count();
CloudStorageAccount sourceAccount = CloudStorageAccount.Parse(sourceConn);
CloudStorageAccount targertAccount = CloudStorageAccount.Parse(targetConn);
var sourceClient = sourceAccount.CreateCloudBlobClient();
var sourceContainerUri = asset.Uri.ToString();
var sourceContainer = sourceClient.GetContainerReference(sourceContainerUri.Substring(sourceContainerUri.LastIndexOf('/') + 1));
var targetClient = targertAccount.CreateCloudBlobClient();
var targetContainerBlob = targetClient.GetContainerReference(targetContainer);
var directory = targetContainerBlob.GetDirectoryReference(targetDirectory);
List <Task> tasks = new List <Task>();
while (true)
{
// Loop through all Jobs (1000 at a time) in the Media Services account
IQueryable _assetCollectionQuery = asset.AssetFiles.Skip(skipSize).Take(batchSize);
foreach (IAssetFile assetFile in _assetCollectionQuery)
{
bool isOk = true;
if (action != null)
{
isOk = action(assetFile.MimeType);
}
if (isOk)
{
CloudBlockBlob sourceBlockBlob = sourceContainer.GetBlockBlobReference(assetFile.Name);
CloudBlockBlob destBlockBlob = directory.GetBlockBlobReference(assetFile.Name);
tasks.Add(TransferManager.CopyAsync(sourceBlockBlob, destBlockBlob, true));
currentBatch++;
i++;
}
}
if (currentBatch == batchSize)
{
skipSize += batchSize;
currentBatch = 0;
}
else
{
break;
}
}
await Task.WhenAll(tasks);
}
private static async Task TransferAzureBlobToAzureBlob(CloudStorageAccount account)
{
// From・Toの情報を取得
CloudBlockBlob sourceBlob = GetBlob(account);
CloudBlockBlob destinationBlob = GetBlob(account);
// 転送処理の実施
Console.WriteLine("\nTransfer started...\n");
var stopWatch = Stopwatch.StartNew();
await TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, GetSingleTransferContext());
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
}
public static async Task TransferAzureBlobToAzureBlob(CloudStorageAccount account)
{
CloudBlockBlob sourceBlob = GetBlob(account);
CloudBlockBlob destinationBlob = GetBlob(account);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Console.WriteLine("\nTransfer started...\nPress 'c' to temporarily cancel your transfer...\n");
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
ConsoleKeyInfo keyinfo;
try
{
task = TransferManager.CopyAsync(sourceBlob, destinationBlob, true, null, context, cancellationSource.Token);
while (!task.IsCompleted)
{
if (Console.KeyAvailable)
{
keyinfo = Console.ReadKey(true);
if (keyinfo.Key == ConsoleKey.C)
{
cancellationSource.Cancel();
}
}
}
await task;
}
catch (Exception e)
{
Console.WriteLine("\nThe transfer is canceled: {0}", e.Message);
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nTransfer will resume in 3 seconds...");
Thread.Sleep(3000);
checkpoint = context.LastCheckpoint;
context = GetSingleTransferContext(checkpoint);
Console.WriteLine("\nResuming transfer...\n");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false, null, context, cancellationSource.Token);
}
stopWatch.Stop();
Console.WriteLine("\nTransfer operation completed in " + stopWatch.Elapsed.TotalSeconds + " seconds.");
//ExecuteChoice(account);
}
/// <summary>
/// Copy data between Azure storage.
/// 1. Copy a CloudBlob
/// 2. Cancel the transfer before it finishes with a CancellationToken
/// 3. Store the transfer checkpoint after transfer being cancelled
/// 4. Resume the transfer with the stored checkpoint
/// </summary>
private static async Task BlobCopySample()
{
string sourceBlobName = "application-user-photos/azure_blockblob.png";
string destinationBlobName = "application-user-photos/azure_blockblob-2.png";
// Create the source CloudBlob instance
CloudBlob sourceBlob = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName, BlobType.BlockBlob);
// Create the destination CloudBlob instance
CloudBlob destinationBlob = await Util.GetCloudBlobAsync(ContainerName, destinationBlobName, BlobType.BlockBlob);
// Create CancellationTokenSource used to cancel the transfer
CancellationTokenSource cancellationSource = new CancellationTokenSource();
TransferCheckpoint checkpoint = null;
SingleTransferContext context = new SingleTransferContext();
// Start the transfer
try
{
Task task = TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, context, cancellationSource.Token);
// Sleep for 1 seconds and cancel the transfer.
// It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will tranferred after resume.
Thread.Sleep(1000);
Console.WriteLine("Cancel the transfer.");
cancellationSource.Cancel();
await task;
}
catch (Exception e)
{
Console.WriteLine("The transfer is cancelled: {0}", e.Message);
}
// Store the transfer checkpoint
checkpoint = context.LastCheckpoint;
// Create a new TransferContext with the store checkpoint
SingleTransferContext resumeContext = new SingleTransferContext(checkpoint);
// Resume transfer from the stored checkpoint
Console.WriteLine("Resume the cancelled transfer.");
await TransferManager.CopyAsync(sourceBlob, destinationBlob, false /* isServiceCopy */, null /* options */, resumeContext);
Console.WriteLine("CloudBlob {0} is copied to {1} successfully.", sourceBlob.Uri.ToString(), destinationBlob.Uri.ToString());
}
public async Task CopyAsync(
string blobName,
CancellationToken token = default(CancellationToken))
{
var sourceBlob = _sourceContainer.GetBlobReference(blobName);
var targetBlob = _targetContainer.GetBlobReference(blobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint, blobName);
await TransferManager.CopyAsync(
sourceBlob : sourceBlob,
destBlob : targetBlob,
copyMethod : CopyMethod.ServiceSideAsyncCopy,
options : null,
context : context,
cancellationToken : token).ConfigureAwait(false);
}
public async Task CopyFile(string sourceFolderName,
string targetFolderName,
string sourceFileName,
string destinationFileName)
{
var(sourceContainerName, sourceBlobName) = GetContainerAndBlobName(sourceFolderName, sourceFileName);
var(targetContainerName, destinationBlobName) =
GetContainerAndBlobName(targetFolderName, destinationFileName);
CloudBlobContainer sourceContainer = await CreateContainer(sourceContainerName);
CloudBlobContainer targetContainer = await CreateContainer(targetContainerName);
var sourceBlob = sourceContainer.GetBlockBlobReference(sourceBlobName);
var targetBlob = targetContainer.GetBlockBlobReference(destinationBlobName);
await TransferManager.CopyAsync(sourceBlob, targetBlob, CopyMethod.ServiceSideAsyncCopy);
}
private Task Copy(dynamic sourceObject, dynamic destObject, TransferItem item)
{
CopyOptions copyOptions = item.Options as CopyOptions;
SingleTransferContext transferContext = item.TransferContext as SingleTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy, copyOptions, transferContext, cancellationToken));
}
else if (transferContext != null || copyOptions != null)
{
return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy, copyOptions, transferContext));
}
else
{
return(TransferManager.CopyAsync(sourceObject, destObject, item.IsServiceCopy));
}
}
public static async Task CopyBlob(string sourceConnection, string destinationConnection, string containerName, string blobName)
{
var sourceAccount = CloudStorageAccount.Parse(sourceConnection);
var destinationAccount = CloudStorageAccount.Parse(destinationConnection);
var sourceClient = sourceAccount.CreateCloudBlobClient();
var destinationClient = destinationAccount.CreateCloudBlobClient();
var destContainer = destinationClient.GetContainerReference(containerName);
destContainer.CreateIfNotExists();
var sourceBlob = sourceClient.GetContainerReference(containerName).GetBlobReference(blobName);
var destBlob = destContainer.GetBlobReference(blobName);
var transferContext = new SingleTransferContext
{
ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite
};
await TransferManager.CopyAsync(sourceBlob, destBlob, true, null, transferContext);
}
public async Task CopyAsync(string blobName, CancellationToken token = default(CancellationToken))
{
var sourceBlob = _sourceContainer.GetBlobReference(blobName);
var targetBlob = _targetContainer.GetBlobReference(blobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint, blobName);
try
{
await TransferManager.CopyAsync(
sourceBlob : sourceBlob,
destBlob : targetBlob,
copyMethod : CopyMethod.ServiceSideAsyncCopy,
options : null,
context : context,
cancellationToken : token);
}
catch (UnauthorizedAccessException)
{
_logger.LogInformation($"Failed to copy blob. You don't have permission to copy");
throw;
}
}
public static async Task CopyFromAssetToBlob(IAzureMediaServicesClient client, string resourceGroup, string accountName,
string assetName, string resultBlobConnString, string resultContainer, string resultsFolder, string filter)
{
AssetContainerSas assetContainerSas = client.Assets.ListContainerSas(resourceGroup, accountName, assetName, permissions: AssetContainerPermission.Read,
expiryTime: DateTime.UtcNow.AddHours(1).ToUniversalTime());
Uri containerSasUrl = new Uri(assetContainerSas.AssetContainerSasUrls.FirstOrDefault());
CloudBlobContainer container = new CloudBlobContainer(containerSasUrl);
CloudStorageAccount targertAccount = CloudStorageAccount.Parse(resultBlobConnString);
var targetClient = targertAccount.CreateCloudBlobClient();
var targetContainer = targetClient.GetContainerReference(resultContainer);
var directory = targetContainer.GetDirectoryReference(resultsFolder);
string filename = string.Empty;
List <Task> tasks = new List <Task>();
foreach (var blobItem in
container.ListBlobs(null, true, BlobListingDetails.None).OfType <CloudBlockBlob>().Where(b => b.Name.EndsWith(filter)))
{
CloudBlockBlob destBlockBlob = directory.GetBlockBlobReference(blobItem.Name);
tasks.Add(TransferManager.CopyAsync(blobItem, destBlockBlob, true));
}
await Task.WhenAll(tasks);
}
static async Task TransferFilesAsync()
{
var primaryStorageConnectionString = "";
var secondaryStorageConnectionString = "";
var primaryAccount = CloudStorageAccount.Parse(primaryStorageConnectionString);
var primaryBlobClient = primaryAccount.CreateCloudBlobClient();
var primaryBlobContainer = primaryBlobClient.GetContainerReference("files");
await primaryBlobContainer.CreateIfNotExistsAsync();
var secondaryAccount = CloudStorageAccount.Parse(secondaryStorageConnectionString);
var secondaryBlobClient = secondaryAccount.CreateCloudBlobClient();
var secondaryBlobContainer = secondaryBlobClient.GetContainerReference("files");
await secondaryBlobContainer.CreateIfNotExistsAsync();
var files = Directory.GetFiles("files").ToList();
var tasks = files.Select(file =>
{
var copyOptions = new CopyOptions {
};
var context = new SingleTransferContext();
context.ShouldOverwriteCallbackAsync = (source, destination) => Task.FromResult(true);
return(Task.Run(async() =>
{
var fileName = Path.GetFileName(file);
var primaryBlob = primaryBlobContainer.GetBlockBlobReference(fileName);
await TransferManager.UploadAsync(file, primaryBlob, null, context);
var secondaryBlob = secondaryBlobContainer.GetBlockBlobReference(fileName);
await TransferManager.CopyAsync(primaryBlob, secondaryBlob, isServiceCopy: true, context: context, options: copyOptions);
}));
}).ToArray();
Task.WaitAll(tasks);
}
async void RenameToolStripMenuItem_ClickAsync(object sender, EventArgs e)
{
TreeNode node = Containers.SelectedNode;
if (node == null)
{
MessageBox.Show("Choose an item to rename first");
return;
}
CloudBlobDirectory sourceDir;
CloudBlobDirectory targetDir;
var options = new CopyDirectoryOptions {
Recursive = true
};
TreeNode parentNode = node.Parent;
switch (node.ImageIndex)
{
case 0:
new AskForm("Rename Container").ShowDialog();
if (Answer == null)
{
return;
}
if (!Answer.Any(n => n.IsLetter()))
{
MessageBox.Show("Russian is not allowed");
return;
}
ProgressBar.Visible = true;
CloudBlobContainer sourceC = Client.GetContainerReference(node.Text);
CloudBlobContainer targetC = Client.GetContainerReference(Answer);
await targetC.CreateIfNotExistsAsync();
foreach (var blob in sourceC.ListBlobs())
{
switch (blob)
{
case CloudBlobDirectory cbd:
sourceDir = sourceC.GetDirectoryReference(blob.Uri.Directory());
targetDir = targetC.GetDirectoryReference(blob.Uri.Directory());
await TransferManager.CopyDirectoryAsync(sourceDir, targetDir, true, options, null);
break;
case CloudBlob cb:
CloudBlob sourceBlob = sourceC.GetBlobReference(blob.Uri.File());
CloudBlob targetBlob = targetC.GetBlobReference(blob.Uri.File());
await TransferManager.CopyAsync(sourceBlob, targetBlob, true);
break;
}
}
await sourceC.DeleteIfExistsAsync();
break;
case 1:
switch (parentNode.ImageIndex)
{
case 0:
CloudBlobContainer container = Client.GetContainerReference(parentNode.Text);
sourceDir = container.GetDirectoryReference(node.Text);
new AskForm("Rename Directory").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
targetDir = container.GetDirectoryReference(Answer);
await TransferManager.CopyDirectoryAsync(sourceDir, targetDir, true, options, null);
await DeleteDirectory(sourceDir);
break;
case 1:
(TreeNode containerNode, int count, string[] hier) = GetContainerNode(parentNode);
Array.Reverse(hier);
container = Client.GetContainerReference(containerNode.Text);
sourceDir = container.GetDirectoryReference(hier[0]);
for (int i = 0; i < count; i++)
{
sourceDir = sourceDir.GetDirectoryReference(hier[i + 1]);
}
sourceDir = sourceDir.GetDirectoryReference(node.Text);
new AskForm("Rename Directory").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
targetDir = sourceDir.Parent.GetDirectoryReference(Answer);
await TransferManager.CopyDirectoryAsync(sourceDir, targetDir, true, options, null);
await DeleteDirectory(sourceDir);
break;
}
break;
case 2:
switch (parentNode.ImageIndex)
{
case 0:
CloudBlobContainer container = Client.GetContainerReference(parentNode.Text);
CloudBlob sourceBlob = container.GetBlobReference(node.Text);
new AskForm("Rename Cloud Blob").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
CloudBlob targetBlob = container.GetBlobReference(Answer);
await TransferManager.CopyAsync(sourceBlob, targetBlob, true);
await sourceBlob.DeleteIfExistsAsync();
break;
case 1:
(TreeNode containerNode, int count, string[] hier) = GetContainerNode(parentNode);
Array.Reverse(hier);
container = Client.GetContainerReference(containerNode.Text);
sourceDir = container.GetDirectoryReference(hier[0]);
for (int i = 0; i < count; i++)
{
sourceDir = sourceDir.GetDirectoryReference(hier[i + 1]);
}
sourceBlob = sourceDir.GetBlobReference(node.Text);
new AskForm("Rename Cloud Blob").ShowDialog();
if (Answer == null)
{
return;
}
ProgressBar.Visible = true;
targetBlob = sourceDir.GetBlobReference(Answer);
await TransferManager.CopyAsync(sourceBlob, targetBlob, true);
await sourceBlob.DeleteIfExistsAsync();
break;
}
break;
}
RefreshAll();
ProgressBar.Visible = false;
}
/// <summary>
/// Demonstrates how to do backup using incremental snapshots. See this article on
/// backing up Azure virtual machine disks with incremental snapshots:
/// https://azure.microsoft.com/en-us/documentation/articles/storage-incremental-snapshots/
/// The article also describes how to restore a disk from incremental snapshots.
/// </summary>
/// <returns>Task</returns>
public static async Task IncrementalSnapshotBackupAsync()
{
const int TotalBlobSize = 512 * 6; // The total amount of data in the blob
const int UpdateWriteSize = 512 * 2; // The amount of data to update in the blob
const int UpdateWriteOffset = 512 * 2; // The offset at which to write updated data
const int ClearPagesOffset = 512 * 5; // The offset at which to begin clearing page data
const int ClearPagesSize = 512; // The amount of data to clear in the blob
string SimulationID = Guid.NewGuid().ToString(); // The simulation ID
string ContainerName = "container-" + SimulationID; // The name of the primary container
string PageBlobName = "binary-data"; // The name of the primary page blob
// Retrieve storage account information from connection strings
// How to create storage connection strings - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudStorageAccount backupStorageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("BackupStorageConnectionString"));
// Create blob clients for interacting with the blob service.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobClient backupBlobClient = backupStorageAccount.CreateCloudBlobClient();
// Create containers for organizing blobs within the storage accounts.
container = blobClient.GetContainerReference(ContainerName);
backupContainer = backupBlobClient.GetContainerReference("copy-of-" + ContainerName);
try
{
// The call below will fail if the sample is configured to use the storage emulator
// in the connection string, but the emulator is not running.
BlobRequestOptions requestOptions = new BlobRequestOptions()
{
RetryPolicy = new NoRetry()
};
await container.CreateIfNotExistsAsync(requestOptions, null);
await backupContainer.CreateIfNotExistsAsync(requestOptions, null);
}
catch (StorageException)
{
Console.WriteLine("If you are running with the default connection string, please make " +
"sure you have started the storage emulator. Press the Windows key and type Azure " +
"Storage to select and run it from the list of applications - then restart the sample.");
Console.ReadLine();
throw;
}
// Create, write to, and snapshot a page blob in the newly created container.
CloudPageBlob pageBlob = container.GetPageBlobReference(PageBlobName);
await pageBlob.CreateAsync(TotalBlobSize);
byte[] samplePagedata = new byte[TotalBlobSize];
Random random = new Random();
random.NextBytes(samplePagedata);
await pageBlob.UploadFromByteArrayAsync(samplePagedata, 0, samplePagedata.Length);
CloudPageBlob pageBlobSnap = await pageBlob.CreateSnapshotAsync();
// Create a blob in the backup account
CloudPageBlob pageBlobBackup = backupContainer.GetPageBlobReference("copy-of-" + PageBlobName);
// Copy to the blob in the backup account. Notice that the <isServiceCopy> flag here is
// set to 'false'. This forces data to be first downloaded from the source, then uploaded
// to the destination. If <isServiceCopy> were set to 'true' we could avoid the I/O of
// downloading and then uploading, but could not guarantee that the copy completed even
// after the local task completed. Since the remainder of this sample depends on the
// successful completion of this operation, we must set <isServiceCopy> to 'false.'
await TransferManager.CopyAsync(pageBlobSnap, pageBlobBackup, false);
// Snapshot the backup copy
CloudPageBlob pageBlobBackupSnap = await pageBlob.CreateSnapshotAsync();
// Change the contents of the original page blob (note: we must convert the byte array to a
// stream in order to write to a non-zero offset in the page blob) and snapshot.
byte[] updatePageData = new byte[UpdateWriteSize];
random.NextBytes(updatePageData);
Stream updatePageDataStream = new MemoryStream(updatePageData);
await pageBlob.WritePagesAsync(updatePageDataStream, UpdateWriteOffset, null);
await pageBlob.ClearPagesAsync(ClearPagesOffset, ClearPagesSize);
CloudPageBlob newPageBlobSnap = await pageBlob.CreateSnapshotAsync();
// Get the incremental changes and write to the backup.
IEnumerable <PageDiffRange> changedPages =
await newPageBlobSnap.GetPageRangesDiffAsync(pageBlobSnap.SnapshotTime.Value);
foreach (PageDiffRange pageRange in changedPages)
{
// If this page range is cleared, remove the old data in the backup.
if (pageRange.IsClearedPageRange)
{
await pageBlobBackup.ClearPagesAsync(
pageRange.StartOffset, pageRange.EndOffset - pageRange.StartOffset + 1);
}
// If this page range is not cleared, write the new data to the backup.
else
{
byte[] toWrite = new byte[pageRange.EndOffset - pageRange.StartOffset + 1];
await newPageBlobSnap.DownloadRangeToByteArrayAsync(
toWrite, 0, pageRange.StartOffset, pageRange.EndOffset - pageRange.StartOffset + 1);
Stream toWriteStream = new MemoryStream(toWrite);
await pageBlobBackup.WritePagesAsync(toWriteStream, pageRange.StartOffset, null);
}
}
// Snapshot the backup blob and delete the old snapshot from the primary account
CloudPageBlob pageBlobBackupSnapv2 = await pageBlobBackup.CreateSnapshotAsync();
await pageBlobSnap.DeleteAsync();
}
/// <summary>
/// CopyBlobFromBackupToRestore
/// </summary>
/// <returns></returns>
public async Task <string> CopyBlobFromBackupToRestore(DestinationBlobInfo backupBlob)
{
string destinationStorageAccountConnectionString = _config.GetConnectionString("RestoreBlobStorage");
string sourceStorageAccountConnectionString = _config.GetConnectionString("BackupBlobStorage");
bool isServerCopy = bool.Parse(_config.GetSection("AppSettings")["IsServerCopy"]);
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
CloudBlobContainer sourceContainer = sourceBlobClient.GetContainerReference(backupBlob.ContainerName);
bool sourceContainerExists = await sourceContainer.ExistsAsync();
if (sourceContainerExists)
{
CloudBlockBlob sourceBlockBlob = sourceContainer.GetBlockBlobReference(backupBlob.BlobName);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(backupBlob.OrgContainerName);
await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(backupBlob.OrgBlobName);
string copyResult = string.Empty;
if (isServerCopy)
{
string blobToken = GenerateSASBlobToken(sourceBlockBlob);
_logger.LogInformation($"About to server copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = await destinationBlob.StartCopyAsync(new Uri(sourceBlockBlob.Uri.AbsoluteUri + blobToken));
}
else
{
_logger.LogInformation($"About to sync copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = "SYNCCOPY";
await TransferManager.CopyAsync(sourceBlockBlob, destinationBlob, false);
}
_logger.LogInformation($"Copy Scheduled. Source Blob Name: {backupBlob.BlobName} Destination Blob Name: {backupBlob.OrgBlobName} Copy Id {copyResult}.");
return(copyResult);
}
else
{
_logger.LogInformation($"Not able to locate the blob {backupBlob.BlobName} in source storage account.");
}
}
else
{
_logger.LogInformation($"Not able to locate the container {backupBlob.ContainerName} in source storage account.");
}
return(string.Empty);
}
/// <summary>
/// CopyBlobFromSourceToBackup
/// </summary>
/// <returns></returns>
public async Task <DestinationBlobInfo> CopyBlobFromSourceToBackup(IBlobEvent eventData)
{
DestinationBlobInfo destinationBlobInfo = null;
string destinationStorageAccountConnectionString = _config.GetConnectionString("BackupBlobStorage");
string sourceStorageAccountConnectionString = _config.GetConnectionString("SourceBlobStorage");
bool isServerCopy = bool.Parse(_config.GetSection("AppSettings")["IsServerCopy"]);
if (eventData is BlobEvent <CreatedEventData> )
{
// Retrieve the storage account from the connection string.
CloudStorageAccount sourceStorageAccount = CloudStorageAccount.Parse(sourceStorageAccountConnectionString);
CloudBlobClient sourceBlobClient = sourceStorageAccount.CreateCloudBlobClient();
// Retrieve the storage account from the connection string.
CloudStorageAccount destinationStorageAccount = CloudStorageAccount.Parse(destinationStorageAccountConnectionString);
CloudBlobClient destinationBlobClient = destinationStorageAccount.CreateCloudBlobClient();
BlobEvent <CreatedEventData> createdEventData = (BlobEvent <CreatedEventData>)eventData;
string url = createdEventData.data.url;
CloudBlockBlob sourceBlockBlob = new CloudBlockBlob(new Uri(url), sourceBlobClient);
bool sourceBlobExists = await sourceBlockBlob.ExistsAsync();
if (sourceBlobExists)
{
long blobSize = sourceBlockBlob.Properties.Length;
EventDateDetails dateDetails = new EventDateDetails(createdEventData.eventTime);
string destinationContaninerName = dateDetails.year.ToString();
string destinationBlobName = $"wk{dateDetails.WeekNumber}/dy{(int)dateDetails.DayOfWeek}/{sourceBlockBlob.Container.Name}/{sourceBlockBlob.Name}";
CloudBlobContainer destinationContainer = destinationBlobClient.GetContainerReference(destinationContaninerName);
bool result = await destinationContainer.CreateIfNotExistsAsync();
CloudBlockBlob destinationBlob = destinationContainer.GetBlockBlobReference(destinationBlobName);
string copyResult = string.Empty;
if (isServerCopy)
{
string blobToken = GenerateSASBlobToken(sourceBlockBlob);
_logger.LogInformation($"About to server copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = await destinationBlob.StartCopyAsync(new Uri(sourceBlockBlob.Uri.AbsoluteUri + blobToken));
}
else
{
_logger.LogInformation($"About to sync copy {sourceBlockBlob.Name}. Blob size {sourceBlockBlob.Properties.Length} bytes");
copyResult = "SYNCCOPY";
await TransferManager.CopyAsync(sourceBlockBlob, destinationBlob, false);
}
destinationBlobInfo = new DestinationBlobInfo();
destinationBlobInfo.ContainerName = destinationContainer.Name;
destinationBlobInfo.BlobName = destinationBlobName;
destinationBlobInfo.CopyReferenceId = copyResult;
destinationBlobInfo.OrgContainerName = sourceBlockBlob.Container.Name;
destinationBlobInfo.OrgBlobName = sourceBlockBlob.Name;
return(destinationBlobInfo);
}
else
{
_logger.LogInformation($"Not able to locate the block blob in source storage account---Block blob Name {sourceBlockBlob.Name}");
}
}
else
{
_logger.LogInformation($"Input event data is not of Created Event Type.");
}
return(destinationBlobInfo);
}
/// <summary>
/// Get job from jobQueue and transfer data into your Azure blob container.
/// </summary>
private static void TransferToAzure()
{
// Create the container if it doesn't exist yet
CloudBlobClient client = azureClient;
CloudBlobContainer container = client.GetContainerReference(AzureContainerName);
container.CreateIfNotExists();
SingleTransferContext context = new SingleTransferContext();
// Add progress handler
context.ProgressHandler = progressRecorder;
context.ShouldOverwriteCallbackAsync = Program.OverwritePrompt;
while (!jobQueue.IsCompleted)
{
S3ToAzureTransferJob job = null;
try
{
job = jobQueue.Take();
}
catch (InvalidOperationException)
{
// No more jobs to do
}
if (job == null)
{
break;
}
countdownEvent.AddCount();
CloudBlockBlob cloudBlob = container.GetBlockBlobReference(job.Name);
ConsoleWriteLine("Start to transfer {0} to azure.", job.Name);
Task task = null;
try
{
if (!job.ServiceSideCopy)
{
// By default, the sample will download an amazon s3 object into a local file and
// then upload it into Microsoft Azure Stroage with DataMovement library.
task = TransferManager.UploadAsync(job.Source, cloudBlob, null, context);
}
else
{
// When server side copy is used, Azure server will copy the data directly from the URI
// provided and no data is downloaded to local.
task = TransferManager.CopyAsync(new Uri(job.Source), cloudBlob, true, null, context);
}
}
catch (Exception e)
{
ConsoleWriteLine("Error occurs when transferring {0}: {1}", job.Name, e.ToString());
}
if (task != null)
{
task.ContinueWith(t =>
{
if (t.IsFaulted)
{
ConsoleWriteLine("Error occurs when transferring {0}: {1}", job.Name, t.Exception.ToString());
}
else
{
ConsoleWriteLine("Succeed to transfer data to blob {0}", job.Name);
}
// Signal the countdown event when one transfer job finishes.
countdownEvent.Signal();
});
}
else
{
// Signal the countdown event when one transfer job finishes.
countdownEvent.Signal();
}
}
// Signal the countdown event to unblock the main thread when all data are transferred.
countdownEvent.Signal();
}
public async Task CopyAsync(StorageKey target, StorageKey source)
{
var targetblob = GetBlobReference(target);
var sourceBlob = GetBlobReference(source);
await TransferManager.CopyAsync(sourceBlob, targetblob, true);
}