/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination CloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
NameUtil.ValidateContainerName(destContainer.Name);
NameUtil.ValidateBlobName(destBlobName);
Dictionary <string, string> BlobPath = new Dictionary <string, string>()
{
{ "Container", destContainer.Name },
{ "Blob", destBlobName }
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
TransferJob startCopyJob = new TransferJob(
new TransferLocation(uri),
new TransferLocation(destContainer.GetBlockBlobReference(destBlobName)),
TransferMethod.AsyncCopyInAzureStorageWithoutMonitor);
await this.EnqueueStartCopyJob(startCopyJob, data);
}
private void OnCopyTaskFinish(object userData, string copyId, Exception e)
{
DataMovementUserData data = userData as DataMovementUserData;
IStorageBlobManagement destChannel = data.Channel;
if (data != null && e == null)
{
OutputStream.WriteVerbose(data.TaskId, copyId);
Dictionary <string, string> destBlobPath = data.Data as Dictionary <string, string>;
if (destBlobPath != null)
{
OutputStream.WriteVerbose(data.TaskId, String.Format(Resources.CopyDestinationBlobPending, destBlobPath["Blob"], destBlobPath["Container"], copyId));
CloudBlobContainer container = destChannel.GetContainerReference(destBlobPath["Container"]);
try
{
ICloudBlob destBlob = GetDestinationBlobWithCopyId(destChannel, container, destBlobPath["Blob"]);
if (destBlob != null)
{
WriteICloudBlobObject(data.TaskId, destChannel, destBlob);
}
}
catch (Exception readException)
{
e = readException;
}
}
}
OnDMJobFinish(userData, e);
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, CloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr,
TotalSize = blob.Properties.Length
};
await DataMovementTransferHelper.DoTransfer(() =>
{
return(this.TransferManager.DownloadAsync(blob, filePath,
new DownloadOptions()
{
DisableContentMD5Validation = !this.checkMd5
},
this.GetTransferContext(data),
this.CmdletCancellationToken));
},
data.Record,
this.OutputStream);
this.WriteCloudBlobObject(data.TaskId, data.Channel, blob);
}
/// <summary>
/// Start copy using transfer mangager by source ICloudBlob object
/// </summary>
/// <param name="blob">Source ICloudBlob object</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement DestChannel, ICloudBlob sourceBlob, ICloudBlob destBlob)
{
ValidateBlobName(sourceBlob.Name);
ValidateContainerName(destBlob.Container.Name);
ValidateBlobName(destBlob.Name);
Dictionary <string, string> BlobPath = new Dictionary <string, string>()
{
{ "Container", destBlob.Container.Name },
{ "Blob", destBlob.Name }
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = DestChannel,
Record = null
};
BlobStartCopyJob startCopyJob = new BlobStartCopyJob()
{
SourceBlob = sourceBlob,
DestBlob = destBlob
};
await this.EnqueueStartCopyJob(startCopyJob, data);
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, ICloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr
};
BlobDownloadJob downloadJob = new BlobDownloadJob()
{
SourceBlob = blob,
DestPath = filePath,
};
BlobRequestOptions requestOptions = downloadJob.BlobRequestOptions;
requestOptions.DisableContentMD5Validation = !checkMd5;
downloadJob.BlobRequestOptions = requestOptions;
await this.RunTransferJob(downloadJob, data);
this.WriteICloudBlobObject(data.TaskId, data.Channel, blob);
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
ValidateContainerName(destContainer.Name);
ValidateBlobName(destBlobName);
Dictionary <string, string> BlobPath = new Dictionary <string, string>()
{
{ "Container", destContainer.Name },
{ "Blob", destBlobName }
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
BlobStartCopyJob startCopyJob = new BlobStartCopyJob()
{
SourceUri = uri,
DestBlob = destContainer.GetBlockBlobReference(destBlobName)
};
await this.EnqueueStartCopyJob(startCopyJob, data);
}
/// <summary>
/// Start copy using transfer mangager by source ICloudBlob object
/// </summary>
/// <param name="blob">Source ICloudBlob object</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement DestChannel, ICloudBlob blob, CloudBlobContainer destContainer, string destBlobName)
{
if (string.IsNullOrEmpty(destBlobName))
{
destBlobName = blob.Name;
}
ValidateBlobName(blob.Name);
ValidateContainerName(destContainer.Name);
ValidateBlobName(destBlobName);
Dictionary <string, string> BlobPath = new Dictionary <string, string>()
{
{ "Container", destContainer.Name },
{ "Blob", destBlobName }
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = DestChannel,
Record = null
};
transferManager.QueueBlobStartCopy(blob, destContainer, destBlobName, null, OnCopyTaskFinish, data);
await data.taskSource.Task;
}
/// <summary>
/// On Task run successfully
/// </summary>
/// <param name="data">User data</param>
protected override void OnTaskSuccessful(DataMovementUserData data)
{
ICloudBlob blob = data.Data as ICloudBlob;
if (blob != null)
{
WriteICloudBlobObject(data.TaskId, data.Channel, blob);
}
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task <bool> DownloadBlob(long taskId, IStorageBlobManagement localChannel, ICloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr
};
transferManager.QueueDownload(blob, filePath, checkMd5, OnDMJobStart, OnDMJobProgress, OnDMJobFinish, data);
return(await data.taskSource.Task);
}
/// <summary>
/// upload file to azure blob
/// </summary>
/// <param name="taskId">Task id</param>
/// <param name="filePath">local file path</param>
/// <param name="blob">destination azure blob object</param>
internal virtual async Task Upload2Blob(long taskId, IStorageBlobManagement localChannel, string filePath, CloudBlob blob)
{
string activity = String.Format(Resources.SendAzureBlobActivity, filePath, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
FileInfo fileInfo = new FileInfo(filePath);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr,
TotalSize = fileInfo.Length
};
SingleTransferContext transferContext = this.GetTransferContext(data);
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
transferContext.SetAttributesCallbackAsync = async(source, destination) =>
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
CloudBlob destBlob = destination as CloudBlob;
SetAzureBlobContentCommand.SetBlobProperties(destBlob, this.BlobProperties);
SetAzureBlobContentCommand.SetBlobMeta(destBlob, this.BlobMetadata);
};
await DataMovementTransferHelper.DoTransfer(() =>
{
return(this.TransferManager.UploadAsync(filePath,
blob,
null,
transferContext,
this.CmdletCancellationToken));
},
data.Record,
this.OutputStream).ConfigureAwait(false);
// Set blob permission with umask, since create Blob API still not support them
SetBlobPermissionWithUMask((CloudBlockBlob)blob, this.Permission, this.Umask);
WriteDataLakeGen2Item(localChannel, fileSystem.GetFileClient(blob.Name), taskId: data.TaskId);
}
protected async Task EnqueueStartCopyJob(BlobStartCopyJob startCopyJob, DataMovementUserData userData)
{
await this.RunTransferJob(startCopyJob, userData);
this.OutputStream.WriteVerbose(userData.TaskId, startCopyJob.CopyId);
Dictionary <string, string> destBlobPath = userData.Data as Dictionary <string, string>;
if (destBlobPath != null)
{
var destChannel = userData.Channel;
this.OutputStream.WriteVerbose(userData.TaskId, String.Format(Resources.CopyDestinationBlobPending, destBlobPath["Blob"], destBlobPath["Container"], startCopyJob.CopyId));
CloudBlobContainer container = destChannel.GetContainerReference(destBlobPath["Container"]);
ICloudBlob destBlob = this.GetDestinationBlobWithCopyId(destChannel, container, destBlobPath["Blob"]);
if (destBlob != null)
{
this.WriteICloudBlobObject(userData.TaskId, destChannel, destBlob);
}
}
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, CloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr,
TotalSize = blob.Properties.Length
};
await DataMovementTransferHelper.DoTransfer(() =>
{
return this.TransferManager.DownloadAsync(blob, filePath,
new DownloadOptions()
{
DisableContentMD5Validation = !this.checkMd5
},
this.GetTransferContext(data),
this.CmdletCancellationToken);
},
data.Record,
this.OutputStream);
this.WriteCloudBlobObject(data.TaskId, data.Channel, blob);
}
/// <summary>
/// On Task run successfully
/// </summary>
/// <param name="data">User data</param>
protected override void OnTaskSuccessful(DataMovementUserData data)
{
ICloudBlob blob = data.Data as ICloudBlob;
if (blob != null)
{
WriteICloudBlobObject(data.TaskId, data.Channel, blob);
}
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task<bool> DownloadBlob(long taskId, IStorageBlobManagement localChannel, ICloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr
};
transferManager.QueueDownload(blob, filePath, checkMd5, OnDMJobStart, OnDMJobProgress, OnDMJobFinish, data);
return await data.taskSource.Task;
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
ValidateContainerName(destContainer.Name);
ValidateBlobName(destBlobName);
Dictionary<string, string> BlobPath = new Dictionary<string, string>()
{
{"Container", destContainer.Name},
{"Blob", destBlobName}
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
BlobStartCopyJob startCopyJob = new BlobStartCopyJob()
{
SourceUri = uri,
DestBlob = destContainer.GetBlockBlobReference(destBlobName)
};
await this.EnqueueStartCopyJob(startCopyJob, data);
}
/// <summary>
/// Start copy using transfer mangager by source ICloudBlob object
/// </summary>
/// <param name="blob">Source ICloudBlob object</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement DestChannel, ICloudBlob sourceBlob, ICloudBlob destBlob)
{
ValidateBlobName(sourceBlob.Name);
ValidateContainerName(destBlob.Container.Name);
ValidateBlobName(destBlob.Name);
Dictionary<string, string> BlobPath = new Dictionary<string, string>()
{
{"Container", destBlob.Container.Name},
{"Blob", destBlob.Name}
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = DestChannel,
Record = null
};
BlobStartCopyJob startCopyJob = new BlobStartCopyJob()
{
SourceBlob = sourceBlob,
DestBlob = destBlob
};
await this.EnqueueStartCopyJob(startCopyJob, data);
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination CloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
NameUtil.ValidateContainerName(destContainer.Name);
NameUtil.ValidateBlobName(destBlobName);
Dictionary<string, string> BlobPath = new Dictionary<string, string>()
{
{"Container", destContainer.Name},
{"Blob", destBlobName}
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
TransferJob startCopyJob = new TransferJob(
new TransferLocation(uri),
new TransferLocation(destContainer.GetBlockBlobReference(destBlobName)),
TransferMethod.AsyncCopyInAzureStorageWithoutMonitor);
await this.EnqueueStartCopyJob(startCopyJob, data);
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
ValidateContainerName(destContainer.Name);
ValidateBlobName(destBlobName);
Dictionary<string, string> BlobPath = new Dictionary<string, string>()
{
{"Container", destContainer.Name},
{"Blob", destBlobName}
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
transferManager.QueueBlobStartCopy(uri, destContainer, destBlobName, null, OnCopyTaskFinish, data);
await data.taskSource.Task;
}
protected void EnqueueStartCopyJob(BlobStartCopyJob startCopyJob, DataMovementUserData userData)
{
startCopyJob.FinishEvent += (eventSource, eventArgs) =>
{
BlobStartCopyJob tempJob = eventArgs.Job as BlobStartCopyJob;
this.OnCopyTaskFinish(userData, tempJob.CopyId, eventArgs.Exception);
};
this.SetRequestOptionsInTransferJob(startCopyJob);
startCopyJob.OverwritePromptCallback = ConfirmOverwrite;
transferManager.EnqueueJob(startCopyJob, CmdletCancellationToken);
}
private void OnCopyTaskFinish(DataMovementUserData userData, string copyId, Exception e)
{
IStorageBlobManagement destChannel = userData.Channel;
if (userData != null && e == null)
{
OutputStream.WriteVerbose(userData.TaskId, copyId);
Dictionary<string, string> destBlobPath = userData.Data as Dictionary<string, string>;
if (destBlobPath != null)
{
OutputStream.WriteVerbose(userData.TaskId, String.Format(Resources.CopyDestinationBlobPending, destBlobPath["Blob"], destBlobPath["Container"], copyId));
CloudBlobContainer container = destChannel.GetContainerReference(destBlobPath["Container"]);
try
{
ICloudBlob destBlob = GetDestinationBlobWithCopyId(destChannel, container, destBlobPath["Blob"]);
if (destBlob != null)
{
WriteICloudBlobObject(userData.TaskId, destChannel, destBlob);
}
}
catch (Exception readException)
{
e = readException;
}
}
}
OnDMJobFinish(userData, e);
}
protected async Task EnqueueStartCopyJob(BlobStartCopyJob startCopyJob, DataMovementUserData userData)
{
await this.RunTransferJob(startCopyJob, userData);
this.OutputStream.WriteVerbose(userData.TaskId, startCopyJob.CopyId);
Dictionary<string, string> destBlobPath = userData.Data as Dictionary<string, string>;
if (destBlobPath != null)
{
var destChannel = userData.Channel;
this.OutputStream.WriteVerbose(userData.TaskId, String.Format(Resources.CopyDestinationBlobPending, destBlobPath["Blob"], destBlobPath["Container"], startCopyJob.CopyId));
CloudBlobContainer container = destChannel.GetContainerReference(destBlobPath["Container"]);
ICloudBlob destBlob = this.GetDestinationBlobWithCopyId(destChannel, container, destBlobPath["Blob"]);
if (destBlob != null)
{
this.WriteICloudBlobObject(userData.TaskId, destChannel, destBlob);
}
}
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task<bool> DownloadBlob(long taskId, IStorageBlobManagement localChannel, ICloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr
};
BlobDownloadJob downloadJob = new BlobDownloadJob()
{
SourceBlob = blob,
DestPath = filePath,
};
BlobRequestOptions requestOptions = downloadJob.BlobRequestOptions;
requestOptions.DisableContentMD5Validation = !checkMd5;
downloadJob.BlobRequestOptions = requestOptions;
this.EnqueueTransferJob(downloadJob, data);
return await data.taskSource.Task;
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, CloudBlob blob, string filePath)
{
string activity = String.Format(Resources.ReceiveAzureBlobActivity, blob.Name, filePath);
string status = Resources.PrepareDownloadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr
};
TransferJob downloadJob = new TransferJob(
new TransferLocation(blob),
new TransferLocation(filePath),
TransferMethod.SyncCopy);
BlobRequestOptions requestOptions = downloadJob.Source.RequestOptions as BlobRequestOptions;
if (null == requestOptions)
{
requestOptions = new BlobRequestOptions();
}
requestOptions.DisableContentMD5Validation = !checkMd5;
downloadJob.Source.RequestOptions = requestOptions;
await this.RunTransferJob(downloadJob, data);
this.WriteCloudBlobObject(data.TaskId, data.Channel, blob);
}
