/// <summary>
/// set the access level of specified container
/// </summary>
/// <param name="name">container name</param>
/// <param name="accessLevel">access level in ("off", "blob", "container")</param>
internal async Task SetContainerAcl(long taskId, IStorageBlobManagement localChannel, string name, BlobContainerPublicAccessType accessLevel)
{
if (!NameUtil.IsValidContainerName(name))
{
throw new ArgumentException(String.Format(Resources.InvalidContainerName, name));
}
BlobContainerPermissions permissions = new BlobContainerPermissions();
permissions.PublicAccess = accessLevel;
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
CloudBlobContainer container = localChannel.GetContainerReference(name);
if (!await localChannel.DoesContainerExistAsync(container, requestOptions, OperationContext, CmdletCancellationToken))
{
throw new ResourceNotFoundException(String.Format(Resources.ContainerNotFound, name));
}
await localChannel.SetContainerPermissionsAsync(container, permissions, accessCondition, requestOptions, OperationContext, CmdletCancellationToken);
if (PassThru)
{
WriteCloudContainerObject(taskId, localChannel, container, permissions);
}
}
internal string CreateAzureContainerStoredAccessPolicy(IStorageBlobManagement localChannel, string containerName, string policyName, DateTime? startTime, DateTime? expiryTime, string permission)
{
if (!NameUtil.IsValidStoredAccessPolicyName(policyName))
{
throw new ArgumentException(String.Format(CultureInfo.CurrentCulture, Resources.InvalidAccessPolicyName, policyName));
}
//Get existing permissions
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
BlobContainerPermissions blobContainerPermissions = localChannel.GetContainerPermissions(container);
//Add new policy
if (blobContainerPermissions.SharedAccessPolicies.Keys.Contains(policyName))
{
throw new ResourceAlreadyExistException(String.Format(CultureInfo.CurrentCulture, Resources.PolicyAlreadyExists, policyName));
}
SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
AccessPolicyHelper.SetupAccessPolicy<SharedAccessBlobPolicy>(policy, startTime, expiryTime, permission);
blobContainerPermissions.SharedAccessPolicies.Add(policyName, policy);
//Set permissions back to container
localChannel.SetContainerPermissions(container, blobContainerPermissions);
return policyName;
}
public void SetDestinationContainer(IStorageBlobManagement channel, string containerName)
{
if (Container == null)
{
if (!NameUtil.IsValidContainerName(containerName))
{
throw new ArgumentException(String.Format(Resources.InvalidContainerName, containerName));
}
Container = channel.GetContainerReference(containerName);
}
}
internal string SetAzureContainerStoredAccessPolicy(IStorageBlobManagement localChannel, string containerName, string policyName, DateTime? startTime, DateTime? expiryTime, string permission, bool noStartTime, bool noExpiryTime)
{
//Get existing permissions
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
BlobContainerPermissions blobContainerPermissions = localChannel.GetContainerPermissions(container);
//Set the policy with new value
if (!blobContainerPermissions.SharedAccessPolicies.Keys.Contains(policyName))
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, Resources.PolicyNotFound, policyName));
}
SharedAccessBlobPolicy policy = blobContainerPermissions.SharedAccessPolicies[policyName];
AccessPolicyHelper.SetupAccessPolicy<SharedAccessBlobPolicy>(policy, startTime, expiryTime, permission, noStartTime, noExpiryTime);
blobContainerPermissions.SharedAccessPolicies[policyName] = policy;
//Set permission back to container
localChannel.SetContainerPermissions(container, blobContainerPermissions);
WriteObject(AccessPolicyHelper.ConstructPolicyOutputPSObject<SharedAccessBlobPolicy>(blobContainerPermissions.SharedAccessPolicies, policyName));
return policyName;
}
/// <summary>
/// remove azure container by container name
/// </summary>
/// <param name="name">container name</param>
internal async Task RemoveAzureContainer(long taskId, IStorageBlobManagement localChannel, string name)
{
if (!NameUtil.IsValidContainerName(name))
{
throw new ArgumentException(String.Format(Resources.InvalidContainerName, name));
}
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
CloudBlobContainer container = localChannel.GetContainerReference(name);
if (!await localChannel.DoesContainerExistAsync(container, requestOptions, OperationContext, CmdletCancellationToken))
{
throw new ResourceNotFoundException(String.Format(Resources.ContainerNotFound, name));
}
string result = string.Empty;
bool removed = false;
if (force || await OutputStream.ConfirmAsync(name))
{
await localChannel.DeleteContainerAsync(container, accessCondition, requestOptions, OperationContext, CmdletCancellationToken);
result = String.Format(Resources.RemoveContainerSuccessfully, name);
removed = true;
}
else
{
result = String.Format(Resources.RemoveContainerCancelled, name);
}
OutputStream.WriteVerbose(taskId, result);
if (PassThru)
{
OutputStream.WriteObject(taskId, removed);
}
}
internal async Task GetAzureContainerStoredAccessPolicyAsync(long taskId, IStorageBlobManagement localChannel, string containerName, string policyName)
{
SharedAccessBlobPolicies shareAccessPolicies = await GetPoliciesAsync(localChannel, containerName, policyName);
if (!String.IsNullOrEmpty(policyName))
{
if (shareAccessPolicies.Keys.Contains(policyName))
{
OutputStream.WriteObject(taskId, AccessPolicyHelper.ConstructPolicyOutputPSObject<SharedAccessBlobPolicy>(shareAccessPolicies, policyName));
}
else
{
throw new ResourceNotFoundException(String.Format(CultureInfo.CurrentCulture, Resources.PolicyNotFound, policyName));
}
}
else
{
foreach (string key in shareAccessPolicies.Keys)
{
OutputStream.WriteObject(taskId, AccessPolicyHelper.ConstructPolicyOutputPSObject<SharedAccessBlobPolicy>(shareAccessPolicies, key));
}
}
}
/// <summary>
/// create a new azure container
/// </summary>
/// <param name="name">container name</param>
internal async Task CreateAzureContainer(long taskId, IStorageBlobManagement localChannel, string name, BlobContainerPublicAccessType accesslevel)
{
if (!NameUtil.IsValidContainerName(name))
{
throw new ArgumentException(String.Format(Resources.InvalidContainerName, name));
}
BlobRequestOptions requestOptions = RequestOptions;
CloudBlobContainer container = localChannel.GetContainerReference(name);
BlobContainerPermissions permissions = new BlobContainerPermissions();
permissions.PublicAccess = accesslevel;
bool created = await localChannel.CreateContainerIfNotExistsAsync(container, permissions.PublicAccess, requestOptions, OperationContext, CmdletCancellationToken);
if (!created)
{
throw new ResourceAlreadyExistException(String.Format(Resources.ContainerAlreadyExists, name));
}
WriteCloudContainerObject(taskId, localChannel, container, permissions);
}
internal bool RemoveAzureContainerStoredAccessPolicy(IStorageBlobManagement localChannel, string containerName, string policyName)
{
bool success = false;
string result = string.Empty;
//Get existing permissions
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
BlobContainerPermissions blobContainerPermissions = localChannel.GetContainerPermissions(container);
//remove the specified policy
if (!blobContainerPermissions.SharedAccessPolicies.Keys.Contains(policyName))
{
throw new ResourceNotFoundException(String.Format(CultureInfo.CurrentCulture, Resources.PolicyNotFound, policyName));
}
if (this.Force || ConfirmRemove(policyName))
{
blobContainerPermissions.SharedAccessPolicies.Remove(policyName);
localChannel.SetContainerPermissions(container, blobContainerPermissions);
success = true;
}
return success;
}
/// <summary>
/// Start copy operation by source ICloudBlob object
/// </summary>
/// <param name="srcICloudBlob">Source ICloudBlob object</param>
/// <param name="destContainer">Destinaion container name</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private void StartCopyBlob(IStorageBlobManagement destChannel, ICloudBlob srcICloudBlob, string destContainer, string destBlobName)
{
CloudBlobContainer container = destChannel.GetContainerReference(destContainer);
Func<long, Task> taskGenerator = (taskId) => StartCopyInTransferManager(taskId, destChannel, srcICloudBlob, container, destBlobName);
RunTask(taskGenerator);
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination CloudBlob object</returns>
private async Task StartCopyAsync(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
NameUtil.ValidateContainerName(destContainer.Name);
NameUtil.ValidateBlobName(destBlobName);
await this.StartCopyFromUri(taskId, destChannel, uri, destContainer.GetBlockBlobReference(destBlobName));
}
/// <summary>
/// Initializes a new instance of the RemoveStorageAzureBlobCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public RemoveStorageAzureBlobCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// execute command
/// </summary>
public override void ExecuteCmdlet()
{
IStorageBlobManagement localChannel = Channel;
bool foundAFolder = false;
DataLakeFileClient fileClient = null;
DataLakeDirectoryClient dirClient = null;
if (ParameterSetName == ManualParameterSet)
{
DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out fileClient, out dirClient);
}
else //BlobParameterSet
{
if (!InputObject.IsDirectory)
{
fileClient = InputObject.File;
}
else
{
dirClient = InputObject.Directory;
foundAFolder = true;
}
}
if (foundAFolder)
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(dirClient), "Update Directory: "))
{
//Set Permission
if (this.Permission != null || this.Owner != null || this.Group != null)
{
//PathAccessControl originPathAccessControl = dirClient.GetAccessControl().Value;
dirClient.SetPermissions(
this.Permission != null ? PathPermissions.ParseSymbolicPermissions(this.Permission) : null,
this.Owner,
this.Group);
}
//Set ACL
if (this.Acl != null)
{
dirClient.SetAccessControlList(PSPathAccessControlEntry.ParseAccessControls(this.Acl));
}
// Set Properties
SetDatalakegen2ItemProperties(dirClient, this.BlobProperties, setToServer: true);
//Set MetaData
SetDatalakegen2ItemMetaData(dirClient, this.BlobMetadata, setToServer: true);
WriteDataLakeGen2Item(localChannel, dirClient);
}
}
else
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(fileClient), "Update File: "))
{
//Set Permission
if (this.Permission != null || this.Owner != null || this.Group != null)
{
fileClient.SetPermissions(
this.Permission != null ? PathPermissions.ParseSymbolicPermissions(this.Permission) : null,
this.Owner,
this.Group);
}
//Set ACL
if (this.Acl != null)
{
fileClient.SetAccessControlList(PSPathAccessControlEntry.ParseAccessControls(this.Acl));
}
// Set Properties
SetDatalakegen2ItemProperties(fileClient, this.BlobProperties, setToServer: true);
//Set MetaData
SetDatalakegen2ItemMetaData(fileClient, this.BlobMetadata, setToServer: true);
WriteDataLakeGen2Item(localChannel, fileClient);
}
}
}
private CloudBlob GetDestBlob(IStorageBlobManagement destChannel, string destContainerName, string destBlobName, BlobType blobType)
{
NameUtil.ValidateContainerName(destContainerName);
NameUtil.ValidateBlobName(destBlobName);
CloudBlobContainer container = destChannel.GetContainerReference(destContainerName);
CloudBlob destBlob = null;
if (BlobType.PageBlob == blobType)
{
destBlob = container.GetPageBlobReference(destBlobName);
}
else if (BlobType.BlockBlob == blobType)
{
destBlob = container.GetBlockBlobReference(destBlobName);
}
else if (BlobType.AppendBlob == blobType)
{
destBlob = container.GetAppendBlobReference(destBlobName);
}
else
{
throw new ArgumentException(String.Format(Resources.InvalidBlobType, blobType, destBlobName));
}
return destBlob;
}
public override void ExecuteCmdlet()
{
IStorageBlobManagement destChannel = GetDestinationChannel();
IStorageBlobManagement srcChannel = Channel;
string target = string.Empty;
Action copyAction = null;
switch (ParameterSetName)
{
case ContainerNameParameterSet:
copyAction = () => StartCopyBlob(srcChannel, destChannel, SrcContainer, SrcBlob, DestContainer, DestBlob);
target = SrcBlob;
break;
case UriParameterSet:
copyAction = () => StartCopyBlob(destChannel, AbsoluteUri, DestContainer, DestBlob, (Context != null ? GetCmdletStorageContext(Context) : null));
target = AbsoluteUri;
break;
case BlobParameterSet:
if (CloudBlob is InvalidCloudBlob || UseTrack2SDK())
{
copyAction = () => StartCopyBlob(destChannel, BlobBaseClient, DestContainer, DestBlob);
target = BlobBaseClient.Name;
}
else
{
copyAction = () => StartCopyBlob(destChannel, CloudBlob, DestContainer, DestBlob);
target = CloudBlob.Name;
}
break;
case ContainerParameterSet:
copyAction = () => StartCopyBlob(srcChannel, destChannel, CloudBlobContainer.Name, SrcBlob, DestContainer, DestBlob);
target = SrcBlob;
break;
case BlobToBlobParameterSet:
if (CloudBlob is InvalidCloudBlob || UseTrack2SDK())
{
BlobBaseClient destBlobClient = AzureStorageBlob.GetTrack2BlobClient(DestCloudBlob, destChannel.StorageContext, ClientOptions);
copyAction = () => StartCopyBlob(destChannel, BlobBaseClient, destBlobClient);
target = BlobBaseClient.Name;
}
else
{
copyAction = () => StartCopyBlob(destChannel, CloudBlob, DestCloudBlob);
target = CloudBlob.Name;
}
break;
case ShareNameParameterSet:
copyAction = () => StartCopyFromFile(
this.GetFileChannel(),
destChannel,
this.SrcShareName,
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
target = SrcFilePath;
break;
case ShareParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcShare.GetRootDirectoryReference(),
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
target = SrcFilePath;
break;
case DirParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcDir,
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
target = SrcFilePath;
break;
case FileParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcFile,
this.DestContainer,
this.DestBlob);
target = SrcFile.Name;
break;
case FileToBlobParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcFile,
this.DestCloudBlob);
target = SrcFile.Name;
break;
}
if (copyAction != null && ShouldProcess(target, VerbsCommon.Copy))
{
copyAction();
}
}
public FakeSetAzureBlobContentCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Initializes a new instance of the RemoveStorageAzureBlobCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public RemoveStorageAzureBlobCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// remove the azure blob
/// </summary>
/// <param name="blob">Cloudblob object</param>
/// <param name="isValidBlob">whether the Cloudblob parameter is validated</param>
/// <returns>true if the blob is removed successfully, false if user cancel the remove operation</returns>
internal async Task RemoveAzureBlob(long taskId, IStorageBlobManagement localChannel, CloudBlob blob, bool isValidBlob)
{
if (!isValidBlob)
{
ValidatePipelineCloudBlob(blob);
}
ValidateBlobType(blob);
DeleteSnapshotsOption deleteSnapshotsOption = DeleteSnapshotsOption.None;
bool retryDeleteSnapshot = false;
if (IsSnapshot(blob))
{
if (deleteSnapshot)
{
throw new ArgumentException(String.Format(Resources.CannotDeleteSnapshotForSnapshot, blob.Name, blob.SnapshotTime));
}
}
else
{
if (deleteSnapshot)
{
deleteSnapshotsOption = DeleteSnapshotsOption.DeleteSnapshotsOnly;
}
else if (force)
{
deleteSnapshotsOption = DeleteSnapshotsOption.IncludeSnapshots;
}
else
{
retryDeleteSnapshot = true;
}
}
try
{
await DeleteCloudAsync(taskId, localChannel, blob, deleteSnapshotsOption).ConfigureAwait(false);
retryDeleteSnapshot = false;
}
catch (StorageException e)
{
if (e.IsConflictException() && retryDeleteSnapshot)
{
//If x-ms-delete-snapshots is not specified on the request and the blob has associated snapshots, the Blob service returns status code 409 (Conflict).
retryDeleteSnapshot = true;
}
else
{
throw;
}
}
if (retryDeleteSnapshot)
{
string message = string.Format(Resources.ConfirmRemoveBlobWithSnapshot, blob.Name, blob.Container.Name);
if (await OutputStream.ConfirmAsync(message).ConfigureAwait(false))
{
deleteSnapshotsOption = DeleteSnapshotsOption.IncludeSnapshots;
await DeleteCloudAsync(taskId, localChannel, blob, deleteSnapshotsOption).ConfigureAwait(false);
}
else
{
string result = String.Format(Resources.RemoveBlobCancelled, blob.Name, blob.Container.Name);
OutputStream.WriteVerbose(taskId, result);
}
}
}
/// <summary>
/// Initializes a new instance of the GetAzureStorageContainerStoredAccessPolicyCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public GetAzureStorageContainerStoredAccessPolicyCommand(IStorageBlobManagement channel)
: base(channel)
{
}
/// <summary>
/// remove the azure blob with Track2 SDK
/// </summary>
/// <param name="blob">BlobBaseClient object</param>
/// <param name="isValidBlob">whether the Cloudblob parameter is validated</param>
/// <returns>true if the blob is removed successfully, false if user cancel the remove operation</returns>
internal async Task RemoveAzureBlobTrack2(long taskId, IStorageBlobManagement localChannel, BlobBaseClient blob, bool isValidBlob)
{
if (!isValidBlob)
{
ValidatePipelineCloudBlobTrack2(blob);
}
Track2Models.DeleteSnapshotsOption deleteSnapshotsOption = Track2Models.DeleteSnapshotsOption.None;
bool retryDeleteSnapshot = false;
if (Util.GetSnapshotTimeFromBlobUri(blob.Uri) != null)
{
if (deleteSnapshot)
{
throw new ArgumentException(String.Format(Resources.CannotDeleteSnapshotForSnapshot, blob.Name, Util.GetSnapshotTimeFromBlobUri(blob.Uri)));
}
}
else if (!string.IsNullOrEmpty(Util.GetVersionIdFromBlobUri(blob.Uri)))
{
if (deleteSnapshot)
{
throw new ArgumentException(String.Format(Resources.CannotDeleteSnapshotForBlobVersion, blob.Name, Util.GetVersionIdFromBlobUri(blob.Uri)));
}
}
else
{
if (deleteSnapshot)
{
deleteSnapshotsOption = Track2Models.DeleteSnapshotsOption.OnlySnapshots;
}
else if (force)
{
deleteSnapshotsOption = Track2Models.DeleteSnapshotsOption.IncludeSnapshots;
}
else
{
retryDeleteSnapshot = true;
}
}
try
{
await DeleteCloudAsyncTrack2(taskId, localChannel, blob, deleteSnapshotsOption).ConfigureAwait(false);
retryDeleteSnapshot = false;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 409)
{
if (retryDeleteSnapshot)
{
//If x-ms-delete-snapshots is not specified on the request and the blob has associated snapshots, the Blob service returns status code 409 (Conflict).
retryDeleteSnapshot = true;
}
else
{
throw;
}
}
if (retryDeleteSnapshot)
{
string message = string.Format(Resources.ConfirmRemoveBlobWithSnapshot, blob.Name, blob.BlobContainerName);
if (await OutputStream.ConfirmAsync(message).ConfigureAwait(false))
{
deleteSnapshotsOption = Track2Models.DeleteSnapshotsOption.IncludeSnapshots;
await DeleteCloudAsyncTrack2(taskId, localChannel, blob, deleteSnapshotsOption).ConfigureAwait(false);
}
else
{
string result = String.Format(Resources.RemoveBlobCancelled, blob.Name, blob.BlobContainerName);
OutputStream.WriteVerbose(taskId, result);
}
}
}
/// <summary>
/// Initializes a new instance of the NewAzureStorageContainerCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public NewAzureStorageContainerCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Initializes a new instance of the SetAzureStorageContainerAclCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public SetAzureStorageContainerAclCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Initializes a new instance of the SetAzDataLakeGen2AclRecursiveCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public SetAzDataLakeGen2AclRecursiveCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Start copy operation by container name and blob name
/// </summary>
/// <param name="srcContainerName">Source container name</param>
/// <param name="srcBlobName">Source blob name</param>
/// <param name="destContainer">Destinaion container name</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private void StartCopyBlob(IStorageBlobManagement SrcChannel, IStorageBlobManagement destChannel, string srcContainerName, string srcBlobName, string destContainerName, string destBlobName)
{
ValidateBlobName(srcBlobName);
ValidateContainerName(srcContainerName);
ValidateContainerName(destContainerName);
if (string.IsNullOrEmpty(destBlobName))
{
destBlobName = srcBlobName;
}
ValidateBlobName(destBlobName);
AccessCondition accessCondition = null;
BlobRequestOptions options = RequestOptions;
CloudBlobContainer container = SrcChannel.GetContainerReference(srcContainerName);
ICloudBlob blob = SrcChannel.GetBlobReferenceFromServer(container, srcBlobName, accessCondition, options, OperationContext);
if (blob == null)
{
throw new ResourceNotFoundException(String.Format(Resources.BlobNotFound, srcBlobName, srcContainerName));
}
CloudBlobContainer destContainer = destChannel.GetContainerReference(destContainerName);
Func<long, Task> taskGenerator = (taskId) => StartCopyInTransferManager(taskId, destChannel, blob, destContainer, destBlobName);
RunTask(taskGenerator);
}
/// <summary>
/// remove azure blob
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="blobName">blob name</param>
/// <returns>true if the blob is removed successfully, false if user cancel the remove operation</returns>
internal async Task RemoveAzureBlob(long taskId, IStorageBlobManagement localChannel, string containerName, string blobName)
{
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
await RemoveAzureBlob(taskId, localChannel, container, blobName).ConfigureAwait(false);
}
/// <summary>
/// Get DestinationBlob with specified copy id
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <param name="copyId">Current CopyId</param>
/// <returns>Destination ICloudBlob object</returns>
private ICloudBlob GetDestinationBlobWithCopyId(IStorageBlobManagement destChannel, CloudBlobContainer container, string blobName)
{
AccessCondition accessCondition = null;
BlobRequestOptions options = RequestOptions;
ICloudBlob blob = destChannel.GetBlobReferenceFromServer(container, blobName, accessCondition, options, OperationContext);
return blob;
}
/// <summary>
/// Initializes a new instance of the NewAzureStorageBlobSasCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public NewAzureStorageBlobSasTokenCommand(IStorageBlobManagement channel)
{
Channel = channel;
EnableMultiThread = false;
}
private async Task StartCopyFromFile(long taskId, IStorageBlobManagement destChannel, CloudFile srcFile, CloudBlockBlob destBlob)
{
await this.StartCopyFromUri(taskId, destChannel, srcFile.GenerateUriWithCredentials(), destBlob);
}
public override void ExecuteCmdlet()
{
IStorageBlobManagement destChannel = GetDestinationChannel();
IStorageBlobManagement srcChannel = Channel;
switch (ParameterSetName)
{
case ContainerNameParameterSet:
StartCopyBlob(srcChannel, destChannel, SrcContainer, SrcBlob, DestContainer, DestBlob);
break;
case UriParameterSet:
StartCopyBlob(destChannel, AbsoluteUri, DestContainer, DestBlob, Context);
break;
case BlobParameterSet:
StartCopyBlob(destChannel, CloudBlob, DestContainer, DestBlob);
break;
case ContainerParameterSet:
StartCopyBlob(srcChannel, destChannel, CloudBlobContainer.Name, SrcBlob, DestContainer, DestBlob);
break;
case BlobToBlobParameterSet:
StartCopyBlob(destChannel, CloudBlob, DestCloudBlob);
break;
case ShareNameParameterSet:
this.StartCopyFromFile(
this.GetFileChannel(),
destChannel,
this.SrcShareName,
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
break;
case ShareParameterSet:
this.StartCopyFromFile(
destChannel,
this.SrcShare.GetRootDirectoryReference(),
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
break;
case DirParameterSet:
StartCopyFromFile(
destChannel,
this.SrcDir,
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
break;
case FileParameterSet:
StartCopyFromFile(
destChannel,
this.SrcFile,
this.DestContainer,
this.DestBlob);
break;
case FileToBlobParameterSet:
StartCopyFromFile(
destChannel,
this.SrcFile,
this.DestCloudBlob);
break;
}
}
/// <summary>
/// Initializes a new instance of the NewAzureStorageBlobSasCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public NewAzureStorageBlobSasTokenCommand(IStorageBlobManagement channel)
{
Channel = channel;
EnableMultiThread = false;
}
/// <summary>
/// Initializes a new instance of the RemoveAzDataLakeGen2ItemCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public RemoveAzDataLakeGen2ItemCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Initializes a new instance of the GetAzureStorageBlobContentCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public GetAzureStorageBlobContentCommand(IStorageBlobManagement channel)
{
Channel = channel;
fileNameResolver = new BlobToFileSystemNameResolver(() => NameUtil.WindowsMaxFileNameLength);
}
/// <summary>
/// Initializes a new instance of the GetAzureStorageBlobCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public GetAzureStorageBlobCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Start copy using transfer mangager by source CloudBlob object
/// </summary>
/// <param name="blob">Source CloudBlob object</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination CloudBlob object</returns>
private async Task StartCopyAsync(long taskId, IStorageBlobManagement destChannel, CloudBlob sourceBlob, CloudBlob destBlob)
{
NameUtil.ValidateBlobName(sourceBlob.Name);
NameUtil.ValidateContainerName(destBlob.Container.Name);
NameUtil.ValidateBlobName(destBlob.Name);
await this.StartCopyFromBlob(taskId, destChannel, sourceBlob, destBlob);
}
/// <summary>
/// Stop copy operation by CloudBlob object
/// </summary>
/// <param name="blob">CloudBlob object</param>
/// <param name="copyId">Copy id</param>
private async Task StopCopyBlob(long taskId, IStorageBlobManagement localChannel, CloudBlob blob, string copyId, bool fetchCopyIdFromBlob = false)
{
ValidateBlobType(blob);
AccessCondition accessCondition = null;
BlobRequestOptions abortRequestOption = RequestOptions ?? new BlobRequestOptions();
//Set no retry to resolve the 409 conflict exception
abortRequestOption.RetryPolicy = new NoRetry();
if (null == blob)
{
throw new ArgumentException(String.Format(Resources.ObjectCannotBeNull, typeof(CloudBlob).Name));
}
string specifiedCopyId = copyId;
if (string.IsNullOrEmpty(specifiedCopyId) && fetchCopyIdFromBlob)
{
if (blob.CopyState != null)
{
specifiedCopyId = blob.CopyState.CopyId;
}
}
string abortCopyId = string.Empty;
if (string.IsNullOrEmpty(specifiedCopyId) || Force)
{
//Make sure we use the correct copy id to abort
//Use default retry policy for FetchBlobAttributes
BlobRequestOptions options = RequestOptions;
await localChannel.FetchBlobAttributesAsync(blob, accessCondition, options, OperationContext, CmdletCancellationToken);
if (blob.CopyState == null || String.IsNullOrEmpty(blob.CopyState.CopyId))
{
ArgumentException e = new ArgumentException(String.Format(Resources.CopyTaskNotFound, blob.Name, blob.Container.Name));
OutputStream.WriteError(taskId, e);
}
else
{
abortCopyId = blob.CopyState.CopyId;
}
if (!Force)
{
string confirmation = String.Format(Resources.ConfirmAbortCopyOperation, blob.Name, blob.Container.Name, abortCopyId);
if (!await OutputStream.ConfirmAsync(confirmation))
{
string cancelMessage = String.Format(Resources.StopCopyOperationCancelled, blob.Name, blob.Container.Name);
OutputStream.WriteVerbose(taskId, cancelMessage);
}
}
}
else
{
abortCopyId = specifiedCopyId;
}
await localChannel.AbortCopyAsync(blob, abortCopyId, accessCondition, abortRequestOption, OperationContext, CmdletCancellationToken);
string message = String.Format(Resources.StopCopyBlobSuccessfully, blob.Name, blob.Container.Name);
OutputStream.WriteObject(taskId, message);
}
/// <summary>
/// Initializes a new instance of the NewAzureStorageBlobSasCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public NewAzureStorageBlobSasTokenCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Initializes a new instance of the SetAzureBlobContentCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public SetAzureBlobContentCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// Start copy operation by source and destination ICloudBlob object
/// </summary>
/// <param name="srcICloudBlob">Source ICloudBlob object</param>
/// <param name="destICloudBlob">Destination ICloudBlob object</param>
/// <returns>Destination ICloudBlob object</returns>
private void StartCopyBlob(IStorageBlobManagement destChannel, ICloudBlob srcICloudBlob, ICloudBlob destICloudBlob)
{
Func<long, Task> taskGenerator = (taskId) => StartCopyInTransferManager(taskId, destChannel, srcICloudBlob, destICloudBlob.Container, destICloudBlob.Name);
RunTask(taskGenerator);
}
private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, Uri srcUri, BlobBaseClient destBlob)
{
bool destExist = true;
Track2Models.BlobType? destBlobType = Util.GetBlobType(destBlob);
Track2Models.BlobProperties properties = null;
try
{
properties = (await destBlob.GetPropertiesAsync(this.BlobRequestConditions, cancellationToken: this.CmdletCancellationToken).ConfigureAwait(false)).Value;
destBlobType = properties.BlobType;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 404)
{
destExist = false;
}
if (destBlobType != null)
{
ValidateBlobTier(Util.convertBlobType_Track2ToTrack1(destBlobType), null, standardBlobTier, rehydratePriority);
}
if (!destExist || this.ConfirmOverwrite(srcUri.AbsoluteUri.ToString(), destBlob.Uri.ToString()))
{
Track2Models.BlobCopyFromUriOptions options = new Track2Models.BlobCopyFromUriOptions();
// The Blob Type and Blob Tier must match, since already checked before
if (standardBlobTier != null || rehydratePriority != null)
{
options.AccessTier = Util.ConvertAccessTier_Track1ToTrack2(standardBlobTier);
options.RehydratePriority = Util.ConvertRehydratePriority_Track1ToTrack2(rehydratePriority);
}
options.SourceConditions = this.BlobRequestConditions;
BlockBlobClient srcBlockblob = new BlockBlobClient(srcUri, ClientOptions);
Track2Models.BlobProperties srcProperties = srcBlockblob.GetProperties(cancellationToken: this.CmdletCancellationToken).Value;
//Prepare progress handler
string activity = String.Format("Copy Blob {0} to {1}", srcBlockblob.Name, destBlob.Name);
string status = "Prepare to Copy Blob";
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
IProgress <long> progressHandler = new Progress <long>((finishedBytes) =>
{
if (pr != null)
{
// Size of the source file might be 0, when it is, directly treat the progress as 100 percent.
pr.PercentComplete = 0 == srcProperties.ContentLength ? 100 : (int)(finishedBytes * 100 / srcProperties.ContentLength);
pr.StatusDescription = string.Format("Percent: {0}%.", pr.PercentComplete);
this.OutputStream.WriteProgress(pr);
}
});
switch (destBlobType)
{
case Track2Models.BlobType.Block:
BlockBlobClient destBlockBlob = (BlockBlobClient)Util.GetTrack2BlobClientWithType(destBlob, Channel.StorageContext, Track2Models.BlobType.Block, ClientOptions);
Track2Models.CommitBlockListOptions commitBlockListOptions = new Track2Models.CommitBlockListOptions();
commitBlockListOptions.HttpHeaders = new Track2Models.BlobHttpHeaders();
commitBlockListOptions.HttpHeaders.ContentType = srcProperties.ContentType;
commitBlockListOptions.HttpHeaders.ContentHash = srcProperties.ContentHash;
commitBlockListOptions.HttpHeaders.ContentEncoding = srcProperties.ContentEncoding;
commitBlockListOptions.HttpHeaders.ContentLanguage = srcProperties.ContentLanguage;
commitBlockListOptions.HttpHeaders.ContentDisposition = srcProperties.ContentDisposition;
commitBlockListOptions.Metadata = srcProperties.Metadata;
try
{
commitBlockListOptions.Tags = srcBlockblob.GetTags(cancellationToken: this.CmdletCancellationToken).Value.Tags;
}
catch (global::Azure.RequestFailedException e) when(e.Status == 403 || e.Status == 404 || e.Status == 401)
{
if (!this.Force && !OutputStream.ConfirmAsync("Can't get source blob Tags, so source blob tags won't be copied to dest blob. Do you want to continue the blob copy?").Result)
{
return;
}
}
long blockLength = GetBlockLength(srcProperties.ContentLength);
string[] blockIDs = GetBlockIDs(srcProperties.ContentLength, blockLength, destBlockBlob.Name);
long copyoffset = 0;
progressHandler.Report(copyoffset);
foreach (string id in blockIDs)
{
long blocksize = blockLength;
if (copyoffset + blocksize > srcProperties.ContentLength)
{
blocksize = srcProperties.ContentLength - copyoffset;
}
destBlockBlob.StageBlockFromUri(srcUri, id, new global::Azure.HttpRange(copyoffset, blocksize), null, null, null, cancellationToken: this.CmdletCancellationToken);
copyoffset += blocksize;
progressHandler.Report(copyoffset);
}
destBlockBlob.CommitBlockList(blockIDs, commitBlockListOptions, this.CmdletCancellationToken);
break;
case Track2Models.BlobType.Page:
case Track2Models.BlobType.Append:
default:
throw new ArgumentException(string.Format("The cmdlet currently only support souce blob and destination blob are both block blob. The dest blob type is {0}.", destBlobType));
}
OutputStream.WriteObject(taskId, new AzureStorageBlob(destBlob, destChannel.StorageContext, null, options: ClientOptions));
}
}
/// <summary>
/// Start copy operation by source uri
/// </summary>
/// <param name="srcICloudBlob">Source uri</param>
/// <param name="destContainer">Destinaion container name</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private void StartCopyBlob(IStorageBlobManagement destChannel, string srcUri, string destContainer, string destBlobName, AzureStorageContext context)
{
if (context != null)
{
Uri sourceUri = new Uri(srcUri);
Uri contextUri = new Uri(context.BlobEndPoint);
if (sourceUri.Host.ToLower() == contextUri.Host.ToLower())
{
CloudBlobClient blobClient = context.StorageAccount.CreateCloudBlobClient();
ICloudBlob blobReference = blobClient.GetBlobReferenceFromServer(sourceUri);
StartCopyBlob(destChannel, blobReference, destContainer, destBlobName);
}
else
{
WriteWarning(String.Format(Resources.StartCopySourceContextMismatch, srcUri, context.BlobEndPoint));
}
}
CloudBlobContainer container = destChannel.GetContainerReference(destContainer);
Func<long, Task> taskGenerator = (taskId) => StartCopyInTransferManager(taskId, destChannel, new Uri(srcUri), container, destBlobName);
RunTask(taskGenerator);
}
/// <summary>
/// Stop copy operation by name
/// </summary>
/// <param name="containerName">Container name</param>
/// <param name="blobName">Blob name</param>
/// <param name="copyId">copy id</param>
private async Task StopCopyBlob(long taskId, IStorageBlobManagement localChannel, string containerName, string blobName, string copyId)
{
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
await StopCopyBlob(taskId, localChannel, container, blobName, copyId).ConfigureAwait(false);
}
/// <summary>
/// Start copy using transfer mangager by source uri
/// </summary>
/// <param name="uri">source uri</param>
/// <param name="destContainer">Destination CloudBlobContainer object</param>
/// <param name="destBlobName">Destination blob name</param>
/// <returns>Destination ICloudBlob object</returns>
private async Task StartCopyInTransferManager(long taskId, IStorageBlobManagement destChannel, Uri uri, CloudBlobContainer destContainer, string destBlobName)
{
ValidateContainerName(destContainer.Name);
ValidateBlobName(destBlobName);
Dictionary<string, string> BlobPath = new Dictionary<string, string>()
{
{"Container", destContainer.Name},
{"Blob", destBlobName}
};
DataMovementUserData data = new DataMovementUserData()
{
Data = BlobPath,
TaskId = taskId,
Channel = destChannel,
Record = null
};
transferManager.QueueBlobStartCopy(uri, destContainer, destBlobName, null, OnCopyTaskFinish, data);
await data.taskSource.Task;
}
/// <summary>
/// get the CloudBlobContainer object by name if container exists
/// </summary>
/// <param name="containerName">container name</param>
/// <returns>return CloudBlobContianer object if specified container exists, otherwise throw an exception</returns>
internal async Task<CloudBlobContainer> GetCloudBlobContainerByName(IStorageBlobManagement localChannel, string containerName, bool skipCheckExists = false)
{
if (!NameUtil.IsValidContainerName(containerName))
{
throw new ArgumentException(String.Format(Resources.InvalidContainerName, containerName));
}
BlobRequestOptions requestOptions = RequestOptions;
CloudBlobContainer container = localChannel.GetContainerReference(containerName);
if (!skipCheckExists && container.ServiceClient.Credentials.IsSharedKey
&& !await localChannel.DoesContainerExistAsync(container, requestOptions, OperationContext, CmdletCancellationToken))
{
throw new ArgumentException(String.Format(Resources.ContainerNotFound, containerName));
}
return container;
}
/// <summary>
/// Initializes a new instance of the GetAzureStorageBlobContentCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public GetAzureStorageBlobContentCommand(IStorageBlobManagement channel)
{
Channel = channel;
fileNameResolver = new AzureToFileSystemFileNameResolver(() => NameUtil.WindowsMaxFileNameLength);
}
/// <summary>
/// list blobs by blob prefix and container name
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="prefix">blob preifx</param>
/// <returns>An enumerable collection of IListBlobItem</returns>
internal async Task ListBlobsByPrefix(long taskId, IStorageBlobManagement localChannel, string containerName, string prefix, Func<CloudBlob, bool> blobFilter = null)
{
CloudBlobContainer container = await GetCloudBlobContainerByName(localChannel, containerName);
BlobRequestOptions requestOptions = RequestOptions;
bool useFlatBlobListing = true;
BlobListingDetails details = BlobListingDetails.Snapshots | BlobListingDetails.Metadata | BlobListingDetails.Copy;
int listCount = InternalMaxCount;
int MaxListCount = 5000;
int requestCount = MaxListCount;
int realListCount = 0;
BlobContinuationToken continuationToken = ContinuationToken;
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
BlobResultSegment blobResult = await localChannel.ListBlobsSegmentedAsync(container, prefix, useFlatBlobListing,
details, requestCount, continuationToken, requestOptions, OperationContext, CmdletCancellationToken);
foreach (IListBlobItem blobItem in blobResult.Results)
{
CloudBlob blob = blobItem as CloudBlob;
if (blob == null)
{
continue;
}
if (blobFilter == null || blobFilter(blob))
{
WriteCloudBlobObject(taskId, localChannel, blob, blobResult.ContinuationToken);
realListCount++;
}
}
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
continuationToken = blobResult.ContinuationToken;
}
while (listCount > 0 && continuationToken != null);
}
/// <summary>
/// Initializes a new instance of the GetAzureStorageBlobCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public GetAzureStorageBlobCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
public override void ExecuteCmdlet()
{
IStorageBlobManagement destChannel = GetDestinationChannel();
IStorageBlobManagement srcChannel = Channel;
string target = string.Empty;
Action copyAction = null;
switch (ParameterSetName)
{
case ContainerNameParameterSet:
copyAction = () => StartCopyBlob(srcChannel, destChannel, SrcContainer, SrcBlob, DestContainer, DestBlob);
target = SrcBlob;
break;
case UriParameterSet:
copyAction = () => StartCopyBlob(destChannel, AbsoluteUri, DestContainer, DestBlob, Context);
target = AbsoluteUri;
break;
case BlobParameterSet:
copyAction = () => StartCopyBlob(destChannel, CloudBlob, DestContainer, DestBlob);
target = CloudBlob.Name;
break;
case ContainerParameterSet:
copyAction = () => StartCopyBlob(srcChannel, destChannel, CloudBlobContainer.Name, SrcBlob, DestContainer, DestBlob);
target = SrcBlob;
break;
case BlobToBlobParameterSet:
copyAction = () => StartCopyBlob(destChannel, CloudBlob, DestCloudBlob);
target = CloudBlob.Name;
break;
case ShareNameParameterSet:
copyAction = () => StartCopyFromFile(
this.GetFileChannel(),
destChannel,
this.SrcShareName,
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
target = SrcFilePath;
break;
case ShareParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcShare.GetRootDirectoryReference(),
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
target = SrcFilePath;
break;
case DirParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcDir,
this.SrcFilePath,
this.DestContainer,
this.DestBlob);
target = SrcFilePath;
break;
case FileParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcFile,
this.DestContainer,
this.DestBlob);
target = SrcFile.Name;
break;
case FileToBlobParameterSet:
copyAction = () => StartCopyFromFile(
destChannel,
this.SrcFile,
this.DestCloudBlob);
target = SrcFile.Name;
break;
}
if (copyAction != null && ShouldProcess(target, VerbsCommon.Copy))
{
copyAction();
}
}
/// <summary>
/// list blobs by blob name and container name
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="blobName">blob name pattern</param>
/// <returns>An enumerable collection of IListBlobItem</returns>
internal async Task ListBlobsByName(long taskId, IStorageBlobManagement localChannel, string containerName, string blobName)
{
CloudBlobContainer container = null;
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
string prefix = string.Empty;
if (String.IsNullOrEmpty(blobName) || WildcardPattern.ContainsWildcardCharacters(blobName))
{
container = await GetCloudBlobContainerByName(localChannel, containerName);
prefix = NameUtil.GetNonWildcardPrefix(blobName);
WildcardOptions options = WildcardOptions.IgnoreCase | WildcardOptions.Compiled;
WildcardPattern wildcard = null;
if (!String.IsNullOrEmpty(blobName))
{
wildcard = new WildcardPattern(blobName, options);
}
Func<CloudBlob, bool> blobFilter = (blob) => wildcard == null || wildcard.IsMatch(blob.Name);
await ListBlobsByPrefix(taskId, localChannel, containerName, prefix, blobFilter);
}
else
{
container = await GetCloudBlobContainerByName(localChannel, containerName, true);
if (!NameUtil.IsValidBlobName(blobName))
{
throw new ArgumentException(String.Format(Resources.InvalidBlobName, blobName));
}
CloudBlob blob = await localChannel.GetBlobReferenceFromServerAsync(container, blobName, accessCondition, requestOptions, OperationContext, CmdletCancellationToken);
if (null == blob)
{
throw new ResourceNotFoundException(String.Format(Resources.BlobNotFound, blobName, containerName));
}
else
{
WriteCloudBlobObject(taskId, localChannel, blob);
}
}
}
/// <summary>
/// Start copy operation by source and destination ICloudBlob object
/// </summary>
/// <param name="srcICloudBlob">Source ICloudBlob object</param>
/// <param name="destICloudBlob">Destination ICloudBlob object</param>
/// <returns>Destination ICloudBlob object</returns>
private void StartCopyBlob(IStorageBlobManagement destChannel, ICloudBlob srcICloudBlob, ICloudBlob destICloudBlob)
{
Func <long, Task> taskGenerator = (taskId) => StartCopyInTransferManager(taskId, destChannel, srcICloudBlob, destICloudBlob.Container, destICloudBlob.Name);
RunTask(taskGenerator);
}
public FakeRemoveAzureContainerCommand(IStorageBlobManagement channel)
: base(channel)
{
}
/// <summary>
/// list blobs by blob name and container name
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="blobName">blob name pattern</param>
/// <returns>An enumerable collection of IListBlobItem</returns>
internal async Task ListBlobsByName(long taskId, IStorageBlobManagement localChannel, string containerName, string blobName, bool includeDeleted = false, bool includeVersion = false)
{
CloudBlobContainer container = null;
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
string prefix = string.Empty;
if (String.IsNullOrEmpty(blobName) || WildcardPattern.ContainsWildcardCharacters(blobName) || includeDeleted)
{
container = await GetCloudBlobContainerByName(localChannel, containerName).ConfigureAwait(false);
prefix = NameUtil.GetNonWildcardPrefix(blobName);
WildcardOptions options = WildcardOptions.IgnoreCase | WildcardOptions.Compiled;
WildcardPattern wildcard = null;
if (!String.IsNullOrEmpty(blobName))
{
wildcard = new WildcardPattern(blobName, options);
}
Func <string, bool> blobFilter = (blobNameToFilte) => wildcard == null || wildcard.IsMatch(blobNameToFilte);
await ListBlobsByPrefix(taskId, localChannel, containerName, prefix, blobFilter, includeDeleted, IncludeVersion).ConfigureAwait(false);
}
else
{
container = await GetCloudBlobContainerByName(localChannel, containerName, true).ConfigureAwait(false);
BlobContainerClient track2container = AzureStorageContainer.GetTrack2BlobContainerClient(container, localChannel.StorageContext, ClientOptions);
if (!NameUtil.IsValidBlobName(blobName))
{
throw new ArgumentException(String.Format(Resources.InvalidBlobName, blobName));
}
BlobBaseClient blobClient = null;
if (UseTrack2Sdk()) // User Track2 SDK
{
blobClient = Util.GetTrack2BlobClient(track2container, blobName, localChannel.StorageContext, this.VersionId, false, this.SnapshotTime is null ? null : this.SnapshotTime.Value.ToString("o"), ClientOptions);
global::Azure.Storage.Blobs.Models.BlobProperties blobProperties;
try
{
blobProperties = blobClient.GetProperties(BlobRequestConditions, cancellationToken: CmdletCancellationToken);
}
catch (global::Azure.RequestFailedException e) when(e.Status == 404)
{
throw new ResourceNotFoundException(String.Format(Resources.BlobNotFound, blobName, containerName));
}
blobClient = Util.GetTrack2BlobClient(track2container, blobName, localChannel.StorageContext, this.VersionId, blobProperties.IsLatestVersion, this.SnapshotTime is null ? null : this.SnapshotTime.Value.ToString("o"), ClientOptions, blobProperties.BlobType);
AzureStorageBlob outputBlob = new AzureStorageBlob(blobClient, localChannel.StorageContext, blobProperties, ClientOptions);
if (this.IncludeTag.IsPresent)
{
IDictionary <string, string> tags = (await blobClient.GetTagsAsync(null, this.CmdletCancellationToken).ConfigureAwait(false)).Value.Tags;
if (tags != null)
{
outputBlob.Tags = tags.ToHashtable();
outputBlob.TagCount = tags.Count;
}
}
OutputStream.WriteObject(taskId, outputBlob);
}
else // Use Track1 SDK
{
CloudBlob blob = await localChannel.GetBlobReferenceFromServerAsync(container, blobName, this.SnapshotTime, accessCondition, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
if (null == blob)
{
throw new ResourceNotFoundException(String.Format(Resources.BlobNotFound, blobName, containerName));
}
else
{
OutputStream.WriteObject(taskId, new AzureStorageBlob(blob, localChannel.StorageContext, ClientOptions));
}
}
}
}
/// <summary>
/// Async get container permission
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="taskId">Task id</param>
/// <param name="context">Azure storage context</param>
/// <returns></returns>
internal async Task GetContainerPermission(long taskId, IStorageBlobManagement localChannel, CloudBlobContainer container, BlobContinuationToken continuationToken)
{
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
BlobContainerPermissions permissions = null;
try
{
permissions = await localChannel.GetContainerPermissionsAsync(container, accessCondition,
requestOptions, OperationContext, CmdletCancellationToken);
}
catch (StorageException e)
{
if (!e.IsNotFoundException())
{
throw;
}
//404 Not found means we don't have permission to query the Permission of the specified container.
}
WriteCloudContainerObject(taskId, localChannel, container, permissions, continuationToken);
}
/// <summary>
/// Initializes a new instance of the RemoveAzureStorageContainerStoredAccessPolicyCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public RemoveAzureStorageContainerStoredAccessPolicyCommand(IStorageBlobManagement channel)
: base(channel)
{
EnableMultiThread = false;
}
/// <summary>
/// Initializes a new instance of the NewAzureStorageContainerCommand class.
/// </summary>
/// <param name="channel">IStorageBlobManagement channel</param>
public NewAzureStorageContainerCommand(IStorageBlobManagement channel)
{
Channel = channel;
}
/// <summary>
/// list blobs by blob prefix and container name
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="prefix">blob preifx</param>
/// <returns>An enumerable collection of IListBlobItem</returns>
internal async Task ListBlobsByPrefix(long taskId, IStorageBlobManagement localChannel, string containerName, string prefix, Func <string, bool> blobFilter = null, bool includeDeleted = false, bool includeVersion = false)
{
CloudBlobContainer container = await GetCloudBlobContainerByName(localChannel, containerName).ConfigureAwait(false);
BlobContainerClient track2container = AzureStorageContainer.GetTrack2BlobContainerClient(container, localChannel.StorageContext, ClientOptions);
int listCount = InternalMaxCount;
int MaxListCount = 5000;
int requestCount = MaxListCount;
int realListCount = 0;
BlobContinuationToken continuationToken = ContinuationToken;
string track2ContinuationToken = this.ContinuationToken is null ? null : this.ContinuationToken.NextMarker;
if (UseTrack2Sdk()) // For new feature only available on Track2 SDK, need list with Track2 SDK.
{
BlobTraits blobTraits = BlobTraits.Metadata | BlobTraits.CopyStatus; // | BlobTraits.Tags;
BlobStates blobStates = BlobStates.Snapshots;
if (includeDeleted)
{
blobStates = blobStates | BlobStates.Deleted;
}
if (includeVersion)
{
blobStates = blobStates | BlobStates.Version;
}
if (IncludeTag.IsPresent)
{
blobTraits = blobTraits | BlobTraits.Tags;
}
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
IEnumerator <Page <BlobItem> > enumerator = track2container.GetBlobs(blobTraits, blobStates, prefix, CmdletCancellationToken)
.AsPages(track2ContinuationToken, requestCount)
.GetEnumerator();
Page <BlobItem> page;
enumerator.MoveNext();
page = enumerator.Current;
foreach (BlobItem item in page.Values)
{
if (blobFilter == null || blobFilter(item.Name))
{
OutputStream.WriteObject(taskId, GetAzureStorageBlob(item, track2container, localChannel.StorageContext, page.ContinuationToken, ClientOptions));
}
realListCount++;
}
track2ContinuationToken = page.ContinuationToken;
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
} while (listCount > 0 && !string.IsNullOrEmpty(track2ContinuationToken));
}
else
{
BlobRequestOptions requestOptions = RequestOptions;
bool useFlatBlobListing = true;
BlobListingDetails details = BlobListingDetails.Snapshots | BlobListingDetails.Metadata | BlobListingDetails.Copy;
if (includeDeleted)
{
details = details | BlobListingDetails.Deleted;
}
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
BlobResultSegment blobResult = await localChannel.ListBlobsSegmentedAsync(container, prefix, useFlatBlobListing,
details, requestCount, continuationToken, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
foreach (IListBlobItem blobItem in blobResult.Results)
{
CloudBlob blob = blobItem as CloudBlob;
if (blob == null)
{
continue;
}
if (blobFilter == null || blobFilter(blob.Name))
{
WriteCloudBlobObject(taskId, localChannel, blob, blobResult.ContinuationToken);
realListCount++;
}
}
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
continuationToken = blobResult.ContinuationToken;
}while (listCount > 0 && continuationToken != null);
}
}
/// <summary>
/// Set up the Channel object for Destination container and blob
/// </summary>
internal void SetUpDestinationChannel()
{
//If destChannel exits, reuse it.
//If desContext exits, use it.
//If Channl object exists, use it.
//Otherwise, create a new channel.
if (destChannel == null)
{
if (DestContext == null)
{
if (Channel != null)
{
destChannel = Channel;
}
else
{
destChannel = base.CreateChannel();
}
}
else
{
destChannel = CreateChannel(DestContext.StorageAccount);
}
}
}
/// <summary>
/// upload file to azure blob
/// </summary>
/// <param name="taskId">Task id</param>
/// <param name="filePath">local file path</param>
/// <param name="blob">destination azure blob object</param>
internal virtual async Task Upload2Blob(long taskId, IStorageBlobManagement localChannel, string filePath, StorageBlob.CloudBlob blob)
{
string activity = String.Format(Resources.SendAzureBlobActivity, filePath, blob.Name, blob.Container.Name);
string status = Resources.PrepareUploadingBlob;
ProgressRecord pr = new ProgressRecord(OutputStream.GetProgressId(taskId), activity, status);
FileInfo fileInfo = new FileInfo(filePath);
DataMovementUserData data = new DataMovementUserData()
{
Data = blob,
TaskId = taskId,
Channel = localChannel,
Record = pr,
TotalSize = fileInfo.Length
};
SingleTransferContext transferContext = this.GetTransferContext(data);
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
transferContext.SetAttributesCallbackAsync = async(destination) =>
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
CloudBlob destBlob = destination as CloudBlob;
SetBlobProperties(destBlob, this.BlobProperties);
SetBlobMeta(destBlob, this.BlobMetadata);
};
await DataMovementTransferHelper.DoTransfer(() =>
{
return(this.TransferManager.UploadAsync(filePath,
blob,
null,
transferContext,
this.CmdletCancellationToken));
},
data.Record,
this.OutputStream).ConfigureAwait(false);
if (this.pageBlobTier != null || this.standardBlobTier != null)
{
await this.SetBlobTier(localChannel, blob, pageBlobTier, standardBlobTier).ConfigureAwait(false);
}
try
{
await localChannel.FetchBlobAttributesAsync(
blob,
AccessCondition.GenerateEmptyCondition(),
this.RequestOptions,
this.OperationContext,
this.CmdletCancellationToken).ConfigureAwait(false);
}
catch (StorageException e)
{
//Handle the limited read permission, and handle the upload with write only permission
if (!e.IsNotFoundException() && !e.IsForbiddenException())
{
throw;
}
}
WriteCloudBlobObject(data.TaskId, localChannel, blob);
}
