public async Task <bool> ExecuteAsync() { try { AzureStorageUtils blobUtils = new AzureStorageUtils(AccountName, AccountKey, ContainerName); if (FailIfExists && await blobUtils.CheckIfContainerExistsAsync()) { Log.LogError($"Container {ContainerName} already exists in storage account {AccountName}."); return(false); } PublicAccessType permissions = IsPublic ? PublicAccessType.Blob : PublicAccessType.None; StorageUri = await blobUtils.CreateContainerAsync(permissions); ReadOnlyToken = blobUtils.CreateSASToken(ReadOnlyTokenDaysValid, BlobContainerSasPermissions.Read); WriteOnlyToken = blobUtils.CreateSASToken(WriteOnlyTokenDaysValid, BlobContainerSasPermissions.Write); } catch (Exception e) { Log.LogErrorFromException(e); } return(!Log.HasLoggedErrors); }
/// <summary> /// The <see cref="CreateFileSystemAsync"/> operation creates a new /// file system under the specified account. If the file system with the /// same name already exists, the operation fails. /// /// For more information, see /// <see href="https://docs.microsoft.com/rest/api/storageservices/create-container"> /// Create Container</see>. /// </summary> /// <param name="fileSystemName"> /// The name of the file system to create. /// </param> /// <param name="publicAccessType"> /// Optionally specifies whether data in the file system may be accessed /// publicly and the level of access. <see cref="PublicAccessType.FileSystem"/> /// specifies full public read access for file system and path data. /// Clients can enumerate paths within the file system via anonymous /// request, but cannot enumerate file systems within the storage /// account. <see cref="PublicAccessType.Path"/> specifies public /// read access for paths. Path data within this file system can be /// read via anonymous request, but file system data is not available. /// Clients cannot enumerate paths within the file system via anonymous /// request. <see cref="PublicAccessType.None"/> specifies that the /// file system data is private to the account owner. /// </param> /// <param name="metadata"> /// Optional custom metadata to set for this file system. /// </param> /// <param name="cancellationToken"> /// Optional <see cref="CancellationToken"/> to propagate /// notifications that the operation should be cancelled. /// </param> /// <returns> /// A <see cref="Response{FileSystemClient}"/> referencing the /// newly created file system. /// </returns> /// <remarks> /// A <see cref="RequestFailedException"/> will be thrown if /// a failure occurs. /// </remarks> public virtual async Task <Response <DataLakeFileSystemClient> > CreateFileSystemAsync( string fileSystemName, PublicAccessType publicAccessType = PublicAccessType.None, Metadata metadata = default, CancellationToken cancellationToken = default) { DiagnosticScope scope = ClientDiagnostics.CreateScope($"{nameof(DataLakeServiceClient)}.{nameof(CreateFileSystem)}"); try { scope.Start(); DataLakeFileSystemClient fileSystem = GetFileSystemClient(fileSystemName); Response <FileSystemInfo> response = await fileSystem.CreateAsync(publicAccessType, metadata, cancellationToken).ConfigureAwait(false); return(Response.FromValue(fileSystem, response.GetRawResponse())); } catch (Exception ex) { scope.Failed(ex); throw; } finally { scope.Dispose(); } }
public async Task <BlobContentInfo> UploadBlob(Stream objectStream, string blobName, string containerName, IDictionary <string, string> metaData, bool allowAnonymousRead = false, CancellationToken cancellationToken = default) { PublicAccessType publicAccessType = allowAnonymousRead ? PublicAccessType.Blob : PublicAccessType.None; BlobClient client = BlobStorageManager.GetBlobClient( azureStorageOptions: _azureStorageOptions, containerName: containerName, blobName: blobName, publicAccessType: publicAccessType); try { return(await client.UploadAsync( content : objectStream, metadata : metaData, cancellationToken : cancellationToken)); } catch (Exception ex) { throw new BlobServiceException( message: ex.Message, blobName: blobName, containerName: containerName, methodName: nameof(UploadBlob), innerException: ex); } }
public AzureBlobSnapshotStoreSettings( string connectionString, string containerName, TimeSpan connectTimeout, TimeSpan requestTimeout, bool verboseLogging, bool development, bool autoInitialize, PublicAccessType containerPublicAccessType) { if (string.IsNullOrWhiteSpace(containerName)) { throw new ConfigurationException("[AzureBlobSnapshotStore] Container name is null or empty."); } NameValidator.ValidateContainerName(containerName); ConnectionString = connectionString; ContainerName = containerName; RequestTimeout = requestTimeout; ConnectTimeout = connectTimeout; VerboseLogging = verboseLogging; Development = development; AutoInitialize = autoInitialize; ContainerPublicAccessType = containerPublicAccessType; }
public static bool IsPublicAccessType(this string str, out PublicAccessType publicAccessType) { bool result = Enum.TryParse(typeof(PublicAccessType), str, out object @enum); publicAccessType = (PublicAccessType)(@enum.IsNull() ? -1 : @enum); return(result); }
public static async Task CreateAsync(string account, string key, string containerName, bool publicAccess) { BlobServiceClient blobServiceClient = new BlobServiceClient(Client.GetConnectionString(account, key)); BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName); PublicAccessType accessType = publicAccess ? PublicAccessType.BlobContainer : PublicAccessType.None; await container.CreateAsync(accessType); }
public string CreateTestTableAndContainerToStorage(string testContext, PublicAccessType publicAccessType) { var name = "test" + Guid.NewGuid().ToString().Substring(0, 8); TableAndContainerNames.Add(testContext, name); CreateStorageTable(name); CreateStorageContainer(name, publicAccessType); return(name); }
public static void CreateContainer(string connectionString, string containerName, PublicAccessType publicAccessType = PublicAccessType.None) { var serviceClient = new BlobServiceClient(connectionString); var containerClient = serviceClient.GetBlobContainerClient(containerName); var response = containerClient.CreateIfNotExists(publicAccessType); CheckStatus(response); }
public virtual Response <DataLakeFileSystemClient> CreateFileSystem( string fileSystemName, PublicAccessType publicAccessType = PublicAccessType.None, Metadata metadata = default, CancellationToken cancellationToken = default) { DataLakeFileSystemClient fileSystem = GetFileSystemClient(fileSystemName); Response <FileSystemInfo> response = fileSystem.Create(publicAccessType, metadata, cancellationToken); return(Response.FromValue(fileSystem, response.GetRawResponse())); }
public virtual Response <BlobContainerClient> CreateBlobContainer( string containerName, PublicAccessType publicAccessType = PublicAccessType.None, Metadata metadata = default, CancellationToken cancellationToken = default) { BlobContainerClient container = GetBlobContainerClient(containerName); Response <ContainerInfo> response = container.Create(publicAccessType, metadata, cancellationToken); return(new Response <BlobContainerClient>(response.GetRawResponse(), container)); }
public virtual async Task <Response <BlobContainerClient> > CreateBlobContainerAsync( string containerName, PublicAccessType publicAccessType = PublicAccessType.None, Metadata metadata = default, CancellationToken cancellationToken = default) { BlobContainerClient container = GetBlobContainerClient(containerName); Response <ContainerInfo> response = await container.CreateAsync(publicAccessType, metadata, cancellationToken).ConfigureAwait(false); return(new Response <BlobContainerClient>(response.GetRawResponse(), container)); }
public virtual async Task <Response <DataLakeFileSystemClient> > CreateFileSystemAsync( string fileSystemName, PublicAccessType publicAccessType = PublicAccessType.None, Metadata metadata = default, CancellationToken cancellationToken = default) { DataLakeFileSystemClient fileSystem = GetFileSystemClient(fileSystemName); Response <FileSystemInfo> response = await fileSystem.CreateAsync(publicAccessType, metadata, cancellationToken).ConfigureAwait(false); return(Response.FromValue(fileSystem, response.GetRawResponse())); }
/// <summary> /// set the access level of specified container /// </summary> /// <param name="name">container name</param> /// <param name="accessLevel">access level in ("off", "blob", "container")</param> internal async Task SetContainerAcl(long taskId, IStorageBlobManagement localChannel, string name, BlobContainerPublicAccessType accessLevel) { if (!NameUtil.IsValidContainerName(name)) { throw new ArgumentException(String.Format(Resources.InvalidContainerName, name)); } BlobRequestOptions requestOptions = RequestOptions; CloudBlobContainer container = localChannel.GetContainerReference(name); BlobContainerClient containerClient = AzureStorageContainer.GetTrack2BlobContainerClient(container, this.Channel.StorageContext, ClientOptions); // Get container permission and set the public access as input BlobContainerAccessPolicy accessPolicy; try { accessPolicy = containerClient.GetAccessPolicy(null, this.CmdletCancellationToken); } catch (global::Azure.RequestFailedException e) when(e.Status == 404) { throw new ResourceNotFoundException(String.Format(Resources.ContainerNotFound, name)); } PublicAccessType publicAccessType = PublicAccessType.None; switch (accessLevel) { case BlobContainerPublicAccessType.Blob: publicAccessType = PublicAccessType.Blob; break; case BlobContainerPublicAccessType.Container: publicAccessType = PublicAccessType.BlobContainer; break; case BlobContainerPublicAccessType.Off: publicAccessType = PublicAccessType.None; break; default: case BlobContainerPublicAccessType.Unknown: throw new ArgumentOutOfRangeException("Permission"); } await containerClient.SetAccessPolicyAsync(publicAccessType, accessPolicy.SignedIdentifiers, null, this.CmdletCancellationToken).ConfigureAwait(false); if (PassThru) { AzureStorageContainer storageContainer = new AzureStorageContainer(containerClient, Channel.StorageContext); storageContainer.SetTrack2Permission(); OutputStream.WriteObject(taskId, storageContainer); } }
public async Task <bool> SetContainerAccessType(string containerName, PublicAccessType accessType, CancellationToken cancellationToken = default) { try { BlobContainerClient containerClient = BlobStorageManager.GetBlobContainerClient(_azureStorageOptions, containerName.ToLower(), accessType); _ = await containerClient.SetAccessPolicyAsync(accessType : accessType); return(true); } catch { return(false); } }
/// <summary> /// create a new azure container /// </summary> /// <param name="name">container name</param> internal async Task CreateAzureContainer(long taskId, IStorageBlobManagement localChannel, string name, BlobContainerPublicAccessType accesslevel) { if (!NameUtil.IsValidContainerName(name)) { throw new ArgumentException(String.Format(Resources.InvalidContainerName, name)); } BlobRequestOptions requestOptions = RequestOptions; CloudBlobContainer container = localChannel.GetContainerReference(name); BlobContainerClient containerClient = AzureStorageContainer.GetTrack2BlobContainerClient(container, localChannel.StorageContext); PublicAccessType containerPublicAccess = PublicAccessType.None; if (accesslevel == BlobContainerPublicAccessType.Blob) { containerPublicAccess = PublicAccessType.Blob; } else if (accesslevel == BlobContainerPublicAccessType.Container) { containerPublicAccess = PublicAccessType.BlobContainer; } global::Azure.Storage.Blobs.Models.BlobContainerEncryptionScopeOptions encryptionScopeOption = null; if (this.DefaultEncryptionScope != null) { encryptionScopeOption = new global::Azure.Storage.Blobs.Models.BlobContainerEncryptionScopeOptions() { // parameterset can ensure the 2 parameters must be set together. DefaultEncryptionScope = this.DefaultEncryptionScope, PreventEncryptionScopeOverride = this.preventEncryptionScopeOverride.Value }; } Response <BlobContainerInfo> responds = await containerClient.CreateIfNotExistsAsync(containerPublicAccess, null, encryptionScopeOption, CmdletCancellationToken).ConfigureAwait(false); if (responds == null || responds.Value == null) // Container already exist so not created again { throw new ResourceAlreadyExistException(String.Format(Resources.ContainerAlreadyExists, name)); } BlobContainerPermissions permissions = new BlobContainerPermissions() { PublicAccess = accesslevel }; container.FetchAttributes(); WriteCloudContainerObject(taskId, localChannel, container, permissions); }
public AzureBlobFileSystem(string connectionString, string containerName, DirectoryInfo cacheDirectory, PublicAccessType publicAccessType = PublicAccessType.None, IDictionary <string, string> containerMetaData = null, BlobContainerEncryptionScopeOptions blobContainerEncryptionScopeOptions = null) { _connectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString)); _containerName = containerName ?? throw new ArgumentNullException(nameof(containerName)); _cacheDirectory = cacheDirectory ?? throw new ArgumentNullException(nameof(cacheDirectory)); _publicAccessType = publicAccessType; _containerMetaData = containerMetaData; _blobContainerEncryptionScopeOptions = blobContainerEncryptionScopeOptions; if (!_cacheDirectory.Exists) { throw new DirectoryNotFoundException(nameof(cacheDirectory)); } }
public async Task <DisposingFileSystem> GetNewFileSystem( DataLakeServiceClient service = default, string fileSystemName = default, IDictionary <string, string> metadata = default, PublicAccessType publicAccessType = PublicAccessType.None, bool premium = default) { fileSystemName ??= GetNewFileSystemName(); service ??= GetServiceClient_SharedKey(); if (publicAccessType == PublicAccessType.None) { publicAccessType = premium ? PublicAccessType.None : PublicAccessType.FileSystem; } DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(fileSystemName)); // due to a service issue, if the initial container creation request times out, subsequent requests // can return a ContainerAlreadyExists code even though the container doesn't really exist. // we delay until after the service cache timeout and then attempt to create the container one more time. // If this attempt still fails, we mark the test as inconclusive. // TODO Remove this handling after the service bug is fixed https://github.com/Azure/azure-sdk-for-net/issues/9399 try { await RetryAsync( async() => await fileSystem.CreateAsync(metadata: metadata, publicAccessType: publicAccessType), ex => ex.ErrorCode == Constants.Blob.Container.AlreadyExists, retryDelay : TestConstants.DataLakeRetryDelay, retryAttempts : 1); } catch (RequestFailedException storageRequestFailedException) when(storageRequestFailedException.ErrorCode == Constants.Blob.Container.AlreadyExists) { // if we still get this error after retrying, mark the test as inconclusive TestContext.Out.WriteLine( $"{TestContext.CurrentContext.Test.Name} is inconclusive due to hitting " + $"the DataLake service bug described in https://github.com/Azure/azure-sdk-for-net/issues/9399"); Assert.Inconclusive(); // passing the message in Inconclusive call doesn't show up in Console output. } return(new DisposingFileSystem(fileSystem)); }
public async Task <DisposingFileSystem> GetNewFileSystem( DataLakeServiceClient service = default, string fileSystemName = default, IDictionary <string, string> metadata = default, PublicAccessType publicAccessType = PublicAccessType.None, bool premium = default) { fileSystemName ??= GetNewFileSystemName(); service ??= GetServiceClient_SharedKey(); if (publicAccessType == PublicAccessType.None) { publicAccessType = premium ? PublicAccessType.None : PublicAccessType.FileSystem; } DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(fileSystemName)); await fileSystem.CreateAsync(metadata : metadata, publicAccessType : publicAccessType); return(new DisposingFileSystem(fileSystem)); }
public async Task ChangeContainerPermission(string containerName, PublicAccessType publicAccessType = PublicAccessType.None) { try { BlobContainerClient container = blobServiceClient.GetBlobContainerClient(containerName); await container.SetAccessPolicyAsync(publicAccessType); } catch (RequestFailedException e) { Console.WriteLine("HTTP error code {0}: {1}", e.Status, e.ErrorCode); Console.WriteLine(e.Message); throw; } catch (Exception ex) { Console.WriteLine(ex.Message); throw; } }
public async Task <DisposingContainer> GetTestContainerAsync( BlobServiceClient service = default, string containerName = default, IDictionary <string, string> metadata = default, PublicAccessType publicAccessType = PublicAccessType.None, bool premium = default) { containerName ??= GetNewContainerName(); service ??= GetServiceClient_SharedKey(); if (publicAccessType == PublicAccessType.None) { publicAccessType = premium ? PublicAccessType.None : PublicAccessType.BlobContainer; } BlobContainerClient container = InstrumentClient(service.GetBlobContainerClient(containerName)); await container.CreateAsync(metadata : metadata, publicAccessType : publicAccessType); return(new DisposingContainer(container)); }
public async Task <BlobContainerInfo> CreateContainer(string containerName, PublicAccessType publicAccessType = PublicAccessType.None) { try { BlobContainerClient container = new BlobContainerClient(connectionString, containerName); BlobContainerInfo containerInfo = await container.CreateAsync(publicAccessType); return(containerInfo); } catch (RequestFailedException e) { Console.WriteLine(e.Message); Console.ReadLine(); throw; } catch (Exception ex) { Console.WriteLine(ex.Message); throw; } }
public async Task <DisposingFileSystem> GetNewFileSystem( DataLakeServiceClient service = default, string fileSystemName = default, IDictionary <string, string> metadata = default, PublicAccessType publicAccessType = PublicAccessType.None, bool premium = default) { fileSystemName ??= GetNewFileSystemName(); service ??= GetServiceClient_SharedKey(); if (publicAccessType == PublicAccessType.None) { publicAccessType = premium ? PublicAccessType.None : PublicAccessType.FileSystem; } DataLakeFileSystemClient fileSystem = InstrumentClient(service.GetFileSystemClient(fileSystemName)); // due to a service issue, if the initial container creation request times out, subsequent requests // can return a ContainerAlreadyExists code even though the container doesn't really exist. // we delay until after the service cache timeout and then attempt to create the container one more time. // If this attempt fails, we let the exception propagate. // TODO Note this issue will be fixed in the 72 rollout. After that time, this try/catch can be removed. try { await RetryAsync( async() => await fileSystem.CreateAsync(metadata: metadata, publicAccessType: publicAccessType), ex => ex.ErrorCode == Constants.Blob.Container.AlreadyExists, retryDelay : TestConstants.DataLakeRetryDelay, retryAttempts : 1); } catch (RequestFailedException storageRequestFailedException) when(storageRequestFailedException.ErrorCode == Constants.Blob.Container.AlreadyExists) { // if we get this error after retrying once, that means the container really does // exist, since the retry attempt occurred after the cache timeout. // so we just swallow this error and continue on in the test. } return(new DisposingFileSystem(fileSystem)); }
public IDisposable GetNewContainer( out BlobContainerClient container, BlobServiceClient service = default, string containerName = default, IDictionary <string, string> metadata = default, PublicAccessType publicAccessType = PublicAccessType.None, bool premium = default) { containerName ??= GetNewContainerName(); service ??= GetServiceClient_SharedKey(); container = InstrumentClient(service.GetBlobContainerClient(containerName)); if (publicAccessType == PublicAccessType.None) { publicAccessType = premium ? PublicAccessType.None : PublicAccessType.BlobContainer; } return(new DisposingContainer( container, metadata ?? new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase), publicAccessType)); }
private async Task <DataLakeFileSystemClient> GetFileSystemAsync(string fileSystemName, PublicAccessType publicAccessType, CancellationToken cancellationToken) { var runningFileSystemName = GetFileSystem(); var filesystem = _serviceClient.GetFileSystemClient($"{runningFileSystemName}-{fileSystemName}"); try { await filesystem.CreateIfNotExistsAsync(publicAccessType, null, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(null); } } catch (Exception) { _logger.LogWarning("Cannot create file system on file server.", new { FileSystemName = fileSystemName }); } return(filesystem); }
public DisposingContainer(BlobContainerClient container, IDictionary <string, string> metadata, PublicAccessType publicAccessType = default) { container.CreateAsync(metadata: metadata, publicAccessType: publicAccessType).Wait(); ContainerClient = container; }
public async Task <string> CreateContainerAsync(PublicAccessType publicAccess) { await Container.CreateIfNotExistsAsync(publicAccess).ConfigureAwait(false); return(Container.Uri.ToString()); }
public async Task <Uri> GetFileAsync(string fileName, Stream content, BlobContainers containers, PublicAccessType accessType = PublicAccessType.None) { // TODO: implement get via sas https://docs.microsoft.com/en-us/azure/storage/blobs/sas-service-create?tabs=dotnet if (containers == BlobContainers.None) { throw new ArgumentException("Containers can not be NONE"); } throw new NotImplementedException(); }
public async Task <Uri> UploadFileAsync(string fileName, Stream content, BlobContainers containers, PublicAccessType accessType = PublicAccessType.None) { if (containers == BlobContainers.None) { throw new ArgumentException("Containers can not be NONE"); } // TODO: get this from config var container = new BlobContainerClient("UseDevelopmentStorage=true", containers.ToString().ToLowerInvariant()); await container.CreateIfNotExistsAsync(accessType); // Get a reference to a blob BlobClient blob = container.GetBlobClient(fileName); // Open the file and upload its data await blob.UploadAsync(content); // Verify we uploaded some content //BlobProperties properties = await blob.GetPropertiesAsync(); return(blob.Uri); }
/// <summary> /// set the access level of specified container /// </summary> /// <param name="name">container name</param> /// <param name="accessLevel">access level in ("off", "blob", "container")</param> internal async Task SetContainerAcl(long taskId, IStorageBlobManagement localChannel, string name, BlobContainerPublicAccessType accessLevel) { if (!NameUtil.IsValidContainerName(name)) { throw new ArgumentException(String.Format(Resources.InvalidContainerName, name)); } BlobRequestOptions requestOptions = RequestOptions; AccessCondition accessCondition = null; bool needUseTrack2 = false; CloudBlobContainer container = localChannel.GetContainerReference(name); // Get container permission and set the public access as input BlobContainerPermissions permissions = null; try { permissions = localChannel.GetContainerPermissions(container, null, requestOptions, OperationContext); } catch (StorageException e) when(e.IsNotFoundException()) { throw new ResourceNotFoundException(String.Format(Resources.ContainerNotFound, name)); } catch (StorageException e) when(e.IsConflictException()) { // 409 Conflict, might caused by the container has an Stored access policy contains a permission that is not supported by Track1 SDK API veresion, so switch to Track2 SDK needUseTrack2 = true; } if (!needUseTrack2) // Track1 { permissions.PublicAccess = accessLevel; await localChannel.SetContainerPermissionsAsync(container, permissions, accessCondition, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false); if (PassThru) { WriteCloudContainerObject(taskId, localChannel, container, permissions); } } else // Track2 { BlobContainerClient containerClient = AzureStorageContainer.GetTrack2BlobContainerClient(container, this.Channel.StorageContext, ClientOptions); // Get container permission and set the public access as input BlobContainerAccessPolicy accessPolicy; accessPolicy = containerClient.GetAccessPolicy(null, this.CmdletCancellationToken); PublicAccessType publicAccessType = PublicAccessType.None; switch (accessLevel) { case BlobContainerPublicAccessType.Blob: publicAccessType = PublicAccessType.Blob; break; case BlobContainerPublicAccessType.Container: publicAccessType = PublicAccessType.BlobContainer; break; case BlobContainerPublicAccessType.Off: publicAccessType = PublicAccessType.None; break; default: case BlobContainerPublicAccessType.Unknown: throw new ArgumentOutOfRangeException("Permission"); } await containerClient.SetAccessPolicyAsync(publicAccessType, accessPolicy.SignedIdentifiers, null, this.CmdletCancellationToken).ConfigureAwait(false); if (PassThru) { AzureStorageContainer storageContainer = new AzureStorageContainer(container, null); storageContainer.Context = localChannel.StorageContext; storageContainer.SetTrack2Permission(); OutputStream.WriteObject(taskId, storageContainer); } } }
public static async Task <BlobContainerClient> CreateBlobContainerIfNotExistsAsync(this BlobServiceClient blobServiceClient, string blobContainerName, PublicAccessType publicAccessType = PublicAccessType.None) { var blobContainerClient = blobServiceClient.GetBlobContainerClient(blobContainerName); if (await blobContainerClient.ExistsAsync()) { return(blobContainerClient); } return(await blobServiceClient.CreateBlobContainerAsync(blobContainerName, publicAccessType)); }