/// <summary> /// Does the actual work of initialization. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> async Task InitInternalAsync(CancellationToken cancellationToken) { logger.Verbose("InitInternalAsync()"); await client.CreateIfNotExistsAsync(cancellationToken : cancellationToken); init = false; }
private async Task PushToDataFabric(string routingKey, Stream stream, CancellationToken stoppingToken) { stoppingToken.ThrowIfCancellationRequested(); //Create a unique name for the blob var containerName = $"{_config.ContainerNamePrefix}-{routingKey}"; var containerClient = new BlobContainerClient(_config.StorageConnectionString, containerName); await containerClient.CreateIfNotExistsAsync(cancellationToken : stoppingToken); stoppingToken.ThrowIfCancellationRequested(); var blobName = $"{_config.BlobNamePrefix}-{DateTime.UtcNow.ToString(string.IsNullOrEmpty(_config.BlobNameRollingTimeFormat) ? "yyyyMMdd" : _config.BlobNameRollingTimeFormat)}"; var blobClient = containerClient.GetAppendBlobClient(blobName); var blobInfo = await blobClient.CreateIfNotExistsAsync(cancellationToken : stoppingToken); stoppingToken.ThrowIfCancellationRequested(); _logger.LogInformation("Appending to Blob storage as blob:\n\t {0}\n", blobClient.Uri); await blobClient.AppendBlockAsync(stream, null, new AppendBlobRequestConditions { IfMatch = blobInfo?.Value.ETag }, cancellationToken : stoppingToken); }
private async Task WriteToDataFabric(string routeKey, Stream stream, CancellationToken cancellationToken) { if (_diagnostic != null && _diagnostic.IsEnabled() && _diagnostic.IsEnabled(DiagnosticOperations.MessageProcess)) { _diagnostic.Write(DiagnosticOperations.DataFabricWrite, _config); } //Create a unique name for the blob var containerName = $"{_config.ContainerNamePrefix}-{routeKey}"; var containerClient = new BlobContainerClient(_config.StorageConnectionString, containerName); await containerClient.CreateIfNotExistsAsync(cancellationToken : cancellationToken); var blobName = $"{_config.BlobNamePrefix}-{DateTime.UtcNow.ToString(string.IsNullOrEmpty(_config.BlobNameRollingTimeFormat) ? "yyyyMMdd" : _config.BlobNameRollingTimeFormat)}"; var blobClient = containerClient.GetAppendBlobClient(blobName); var blobInfo = await blobClient.CreateIfNotExistsAsync(cancellationToken : cancellationToken); Console.WriteLine("Appending to Blob storage as blob:\n\t {0}\n", blobClient.Uri); await blobClient.AppendBlockAsync(stream, null, new AppendBlobRequestConditions { IfMatch = blobInfo?.Value.ETag }, cancellationToken : cancellationToken); }
private async Task <BlobContainerClient> GetBlobContainerClientAsync(AzureBlobEgressProviderOptions options, CancellationToken token) { BlobServiceClient serviceClient; if (!string.IsNullOrWhiteSpace(options.SharedAccessSignature)) { var serviceUriBuilder = new UriBuilder(options.AccountUri) { Query = options.SharedAccessSignature }; serviceClient = new BlobServiceClient(serviceUriBuilder.Uri); } else if (!string.IsNullOrEmpty(options.AccountKey)) { // Remove Query in case SAS token was specified Uri accountUri = GetAccountUri(options, out string accountName); StorageSharedKeyCredential credential = new StorageSharedKeyCredential( accountName, options.AccountKey); serviceClient = new BlobServiceClient(accountUri, credential); } else { throw CreateException(Strings.ErrorMessage_EgressMissingSasOrKey); } BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(options.ContainerName); await containerClient.CreateIfNotExistsAsync(cancellationToken : token); return(containerClient); }
private async Task <Uri> GetServiceSasUriForBlob() { var containerClient = new BlobContainerClient(_settings.ConnectionString, _settings.ContainerName); try { await containerClient.CreateIfNotExistsAsync(); } catch (Azure.RequestFailedException ex) { if (ex.Status != 409) { throw; } } if (!containerClient.CanGenerateSasUri) { throw new NotSupportedException("cannot generate sas token"); } BlobSasBuilder sasBuilder = new BlobSasBuilder() { BlobContainerName = containerClient.Name, Resource = "c" }; sasBuilder.ExpiresOn = DateTimeOffset.UtcNow.AddMinutes(_settings.ExpiredMunites); sasBuilder.SetPermissions(BlobSasPermissions.Create | BlobSasPermissions.Write | BlobSasPermissions.Read); return(containerClient.GenerateSasUri(sasBuilder)); }
/// <summary> Initialization function for this storage provider. </summary> private async Task Init(CancellationToken ct) { var stopWatch = Stopwatch.StartNew(); try { this.logger.LogInformation((int)AzureProviderErrorCode.AzureTableProvider_InitProvider, $"AzureTableGrainStorage initializing: {this.options.ToString()}"); this.logger.LogInformation((int)AzureProviderErrorCode.AzureTableProvider_ParamConnectionString, "AzureTableGrainStorage is using DataConnectionString: {0}", ConfigUtilities.RedactConnectionStringInfo(this.options.ConnectionString)); this.jsonSettings = OrleansJsonSerializer.UpdateSerializerSettings(OrleansJsonSerializer.GetDefaultSerializerSettings(this.services), this.options.UseFullAssemblyNames, this.options.IndentJson, this.options.TypeNameHandling); this.options.ConfigureJsonSerializerSettings?.Invoke(this.jsonSettings); var client = this.options.ServiceUri != null ? new BlobServiceClient(this.options.ServiceUri, this.options.TokenCredential) : new BlobServiceClient(this.options.ConnectionString); container = client.GetBlobContainerClient(this.options.ContainerName); await container.CreateIfNotExistsAsync().ConfigureAwait(false); stopWatch.Stop(); this.logger.LogInformation((int)AzureProviderErrorCode.AzureBlobProvider_InitProvider, $"Initializing provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} took {stopWatch.ElapsedMilliseconds} Milliseconds."); } catch (Exception ex) { stopWatch.Stop(); this.logger.LogError((int)ErrorCode.Provider_ErrorFromInit, $"Initialization failed for provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} in {stopWatch.ElapsedMilliseconds} Milliseconds.", ex); throw; } }
private async Task WriteStateAndCreateContainerIfNotExists(string grainType, GrainReference grainId, IGrainState grainState, byte[] contents, string mimeType, BlobClient blob) { try { var conditions = grainState.ETag == null ? new BlobRequestConditions { IfNoneMatch = new ETag("*") } : new BlobRequestConditions { IfMatch = new ETag(grainState.ETag) }; using var stream = new MemoryStream(contents); var result = await DoOptimisticUpdate(() => blob.UploadAsync(stream, conditions: conditions, httpHeaders: new BlobHttpHeaders { ContentType = mimeType }), blob, grainState.ETag).ConfigureAwait(false); grainState.ETag = result.Value.ETag.ToString(); grainState.RecordExists = true; } catch (RequestFailedException exception) when(exception.IsContainerNotFound()) { // if the container does not exist, create it, and make another attempt if (this.logger.IsEnabled(LogLevel.Trace)) { this.logger.Trace((int)AzureProviderErrorCode.AzureBlobProvider_ContainerNotFound, "Creating container: GrainType={0} Grainid={1} ETag={2} to BlobName={3} in Container={4}", grainType, grainId, grainState.ETag, blob.Name, container.Name); } await container.CreateIfNotExistsAsync().ConfigureAwait(false); await WriteStateAndCreateContainerIfNotExists(grainType, grainId, grainState, contents, mimeType, blob).ConfigureAwait(false); } }
/// <summary> Initialization function for this storage provider. </summary> private async Task Init(CancellationToken ct) { var stopWatch = Stopwatch.StartNew(); try { this.logger.LogInformation((int) AzureProviderErrorCode.AzureTableProvider_InitProvider, $"AzureBlobGrainStorage initializing: {this.options.ToString()}"); if (options.CreateClient is not { } createClient) { throw new OrleansConfigurationException($"No credentials specified. Use the {options.GetType().Name}.{nameof(AzureBlobStorageOptions.ConfigureBlobServiceClient)} method to configure the Azure Blob Service client."); } var client = await createClient(); container = client.GetBlobContainerClient(this.options.ContainerName); await container.CreateIfNotExistsAsync().ConfigureAwait(false); stopWatch.Stop(); this.logger.LogInformation((int)AzureProviderErrorCode.AzureBlobProvider_InitProvider, $"Initializing provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} took {stopWatch.ElapsedMilliseconds} Milliseconds."); } catch (Exception ex) { stopWatch.Stop(); this.logger.LogError((int)ErrorCode.Provider_ErrorFromInit, $"Initialization failed for provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} in {stopWatch.ElapsedMilliseconds} Milliseconds.", ex); throw; } }
/// <summary> /// Uploads the image to azure blob storage /// </summary> /// <param name="file"> file to upload </param> /// <returns> new BlobClient object </returns> public async Task <BlobClient> UploadImage(IFormFile file) { BlobContainerClient container = new BlobContainerClient(Configuration["ImageBlob"], "images"); await container.CreateIfNotExistsAsync(); BlobClient blob = container.GetBlobClient(file.FileName); using var stream = file.OpenReadStream(); BlobUploadOptions options = new BlobUploadOptions() { HttpHeaders = new BlobHttpHeaders() { ContentType = file.ContentType } }; if (!blob.Exists()) { await blob.UploadAsync(stream, options); } return(blob); }
async static Task CreateBlockBlobAsync(string accountName, string containerName) { // Construct the blob container endpoint from the arguments. string containerEndpoint = $"https://{accountName}.blob.core.windows.net/{containerName}"; // Get a credential and create a client object for the blob container. var credential = new DefaultAzureCredential(); BlobContainerClient containerClient = new BlobContainerClient(new Uri(containerEndpoint), credential); try { // Create the container if it does not exist. await containerClient.CreateIfNotExistsAsync(); // Upload text to a new block blob. string blobContents = "This is a block blob."; byte[] byteArray = Encoding.ASCII.GetBytes(blobContents); using (MemoryStream stream = new MemoryStream(byteArray)) { var blobName = System.Environment.TickCount.ToString() + ".txt"; await containerClient.UploadBlobAsync(blobName, stream); } } catch (RequestFailedException e) { Console.WriteLine(e.Message); Console.ReadLine(); throw; } }
public async Task CreateBlockBlobAsync(string blobName) { // Get a credential and create a client object for the blob container. BlobContainerClient containerClient = GetBlobContainerClient(); try { // Create the container if it does not exist. await containerClient.CreateIfNotExistsAsync(); // Upload text to a new block blob. string blobContents = "This is a block blob."; byte[] byteArray = Encoding.ASCII.GetBytes(blobContents); using (MemoryStream stream = new MemoryStream(byteArray)) { await containerClient.UploadBlobAsync(blobName, stream); } } catch (RequestFailedException e) { Console.WriteLine(e.Message); Console.ReadLine(); throw; } }
private async Task InitializeAsync() { BlobContainerClient = new BlobContainerClient(Configuration.Storage.ConnectionString, ContainerName); await BlobContainerClient.CreateIfNotExistsAsync().ConfigureAwait(false); Uri = BlobContainerClient.Uri; UpdateSasUri(); }
private static async Task <BlobContainerClient> CreateClientAsync(string connectionString) { var client = new BlobContainerClient(connectionString, typeof(T).Name.ToLowerInvariant()); await client.CreateIfNotExistsAsync(); return(client); }
public async Task CreateIfNotExistsAsync() { await Task.WhenAll( _container.CreateIfNotExistsAsync(), _queue.CreateIfNotExistsAsync(), _dlQueue.CreateIfNotExistsAsync() ).ConfigureAwait(false); }
private async Task <BlobContainerClient> GetCloudBlobContainer(string containerName) { BlobServiceClient serviceClient = new BlobServiceClient(_options.StorageConnectionString); BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(containerName); await containerClient.CreateIfNotExistsAsync(); return(containerClient); }
private async Task InitAsync() { if (_sendFilesContainerClient == null) { _sendFilesContainerClient = _blobServiceClient.GetBlobContainerClient(FilesContainerName); await _sendFilesContainerClient.CreateIfNotExistsAsync(PublicAccessType.None, null, null); } }
public async Task StartAsync(CancellationToken cancellationToken) { /* create the blob */ await _storageClient.CreateIfNotExistsAsync(); /* Start the processing */ await _processor.StartProcessingAsync(); }
public async Task <IBlobClient> CreateBlobClient() { BlobServiceClient blobService = new BlobServiceClient(_connectionString); BlobContainerClient client = blobService.GetBlobContainerClient(_blobContainerName); await client.CreateIfNotExistsAsync(PublicAccessType.BlobContainer); return(new AzureBlobClient(client)); }
public async Task UploadFile(string filename, Stream stream) { var containerClient = new BlobContainerClient(_connectionString, _mainImageContainer); await containerClient.CreateIfNotExistsAsync(publicAccessType : PublicAccessType.Blob); var blobClient = containerClient.GetBlobClient(filename); await blobClient.UploadAsync(stream); }
//Create new container private static async Task <BlobContainerClient> GetContainersAsync(BlobServiceClient client, string containerName) { BlobContainerClient container = client.GetBlobContainerClient(containerName); await container.CreateIfNotExistsAsync(PublicAccessType.Blob); await Console.Out.WriteLineAsync($"New Container:\t{container.Name}"); return(container); }
public async Task AnonymizeDataset(ActivityInputData inputData, bool force) { var inputContainer = new BlobContainerClient(inputData.SourceStorageConnectionString, inputData.SourceContainerName.ToLower()); if (!await inputContainer.ExistsAsync()) { throw new Exception($"Error: The specified container {inputData.SourceContainerName} does not exist."); } string outputContainerName = inputData.DestinationContainerName.ToLower(); var outputContainer = new BlobContainerClient(inputData.DestinationStorageConnectionString, outputContainerName); await outputContainer.CreateIfNotExistsAsync(); string inputBlobPrefix = GetBlobPrefixFromFolderPath(inputData.SourceFolderPath);; string outputBlobPrefix = GetBlobPrefixFromFolderPath(inputData.DestinationFolderPath);; var skippedBlobCount = 0; var skippedBlobList = new List <string>(); await foreach (BlobItem blob in inputContainer.GetBlobsAsync(BlobTraits.None, BlobStates.None, inputBlobPrefix, default)) { string outputBlobName = GetOutputBlobName(blob.Name, inputBlobPrefix, outputBlobPrefix); Console.WriteLine($"[{blob.Name}]:Processing... output to container '{outputContainerName}'"); var inputBlobClient = inputContainer.GetBlobClient(blob.Name); var outputBlobClient = outputContainer.GetBlobClient(outputBlobName); var isOutputExist = await outputBlobClient.ExistsAsync(); if (!force && isOutputExist) { Console.WriteLine($"Blob file {blob.Name} already exists in {inputData.DestinationContainerName}, skipping.."); skippedBlobCount += 1; skippedBlobList.Add(blob.Name); continue; } else if (force && isOutputExist) { await outputBlobClient.DeleteAsync(); } if (IsInputFileInJsonFormat(blob.Name)) { await AnonymizeBlobInJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name); } else { await AnonymizeBlobInNdJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name); } } if (skippedBlobCount > 0) { Console.WriteLine($"Skipped {skippedBlobCount} files already exists in destination container: {skippedBlobList.ToString()}"); Console.WriteLine($"If you want to overwrite existing blob in {inputData.DestinationContainerName} container, please use the -f or --force flag"); } }
private async Task <BlobContainerClient> AccesoContenedor(string connectionString, string contenedor) { BlobContainerClient cliente; cliente = new BlobContainerClient(connectionString, contenedor); //Conectamos azure storage con un cliente await cliente.CreateIfNotExistsAsync(); //creamos el contenedor si no existe return(cliente); }
private async Task <BlobContainerClient> GetAzureContainer(string username) { string connectionString = this.configuration["AzureStorage_ConnectionString"]; string containerName = UserContainerPrefix + username.ToLower(); BlobContainerClient client = new BlobContainerClient(connectionString, containerName); await client.CreateIfNotExistsAsync(); return(client); }
private async Task EnsureContainerExists() { if (!_containerExists) { await _blobContainerClient.CreateIfNotExistsAsync(); _containerExists = true; } }
public async Task <IBlobContainer> GetContainerAsync(string requestedName, string targetQueue, CancellationToken cancellationToken) { BlobServiceClient account = new BlobServiceClient(_connectionString, StorageRetryPolicy.GetBlobClientOptionsRetrySettings()); BlobContainerClient container = account.GetBlobContainerClient(requestedName); await container.CreateIfNotExistsAsync(); return(new Container(container)); }
private static async Task <BlobContainerClient> CreateBlobContainer(string containerName) { BlobContainerClient cloudBlobContainer = CreateBlobContainerClient(containerName); await cloudBlobContainer.CreateIfNotExistsAsync(); await cloudBlobContainer.SetAccessPolicyAsync(PublicAccessType.Blob); return(cloudBlobContainer); }
private async Task <BlobClient> SetupAzureStoreageAsync(string fileName) { var connectionString = _config["Azure:StorageConnectionString"].ToString(); BlobContainerClient container = new BlobContainerClient(connectionString, "screenshot"); await container.CreateIfNotExistsAsync(); // Get a reference to a blob named "sample-file" in a container named "sample-container" return(container.GetBlobClient(fileName)); }
public IndexModel(ILogger <IndexModel> logger, BlobServiceClient bsc) { _logger = logger; _bsc = bsc; _container = _bsc.GetBlobContainerClient("pictures"); _container.CreateIfNotExistsAsync(); uri = _bsc.Uri; pictures = getPictureNames(); }
/// <summary> /// Lists all the blobs in a folder. /// </summary> /// <param name="containerName">Name of the container holding the blobs to list.</param> /// <param name="prefix">Name of the prefix to filter the list of blobs by.</param> /// <returns>List of <see cref="Blob"/> whose name starts with the prefix.</returns> public async Task <IList <Blob> > ListBlobsInFolderAsync(string containerName, string prefix) { var containerClient = new BlobContainerClient(BlobConnectionString, containerName); await containerClient.CreateIfNotExistsAsync().ConfigureAwait(false); // list all blobs in folder var blobsInFolder = new List <Blob>(); await foreach (BlobItem blob in containerClient.GetBlobsAsync(default, default, prefix, default))
public async Task StartReadAsync(Func <ProcessEventArgs, Task> onMessage, Func <ProcessErrorEventArgs, Task> onError, CancellationToken cancellationToken = default) { OnMessage = onMessage; OnError = onError; ClientReader.ProcessEventAsync += OnMessage; ClientReader.ProcessErrorAsync += OnError; await Task.WhenAll(BlobContainerClient.CreateIfNotExistsAsync(), ClientReader.StartProcessingAsync(cancellationToken)).NoContext(); }