/// <summary>
/// Does the actual work of initialization.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
async Task InitInternalAsync(CancellationToken cancellationToken)
{
logger.Verbose("InitInternalAsync()");
await client.CreateIfNotExistsAsync(cancellationToken : cancellationToken);
init = false;
}
private async Task PushToDataFabric(string routingKey, Stream stream, CancellationToken stoppingToken)
{
stoppingToken.ThrowIfCancellationRequested();
//Create a unique name for the blob
var containerName = $"{_config.ContainerNamePrefix}-{routingKey}";
var containerClient = new BlobContainerClient(_config.StorageConnectionString, containerName);
await containerClient.CreateIfNotExistsAsync(cancellationToken : stoppingToken);
stoppingToken.ThrowIfCancellationRequested();
var blobName =
$"{_config.BlobNamePrefix}-{DateTime.UtcNow.ToString(string.IsNullOrEmpty(_config.BlobNameRollingTimeFormat) ? "yyyyMMdd" : _config.BlobNameRollingTimeFormat)}";
var blobClient = containerClient.GetAppendBlobClient(blobName);
var blobInfo =
await blobClient.CreateIfNotExistsAsync(cancellationToken : stoppingToken);
stoppingToken.ThrowIfCancellationRequested();
_logger.LogInformation("Appending to Blob storage as blob:\n\t {0}\n", blobClient.Uri);
await blobClient.AppendBlockAsync(stream, null,
new AppendBlobRequestConditions { IfMatch = blobInfo?.Value.ETag }, cancellationToken : stoppingToken);
}
private async Task WriteToDataFabric(string routeKey, Stream stream, CancellationToken cancellationToken)
{
if (_diagnostic != null && _diagnostic.IsEnabled() && _diagnostic.IsEnabled(DiagnosticOperations.MessageProcess))
{
_diagnostic.Write(DiagnosticOperations.DataFabricWrite, _config);
}
//Create a unique name for the blob
var containerName = $"{_config.ContainerNamePrefix}-{routeKey}";
var containerClient = new BlobContainerClient(_config.StorageConnectionString, containerName);
await containerClient.CreateIfNotExistsAsync(cancellationToken : cancellationToken);
var blobName =
$"{_config.BlobNamePrefix}-{DateTime.UtcNow.ToString(string.IsNullOrEmpty(_config.BlobNameRollingTimeFormat) ? "yyyyMMdd" : _config.BlobNameRollingTimeFormat)}";
var blobClient = containerClient.GetAppendBlobClient(blobName);
var blobInfo =
await blobClient.CreateIfNotExistsAsync(cancellationToken : cancellationToken);
Console.WriteLine("Appending to Blob storage as blob:\n\t {0}\n", blobClient.Uri);
await blobClient.AppendBlockAsync(stream, null,
new AppendBlobRequestConditions { IfMatch = blobInfo?.Value.ETag }, cancellationToken : cancellationToken);
}
private async Task <BlobContainerClient> GetBlobContainerClientAsync(AzureBlobEgressProviderOptions options, CancellationToken token)
{
BlobServiceClient serviceClient;
if (!string.IsNullOrWhiteSpace(options.SharedAccessSignature))
{
var serviceUriBuilder = new UriBuilder(options.AccountUri)
{
Query = options.SharedAccessSignature
};
serviceClient = new BlobServiceClient(serviceUriBuilder.Uri);
}
else if (!string.IsNullOrEmpty(options.AccountKey))
{
// Remove Query in case SAS token was specified
Uri accountUri = GetAccountUri(options, out string accountName);
StorageSharedKeyCredential credential = new StorageSharedKeyCredential(
accountName,
options.AccountKey);
serviceClient = new BlobServiceClient(accountUri, credential);
}
else
{
throw CreateException(Strings.ErrorMessage_EgressMissingSasOrKey);
}
BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(options.ContainerName);
await containerClient.CreateIfNotExistsAsync(cancellationToken : token);
return(containerClient);
}
private async Task <Uri> GetServiceSasUriForBlob()
{
var containerClient = new BlobContainerClient(_settings.ConnectionString, _settings.ContainerName);
try
{
await containerClient.CreateIfNotExistsAsync();
}
catch (Azure.RequestFailedException ex)
{
if (ex.Status != 409)
{
throw;
}
}
if (!containerClient.CanGenerateSasUri)
{
throw new NotSupportedException("cannot generate sas token");
}
BlobSasBuilder sasBuilder = new BlobSasBuilder()
{
BlobContainerName = containerClient.Name,
Resource = "c"
};
sasBuilder.ExpiresOn = DateTimeOffset.UtcNow.AddMinutes(_settings.ExpiredMunites);
sasBuilder.SetPermissions(BlobSasPermissions.Create | BlobSasPermissions.Write | BlobSasPermissions.Read);
return(containerClient.GenerateSasUri(sasBuilder));
}
/// <summary> Initialization function for this storage provider. </summary>
private async Task Init(CancellationToken ct)
{
var stopWatch = Stopwatch.StartNew();
try
{
this.logger.LogInformation((int)AzureProviderErrorCode.AzureTableProvider_InitProvider, $"AzureTableGrainStorage initializing: {this.options.ToString()}");
this.logger.LogInformation((int)AzureProviderErrorCode.AzureTableProvider_ParamConnectionString, "AzureTableGrainStorage is using DataConnectionString: {0}", ConfigUtilities.RedactConnectionStringInfo(this.options.ConnectionString));
this.jsonSettings = OrleansJsonSerializer.UpdateSerializerSettings(OrleansJsonSerializer.GetDefaultSerializerSettings(this.services), this.options.UseFullAssemblyNames, this.options.IndentJson, this.options.TypeNameHandling);
this.options.ConfigureJsonSerializerSettings?.Invoke(this.jsonSettings);
var client = this.options.ServiceUri != null ? new BlobServiceClient(this.options.ServiceUri, this.options.TokenCredential) : new BlobServiceClient(this.options.ConnectionString);
container = client.GetBlobContainerClient(this.options.ContainerName);
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
stopWatch.Stop();
this.logger.LogInformation((int)AzureProviderErrorCode.AzureBlobProvider_InitProvider, $"Initializing provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} took {stopWatch.ElapsedMilliseconds} Milliseconds.");
}
catch (Exception ex)
{
stopWatch.Stop();
this.logger.LogError((int)ErrorCode.Provider_ErrorFromInit, $"Initialization failed for provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} in {stopWatch.ElapsedMilliseconds} Milliseconds.", ex);
throw;
}
}
private async Task WriteStateAndCreateContainerIfNotExists(string grainType, GrainReference grainId, IGrainState grainState, byte[] contents, string mimeType, BlobClient blob)
{
try
{
var conditions = grainState.ETag == null
? new BlobRequestConditions {
IfNoneMatch = new ETag("*")
}
: new BlobRequestConditions {
IfMatch = new ETag(grainState.ETag)
};
using var stream = new MemoryStream(contents);
var result = await DoOptimisticUpdate(() => blob.UploadAsync(stream,
conditions: conditions,
httpHeaders: new BlobHttpHeaders {
ContentType = mimeType
}),
blob, grainState.ETag).ConfigureAwait(false);
grainState.ETag = result.Value.ETag.ToString();
grainState.RecordExists = true;
}
catch (RequestFailedException exception) when(exception.IsContainerNotFound())
{
// if the container does not exist, create it, and make another attempt
if (this.logger.IsEnabled(LogLevel.Trace))
{
this.logger.Trace((int)AzureProviderErrorCode.AzureBlobProvider_ContainerNotFound, "Creating container: GrainType={0} Grainid={1} ETag={2} to BlobName={3} in Container={4}", grainType, grainId, grainState.ETag, blob.Name, container.Name);
}
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
await WriteStateAndCreateContainerIfNotExists(grainType, grainId, grainState, contents, mimeType, blob).ConfigureAwait(false);
}
}
/// <summary> Initialization function for this storage provider. </summary>
private async Task Init(CancellationToken ct)
{
var stopWatch = Stopwatch.StartNew();
try
{
this.logger.LogInformation((int) AzureProviderErrorCode.AzureTableProvider_InitProvider, $"AzureBlobGrainStorage initializing: {this.options.ToString()}");
if (options.CreateClient is not {
} createClient)
{
throw new OrleansConfigurationException($"No credentials specified. Use the {options.GetType().Name}.{nameof(AzureBlobStorageOptions.ConfigureBlobServiceClient)} method to configure the Azure Blob Service client.");
}
var client = await createClient();
container = client.GetBlobContainerClient(this.options.ContainerName);
await container.CreateIfNotExistsAsync().ConfigureAwait(false);
stopWatch.Stop();
this.logger.LogInformation((int)AzureProviderErrorCode.AzureBlobProvider_InitProvider, $"Initializing provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} took {stopWatch.ElapsedMilliseconds} Milliseconds.");
}
catch (Exception ex)
{
stopWatch.Stop();
this.logger.LogError((int)ErrorCode.Provider_ErrorFromInit, $"Initialization failed for provider {this.name} of type {this.GetType().Name} in stage {this.options.InitStage} in {stopWatch.ElapsedMilliseconds} Milliseconds.", ex);
throw;
}
}
/// <summary>
/// Uploads the image to azure blob storage
/// </summary>
/// <param name="file"> file to upload </param>
/// <returns> new BlobClient object </returns>
public async Task <BlobClient> UploadImage(IFormFile file)
{
BlobContainerClient container = new BlobContainerClient(Configuration["ImageBlob"], "images");
await container.CreateIfNotExistsAsync();
BlobClient blob = container.GetBlobClient(file.FileName);
using var stream = file.OpenReadStream();
BlobUploadOptions options = new BlobUploadOptions()
{
HttpHeaders = new BlobHttpHeaders()
{
ContentType = file.ContentType
}
};
if (!blob.Exists())
{
await blob.UploadAsync(stream, options);
}
return(blob);
}
async static Task CreateBlockBlobAsync(string accountName, string containerName)
{
// Construct the blob container endpoint from the arguments.
string containerEndpoint = $"https://{accountName}.blob.core.windows.net/{containerName}";
// Get a credential and create a client object for the blob container.
var credential = new DefaultAzureCredential();
BlobContainerClient containerClient = new BlobContainerClient(new Uri(containerEndpoint),
credential);
try
{
// Create the container if it does not exist.
await containerClient.CreateIfNotExistsAsync();
// Upload text to a new block blob.
string blobContents = "This is a block blob.";
byte[] byteArray = Encoding.ASCII.GetBytes(blobContents);
using (MemoryStream stream = new MemoryStream(byteArray))
{
var blobName = System.Environment.TickCount.ToString() + ".txt";
await containerClient.UploadBlobAsync(blobName, stream);
}
}
catch (RequestFailedException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
public async Task CreateBlockBlobAsync(string blobName)
{
// Get a credential and create a client object for the blob container.
BlobContainerClient containerClient = GetBlobContainerClient();
try
{
// Create the container if it does not exist.
await containerClient.CreateIfNotExistsAsync();
// Upload text to a new block blob.
string blobContents = "This is a block blob.";
byte[] byteArray = Encoding.ASCII.GetBytes(blobContents);
using (MemoryStream stream = new MemoryStream(byteArray))
{
await containerClient.UploadBlobAsync(blobName, stream);
}
}
catch (RequestFailedException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
private async Task InitializeAsync()
{
BlobContainerClient = new BlobContainerClient(Configuration.Storage.ConnectionString, ContainerName);
await BlobContainerClient.CreateIfNotExistsAsync().ConfigureAwait(false);
Uri = BlobContainerClient.Uri;
UpdateSasUri();
}
private static async Task <BlobContainerClient> CreateClientAsync(string connectionString)
{
var client = new BlobContainerClient(connectionString, typeof(T).Name.ToLowerInvariant());
await client.CreateIfNotExistsAsync();
return(client);
}
public async Task CreateIfNotExistsAsync()
{
await Task.WhenAll(
_container.CreateIfNotExistsAsync(),
_queue.CreateIfNotExistsAsync(),
_dlQueue.CreateIfNotExistsAsync()
).ConfigureAwait(false);
}
private async Task <BlobContainerClient> GetCloudBlobContainer(string containerName)
{
BlobServiceClient serviceClient = new BlobServiceClient(_options.StorageConnectionString);
BlobContainerClient containerClient = serviceClient.GetBlobContainerClient(containerName);
await containerClient.CreateIfNotExistsAsync();
return(containerClient);
}
private async Task InitAsync()
{
if (_sendFilesContainerClient == null)
{
_sendFilesContainerClient = _blobServiceClient.GetBlobContainerClient(FilesContainerName);
await _sendFilesContainerClient.CreateIfNotExistsAsync(PublicAccessType.None, null, null);
}
}
public async Task StartAsync(CancellationToken cancellationToken)
{
/* create the blob */
await _storageClient.CreateIfNotExistsAsync();
/* Start the processing */
await _processor.StartProcessingAsync();
}
public async Task <IBlobClient> CreateBlobClient()
{
BlobServiceClient blobService = new BlobServiceClient(_connectionString);
BlobContainerClient client = blobService.GetBlobContainerClient(_blobContainerName);
await client.CreateIfNotExistsAsync(PublicAccessType.BlobContainer);
return(new AzureBlobClient(client));
}
public async Task UploadFile(string filename, Stream stream)
{
var containerClient = new BlobContainerClient(_connectionString, _mainImageContainer);
await containerClient.CreateIfNotExistsAsync(publicAccessType : PublicAccessType.Blob);
var blobClient = containerClient.GetBlobClient(filename);
await blobClient.UploadAsync(stream);
}
//Create new container
private static async Task <BlobContainerClient> GetContainersAsync(BlobServiceClient client, string containerName)
{
BlobContainerClient container = client.GetBlobContainerClient(containerName);
await container.CreateIfNotExistsAsync(PublicAccessType.Blob);
await Console.Out.WriteLineAsync($"New Container:\t{container.Name}");
return(container);
}
public async Task AnonymizeDataset(ActivityInputData inputData, bool force)
{
var inputContainer = new BlobContainerClient(inputData.SourceStorageConnectionString, inputData.SourceContainerName.ToLower());
if (!await inputContainer.ExistsAsync())
{
throw new Exception($"Error: The specified container {inputData.SourceContainerName} does not exist.");
}
string outputContainerName = inputData.DestinationContainerName.ToLower();
var outputContainer = new BlobContainerClient(inputData.DestinationStorageConnectionString, outputContainerName);
await outputContainer.CreateIfNotExistsAsync();
string inputBlobPrefix = GetBlobPrefixFromFolderPath(inputData.SourceFolderPath);;
string outputBlobPrefix = GetBlobPrefixFromFolderPath(inputData.DestinationFolderPath);;
var skippedBlobCount = 0;
var skippedBlobList = new List <string>();
await foreach (BlobItem blob in inputContainer.GetBlobsAsync(BlobTraits.None, BlobStates.None, inputBlobPrefix, default))
{
string outputBlobName = GetOutputBlobName(blob.Name, inputBlobPrefix, outputBlobPrefix);
Console.WriteLine($"[{blob.Name}]：Processing... output to container '{outputContainerName}'");
var inputBlobClient = inputContainer.GetBlobClient(blob.Name);
var outputBlobClient = outputContainer.GetBlobClient(outputBlobName);
var isOutputExist = await outputBlobClient.ExistsAsync();
if (!force && isOutputExist)
{
Console.WriteLine($"Blob file {blob.Name} already exists in {inputData.DestinationContainerName}, skipping..");
skippedBlobCount += 1;
skippedBlobList.Add(blob.Name);
continue;
}
else if (force && isOutputExist)
{
await outputBlobClient.DeleteAsync();
}
if (IsInputFileInJsonFormat(blob.Name))
{
await AnonymizeBlobInJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name);
}
else
{
await AnonymizeBlobInNdJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name);
}
}
if (skippedBlobCount > 0)
{
Console.WriteLine($"Skipped {skippedBlobCount} files already exists in destination container: {skippedBlobList.ToString()}");
Console.WriteLine($"If you want to overwrite existing blob in {inputData.DestinationContainerName} container, please use the -f or --force flag");
}
}
private async Task <BlobContainerClient> AccesoContenedor(string connectionString, string contenedor)
{
BlobContainerClient cliente;
cliente = new BlobContainerClient(connectionString, contenedor); //Conectamos azure storage con un cliente
await cliente.CreateIfNotExistsAsync(); //creamos el contenedor si no existe
return(cliente);
}
private async Task <BlobContainerClient> GetAzureContainer(string username)
{
string connectionString = this.configuration["AzureStorage_ConnectionString"];
string containerName = UserContainerPrefix + username.ToLower();
BlobContainerClient client = new BlobContainerClient(connectionString, containerName);
await client.CreateIfNotExistsAsync();
return(client);
}
private async Task EnsureContainerExists()
{
if (!_containerExists)
{
await _blobContainerClient.CreateIfNotExistsAsync();
_containerExists = true;
}
}
public async Task <IBlobContainer> GetContainerAsync(string requestedName, string targetQueue, CancellationToken cancellationToken)
{
BlobServiceClient account = new BlobServiceClient(_connectionString, StorageRetryPolicy.GetBlobClientOptionsRetrySettings());
BlobContainerClient container = account.GetBlobContainerClient(requestedName);
await container.CreateIfNotExistsAsync();
return(new Container(container));
}
private static async Task <BlobContainerClient> CreateBlobContainer(string containerName)
{
BlobContainerClient cloudBlobContainer = CreateBlobContainerClient(containerName);
await cloudBlobContainer.CreateIfNotExistsAsync();
await cloudBlobContainer.SetAccessPolicyAsync(PublicAccessType.Blob);
return(cloudBlobContainer);
}
private async Task <BlobClient> SetupAzureStoreageAsync(string fileName)
{
var connectionString = _config["Azure:StorageConnectionString"].ToString();
BlobContainerClient container = new BlobContainerClient(connectionString, "screenshot");
await container.CreateIfNotExistsAsync();
// Get a reference to a blob named "sample-file" in a container named "sample-container"
return(container.GetBlobClient(fileName));
}
public IndexModel(ILogger <IndexModel> logger, BlobServiceClient bsc)
{
_logger = logger;
_bsc = bsc;
_container = _bsc.GetBlobContainerClient("pictures");
_container.CreateIfNotExistsAsync();
uri = _bsc.Uri;
pictures = getPictureNames();
}
/// <summary>
/// Lists all the blobs in a folder.
/// </summary>
/// <param name="containerName">Name of the container holding the blobs to list.</param>
/// <param name="prefix">Name of the prefix to filter the list of blobs by.</param>
/// <returns>List of <see cref="Blob"/> whose name starts with the prefix.</returns>
public async Task <IList <Blob> > ListBlobsInFolderAsync(string containerName, string prefix)
{
var containerClient = new BlobContainerClient(BlobConnectionString, containerName);
await containerClient.CreateIfNotExistsAsync().ConfigureAwait(false);
// list all blobs in folder
var blobsInFolder = new List <Blob>();
await foreach (BlobItem blob in containerClient.GetBlobsAsync(default, default, prefix, default))
public async Task StartReadAsync(Func <ProcessEventArgs, Task> onMessage, Func <ProcessErrorEventArgs, Task> onError, CancellationToken cancellationToken = default)
{
OnMessage = onMessage;
OnError = onError;
ClientReader.ProcessEventAsync += OnMessage;
ClientReader.ProcessErrorAsync += OnError;
await Task.WhenAll(BlobContainerClient.CreateIfNotExistsAsync(),
ClientReader.StartProcessingAsync(cancellationToken)).NoContext();
}