public async Task <bool> IsFileExistsInBlobAsync(string fileName)
{
if (!await _blobContainerClient.ExistsAsync())
{
return(false);
}
var blobClient = _blobContainerClient.GetBlobClient(fileName);
return(await blobClient.ExistsAsync());
}
// Instead of streaming the blob through your server, you could download it directly from the blob storage. http://stackoverflow.com/questions/24312527/azure-blob-storage-downloadtobytearray-vs-downloadtostream
/// <summary>
/// Get the file's properties (metadata).
/// </summary>
/// <param name="filepath">The path to the file.</param>
public async Task <FileProperties> GetPropertiesAsync(string filepath)
{
filepath = filepath.TrimStart('\\', '/');
var folder = _environmentName ?? Path.GetDirectoryName(filepath);
var filename = _environmentName == null?filepath.Substring(folder.Length) : filepath;
var container = new BlobContainerClient(_connectionString, folder);
var exists = await container.ExistsAsync();
if (!exists)
{
throw new FileNotFoundServiceException($"Container {folder} not found.");
}
var blob = container.GetBlobClient(filename);
try {
var response = await blob.GetPropertiesAsync();
return(new FileProperties {
CacheControl = response.Value.CacheControl,
ContentDisposition = response.Value.ContentDisposition,
ContentEncoding = response.Value.ContentEncoding,
ContentHash = System.Text.Encoding.UTF8.GetString(response.Value.ContentHash),
ContentType = response.Value.ContentType,
Length = response.Value.ContentLength,
ETag = response.Value.ETag.ToString(),
LastModified = response.Value.LastModified
});
} catch (RequestFailedException ex)
when(ex.ErrorCode == BlobErrorCode.BlobNotFound)
{
throw new FileNotFoundServiceException($"File {filename} not found.");
}
}
/// <summary>
/// Deletes a file or folder.
/// </summary>
/// <param name="filepath">The path to the file.</param>
/// <param name="isDirectory">Determines if the <paramref name="filepath"/> points to a single file or a directory.</param>
public async Task <bool> DeleteAsync(string filepath, bool isDirectory = false)
{
filepath = filepath.TrimStart('\\', '/');
var folder = _environmentName ?? Path.GetDirectoryName(filepath);
var filename = _environmentName == null?filepath.Substring(folder.Length) : filepath;
var container = new BlobContainerClient(_connectionString, folder);
var exists = await container.ExistsAsync();
if (!exists)
{
throw new FileNotFoundServiceException($"Container {folder} not found.");
}
bool deleted;
if (!isDirectory)
{
var blob = container.GetBlobClient(filename);
deleted = await blob.DeleteIfExistsAsync();
}
else
{
var segment = container.GetBlobsAsync(prefix: filename);
await foreach (var blob in segment)
{
await container.DeleteBlobIfExistsAsync(blob.Name, DeleteSnapshotsOption.IncludeSnapshots);
}
deleted = true;
}
return(deleted);
}
/// <summary>
/// Create blob container client.
/// </summary>
/// <param name="destination"></param>
/// <param name="createContainerIfNotExists"></param>
/// <seealso cref="https://www.nuget.org/packages/Azure.Storage.Blobs"/>
/// <seealso cref="https://docs.microsoft.com/pt-br/azure/storage/blobs/storage-quickstart-blobs-dotnet#configure-your-storage-connection-string"/>
/// <seealso cref="https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/storage"/>
/// <returns></returns>
private async Task <BlobContainerClient> CreateBlobContainerClient(FilePath destination, bool createContainerIfNotExists = true)
{
string containerName = destination.Folder;
string connectionString = GetConnectionString();
// Create a BlobServiceClient object which will be used to create a container client
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(containerName);
//check if the container exists
var containerExists = await containerClient.ExistsAsync();
if (containerClient != null && createContainerIfNotExists == false || containerExists)
{
return(containerClient);
}
try
{
// Create the container and return a container client object
return(await blobServiceClient.CreateBlobContainerAsync(containerName));
}
catch (Exception ex)
{
throw ex;
}
}
// Instead of streaming the blob through your server, you could download it directly from the blob storage.
// http://stackoverflow.com/questions/24312527/azure-blob-storage-downloadtobytearray-vs-downloadtostream
/// <summary>
/// Retrieve the data for a file.
/// </summary>
/// <param name="filepath">The path to the file.</param>
public async Task <byte[]> GetAsync(string filepath)
{
filepath = filepath.TrimStart('\\', '/');
var folder = _environmentName ?? Path.GetDirectoryName(filepath);
var filename = _environmentName == null?filepath.Substring(folder.Length) : filepath;
var container = new BlobContainerClient(_connectionString, folder);
await container.CreateIfNotExistsAsync();
var exists = await container.ExistsAsync();
if (!exists)
{
throw new FileNotFoundServiceException($"Container {folder} not found.");
}
var blob = container.GetBlobClient(filename);
//exists = await blob.ExistsAsync();
//if (!exists) {
// throw new FileNotFoundServiceException($"File {filename} not found.");
//}
try {
using (var s = new MemoryStream()) {
await blob.DownloadToAsync(s);
return(s.ToArray());
}
} catch (RequestFailedException ex)
when(ex.ErrorCode == BlobErrorCode.BlobNotFound)
{
throw new FileNotFoundServiceException($"File {filename} not found.");
}
}
public async Task <IActionResult> Download(string id)
{
var directory = Path.Combine(Directory.GetCurrentDirectory(), "Photos");
var filename = Path.Combine(directory, id);
var mimeType = "application/octet-stream";
// Local
//var stream = new FileStream(filename, FileMode.Open);
// Azure Storage
BlobContainerClient containerClient = _blobServiceClient.GetBlobContainerClient("photos");
Stream stream = null;
if (await containerClient.ExistsAsync())
{
BlobClient blobClient = containerClient.GetBlobClient(id);
if (await blobClient.ExistsAsync())
{
stream = new MemoryStream();
BlobDownloadInfo download = await blobClient.DownloadAsync();
await download.Content.CopyToAsync(stream);
stream.Seek(0, SeekOrigin.Begin);
}
}
if (stream == null)
{
return(NotFound()); // returns a NotFoundResult with Status404NotFound response.
}
return(File(stream, mimeType)); // returns a FileStreamResult
}
public async Task <List <ChapterRules> > GetChapterRulesFromBlobContainer(string containerName, Language language)
{
var blobContainerClient = new BlobContainerClient(_configuration["StorageAccountConnectionString"], $"{containerName}-{language}");
if (!await blobContainerClient.ExistsAsync())
{
blobContainerClient = new BlobContainerClient(_configuration["StorageAccountConnectionString"], $"{containerName}-{Language.en}");
}
var chapterRules = new List <ChapterRules>();
var chapterBlobs = new List <BlobItem>();
await foreach (var blob in blobContainerClient.GetBlobsAsync())
{
chapterBlobs.Add(blob);
}
foreach (var chapterBlob in chapterBlobs)
{
var blobClient = blobContainerClient.GetBlobClient(chapterBlob.Name);
var stream = new MemoryStream();
await blobClient.DownloadToAsync(stream);
stream.Seek(0, SeekOrigin.Begin);
var chapterRule = JsonConvert.DeserializeObject <ChapterRules>(await new StreamReader(stream).ReadToEndAsync());
chapterRules.Add(chapterRule);
}
return(chapterRules);
}
public static async Task ArchiveBlobAsync(string blobName, ILogger log)
{
log.LogInformation($"Archiving blob: {blobName}");
string containerEndpoint = string.Format("https://{0}.blob.core.windows.net/archive", Common.GetEnvironmentVariable("SA_NAME"));
BlobContainerClient sourceBlobContainerClient = new BlobContainerClient(new Uri(containerEndpoint), new DefaultAzureCredential());
try
{
if (await sourceBlobContainerClient.ExistsAsync())
{
BlobClient blob = sourceBlobContainerClient.GetBlobClient(blobName);
if (await blob.ExistsAsync())
{
await blob.SetAccessTierAsync(AccessTier.Archive);
log.LogInformation($"Access tier set to Archive. Blob name: {blobName}");
}
}
}
catch (RequestFailedException)
{
log.LogInformation($"Failed to complete blob archiving operation: {blobName}");
throw;
}
}
public static async Task DeleteBlobAsync(BlobMetadata blobMetadata, ILogger log)
{
log.LogInformation($"Delete queue process item: {blobMetadata}");
string containerEndpoint = string.Format("https://{0}.blob.core.windows.net/{1}", Common.GetEnvironmentVariable("SA_NAME"), blobMetadata.ContainerName);
var blobPathAndName = $"{blobMetadata.BlobPath}/{blobMetadata.BlobName}";
BlobContainerClient sourceBlobContainerClient = new BlobContainerClient(new Uri(containerEndpoint), new DefaultAzureCredential());
try
{
if (await sourceBlobContainerClient.ExistsAsync())
{
BlobClient blob = sourceBlobContainerClient.GetBlobClient(blobPathAndName);
if (await blob.ExistsAsync())
{
await blob.DeleteAsync();
log.LogInformation($"Blob {blobMetadata.BlobName} is deleted");
}
}
}
catch (RequestFailedException)
{
log.LogInformation($"Failed to complete blob deleting operation: {blobMetadata}");
throw;
}
}
public async Task <MemoryStream?> Get(string key, MemoryStream? @default = null)
{
var(containerName, filePath) = SplitKey(key);
var container = new BlobContainerClient(_connectionString, containerName);
var containerExists = await container.ExistsAsync();
if (!containerExists.Value)
{
return(@default);
}
var blob = container.GetBlobClient(filePath);
var blobExists = await blob.ExistsAsync();
if (!blobExists.Value)
{
return(@default);
}
var stream = new MemoryStream();
await blob.DownloadToAsync(stream);
return(stream);
}
private static async Task EnsureBlobContainerExists(BlobContainerClient container)
{
if (!await container.ExistsAsync())
{
throw new InvalidOperationException($"Container {container.Name} does not exist.");
}
}
//-------------------------------------------------
// Create a container
//-------------------------------------------------
private static async Task <BlobContainerClient> CreateSampleContainerAsync(BlobServiceClient blobServiceClient)
{
// Name the sample container based on new GUID to ensure uniqueness.
// The container name must be lowercase.
string containerName = "container-" + Guid.NewGuid();
try
{
// Create the container
BlobContainerClient container = await blobServiceClient.CreateBlobContainerAsync(containerName);
if (await container.ExistsAsync())
{
Console.WriteLine("Created container {0}", container.Name);
return(container);
}
}
catch (RequestFailedException e)
{
Console.WriteLine("HTTP error code {0}: {1}",
e.Status, e.ErrorCode);
Console.WriteLine(e.Message);
}
return(null);
}
private async Task <List <MigrationCheckpoint> > ReadLegacyCheckpoints(
string connectionString,
string container,
string consumerGroup,
CancellationToken cancellationToken)
{
var storageClient = new BlobContainerClient(connectionString, container);
// If there is no container, no action can be taken.
if (!(await storageClient.ExistsAsync(cancellationToken)))
{
throw new ArgumentException("The source container does not exist.", nameof(container));
}
// Read and process the legacy checkpoints.
var checkpoints = new List <MigrationCheckpoint>();
await foreach (var blobItem in storageClient.GetBlobsAsync(BlobTraits.All, BlobStates.All, consumerGroup, cancellationToken))
{
using var blobContentStream = new MemoryStream();
await(storageClient.GetBlobClient(blobItem.Name)).DownloadToAsync(blobContentStream);
var checkpoint = JsonSerializer.Deserialize <MigrationCheckpoint>(Encoding.UTF8.GetString(blobContentStream.ToArray()));
if (!string.IsNullOrEmpty(checkpoint.Offset))
{
checkpoints.Add(checkpoint);
}
}
return(checkpoints);
}
/// <summary>
/// Gets a path list for a given folder.
/// </summary>
/// <param name="filepath">The path to the file.</param>
public async Task <IEnumerable <string> > SearchAsync(string filepath)
{
filepath = filepath.TrimStart('\\', '/');
var folder = _environmentName ?? Path.GetDirectoryName(filepath);
var filename = _environmentName == null?filepath.Substring(folder.Length) : filepath;
var container = new BlobContainerClient(_connectionString, folder);
var exists = await container.ExistsAsync();
if (!exists)
{
throw new FileNotFoundServiceException($"Container {folder} not found.");
}
var results = new List <string>();
var segment = container.GetBlobsAsync(prefix: filename);
// Enumerate the blobs returned for each page.
await foreach (var blob in segment)
{
results.Add(blob.Name);
}
return(results);
/* in the future we may need to get the contents of a specific folder structure
* the following code will be verry usefull
* var results = new List<string>();
* var resultSegment = container.GetBlobsByHierarchyAsync(prefix: filename, delimiter: "/").AsPages(default, null);
* // Enumerate the blobs returned for each page.
* await foreach (var page in resultSegment) {
* results.AddRange(page.Values.Where(item => item.IsBlob).Select(item => $"{item.Prefix}{item.Blob.Name}"));
* }
* return results;
*/
}
internal static async Task <bool> StorageContainerExists(string storageAccountName, string storageAccountKey, string containerName)
{
var connstr = GetStorageConnectionString(storageAccountName, storageAccountKey);
BlobContainerClient container = new BlobContainerClient(connstr, containerName);
var r = await container.ExistsAsync();
return(r.Value);
}
public async Task AnonymizeDataset(ActivityInputData inputData, bool force)
{
var inputContainer = new BlobContainerClient(inputData.SourceStorageConnectionString, inputData.SourceContainerName.ToLower());
if (!await inputContainer.ExistsAsync())
{
throw new Exception($"Error: The specified container {inputData.SourceContainerName} does not exist.");
}
string outputContainerName = inputData.DestinationContainerName.ToLower();
var outputContainer = new BlobContainerClient(inputData.DestinationStorageConnectionString, outputContainerName);
await outputContainer.CreateIfNotExistsAsync();
string inputBlobPrefix = GetBlobPrefixFromFolderPath(inputData.SourceFolderPath);;
string outputBlobPrefix = GetBlobPrefixFromFolderPath(inputData.DestinationFolderPath);;
var skippedBlobCount = 0;
var skippedBlobList = new List <string>();
await foreach (BlobItem blob in inputContainer.GetBlobsAsync(BlobTraits.None, BlobStates.None, inputBlobPrefix, default))
{
string outputBlobName = GetOutputBlobName(blob.Name, inputBlobPrefix, outputBlobPrefix);
Console.WriteLine($"[{blob.Name}]：Processing... output to container '{outputContainerName}'");
var inputBlobClient = inputContainer.GetBlobClient(blob.Name);
var outputBlobClient = outputContainer.GetBlobClient(outputBlobName);
var isOutputExist = await outputBlobClient.ExistsAsync();
if (!force && isOutputExist)
{
Console.WriteLine($"Blob file {blob.Name} already exists in {inputData.DestinationContainerName}, skipping..");
skippedBlobCount += 1;
skippedBlobList.Add(blob.Name);
continue;
}
else if (force && isOutputExist)
{
await outputBlobClient.DeleteAsync();
}
if (IsInputFileInJsonFormat(blob.Name))
{
await AnonymizeBlobInJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name);
}
else
{
await AnonymizeBlobInNdJsonFormatAsync(inputBlobClient, outputBlobClient, blob.Name);
}
}
if (skippedBlobCount > 0)
{
Console.WriteLine($"Skipped {skippedBlobCount} files already exists in destination container: {skippedBlobList.ToString()}");
Console.WriteLine($"If you want to overwrite existing blob in {inputData.DestinationContainerName} container, please use the -f or --force flag");
}
}
private static async Task DeleteContainer(BlobContainerClient container)
{
await container.DeleteIfExistsAsync();
do
{
await Task.Delay(1000);
} while (await container.ExistsAsync());
}
private async Task <BlobContainerClient> GetBlobContainer()
{
BlobServiceClient service = this.GetBlobService();
BlobContainerClient container = service.GetBlobContainerClient(this.ContainerName);
if (!(await container.ExistsAsync()))
{
container = await service.CreateBlobContainerAsync(this.ContainerName);
}
return(container);
}
/// <summary>
/// Method to check if Blob Storage Container exists or not
/// </summary>
/// <param name="connectionString">Connection String details for Azure Blob Storage</param>
/// <param name="containerName">Container Name to upload files</param>
/// <returns>Boolean</returns>
public static async Task <bool> IsContainerAvailableAsync(string connectionString, string containerName)
{
try
{
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient(containerName);
return(await blobContainerClient.ExistsAsync());
}
catch (Exception ex)
{
throw new Exception($"Exception checking the container availability. Exception: {ex.Message}");
}
}
public async Task TestConnectionAsync()
{
try
{
if (await _client.ExistsAsync(cancellationToken: _cancellationToken) == false)
{
throw new ContainerNotFoundException($"Container '{_storageContainer}' wasn't found!");
}
}
catch (UnauthorizedAccessException)
{
// we don't have the permissions to see if the container exists
}
}
public async Task GivenAllPatientGroupWithFilters_WhenProcessGroupScope_CorrectResultShouldBeReturnedAsync()
{
Skip.If(_blobServiceClient == null);
var uniqueContainerName = Guid.NewGuid().ToString("N");
BlobContainerClient blobContainerClient = _blobServiceClient.GetBlobContainerClient(uniqueContainerName);
// Make sure the container is deleted before running the tests
Assert.False(await blobContainerClient.ExistsAsync());
// Load configuration
Environment.SetEnvironmentVariable("job:containerName", uniqueContainerName);
Environment.SetEnvironmentVariable("filter:filterScope", "Group");
Environment.SetEnvironmentVariable("filter:requiredTypes", "Condition,MedicationRequest,Patient");
Environment.SetEnvironmentVariable("filter:typeFilters", "MedicationRequest?status=active,MedicationRequest?status=completed&date=gt2018-07-01T00:00:00Z");
// this group includes all the 80 patients
Environment.SetEnvironmentVariable("filter:groupId", "72d653ce-2dbb-4432-bfa0-9ac47d0e0a2c");
var configuration = new ConfigurationBuilder()
.AddJsonFile(TestConfigurationPath)
.AddEnvironmentVariables()
.Build();
try
{
// Run e2e
var host = CreateHostBuilder(configuration).Build();
await host.RunAsync();
// Check job status
var fileName = Path.Combine(_expectedDataFolder, "GroupScope_AllPatient_Filters.json");
var expectedJob = JsonConvert.DeserializeObject <Job>(File.ReadAllText(fileName));
await CheckJobStatus(blobContainerClient, expectedJob);
// Check result files
Assert.Equal(1, await GetResultFileCount(blobContainerClient, "result/Patient/2022/07/01"));
Assert.Equal(1, await GetResultFileCount(blobContainerClient, "result/Condition/2022/07/01"));
Assert.Equal(1, await GetResultFileCount(blobContainerClient, "result/MedicationRequest/2022/07/01"));
var schedulerMetadata = await GetSchedulerMetadata(blobContainerClient);
Assert.Empty(schedulerMetadata.FailedJobs);
Assert.Equal(80, schedulerMetadata.ProcessedPatients.Count());
}
finally
{
await blobContainerClient.DeleteIfExistsAsync();
}
}
private async Task EnsureContainerExistsAsync()
{
if (_ensured)
{
return;
}
if (!await _containerClient.ExistsAsync())
{
throw new InvalidOperationException($"Blob container '{ContainerName}' does not exist");
}
_ensured = true;
}
public async Task <HealthCheckResult> CheckBlobContainerHealth(string storageAccountConnectionString, string containerName)
{
BlobContainerClient container = new BlobContainerClient(storageAccountConnectionString, containerName);
var blobContainerExists = await container.ExistsAsync();
if (blobContainerExists)
{
return(HealthCheckResult.Healthy("Azure blob storage is healthy"));
}
else
{
return(HealthCheckResult.Unhealthy("Azure blob storage is unhealthy", new Exception("Azure blob storage connection failed or not available")));
}
}
static async Task Main(string[] args)
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json");
var configuration = builder.Build();
var connectionString = configuration["StorageAccountConnectionString"];
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync("photoblobs");
await containerClient.ExistsAsync();
}
private async Task Upload(CancellationToken token)
{
var container = new BlobContainerClient(
ConfigurationManager.AppSettings["BlobConnectionString"],
ConfigurationManager.AppSettings["BlobContainerName"]);
var subDirectoriesSettings = ConfigurationManager.AppSettings["BackupSubDirectories"];
var subDirectories = string.IsNullOrEmpty(subDirectoriesSettings) ? null : subDirectoriesSettings.Split(',');
var containerExists = await container.ExistsAsync(token);
await container.CreateIfNotExistsAsync(cancellationToken : token);
var files = Directory.GetFiles(this.LocalDirectory, "*", SearchOption.AllDirectories)
.Where(e => subDirectories == null || subDirectories.Contains(e.Replace(this.LocalDirectory, string.Empty).Split('\\', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault()))
.Select(e => new { FilePath = e, BlobPath = ConvertBlobPath(e) }).ToList();
if (containerExists.Value)
{
var currentBlobItems = container.GetBlobsAsync(cancellationToken: token);
await foreach (var currentBlobItem in currentBlobItems)
{
var fileItem = files.Where(e => e.BlobPath == currentBlobItem.Name).FirstOrDefault();
if (fileItem != null)
{
files.Remove(fileItem);
}
}
}
if (!files.Any())
{
return;
}
var progressor = new Progressor(files.Count);
foreach (var file in files)
{
var blob = container.GetBlobClient(file.BlobPath);
using var stream = File.OpenRead(file.FilePath);
await blob.UploadAsync(stream, new BlobUploadOptions()
{
AccessTier = AccessTier.Archive,
ProgressHandler = progressor.CreateUploadProgress((int)stream.Length, file.BlobPath)
}, token);
var properties = await blob.GetPropertiesAsync(cancellationToken : token);
Debug.Assert(stream.Length == properties.Value.ContentLength);
progressor.Tick();
}
}
public async Task GivenOnePatientGroup_WhenProcessGroupScope_CorrectResultShouldBeReturnedAsync()
{
Skip.If(_blobServiceClient == null);
var uniqueContainerName = Guid.NewGuid().ToString("N");
BlobContainerClient blobContainerClient = _blobServiceClient.GetBlobContainerClient(uniqueContainerName);
// Make sure the container is deleted before running the tests
Assert.False(await blobContainerClient.ExistsAsync());
// Load configuration
Environment.SetEnvironmentVariable("job:containerName", uniqueContainerName);
Environment.SetEnvironmentVariable("filter:filterScope", "Group");
// only patient cbe1a164-c5c8-65b4-747a-829a6bd4e85f is included in this group
Environment.SetEnvironmentVariable("filter:groupId", "af3aba4f-7bb6-41e3-85e4-d931d7653ede");
Environment.SetEnvironmentVariable("filter:requiredTypes", string.Empty);
Environment.SetEnvironmentVariable("filter:typeFilters", string.Empty);
var configuration = new ConfigurationBuilder()
.AddJsonFile(TestConfigurationPath)
.AddEnvironmentVariables()
.Build();
try
{
// Run e2e
var host = CreateHostBuilder(configuration).Build();
await host.RunAsync();
// Check job status
var fileName = Path.Combine(_expectedDataFolder, "GroupScope_OnePatient_All.json");
var expectedJob = JsonConvert.DeserializeObject <Job>(File.ReadAllText(fileName));
await CheckJobStatus(blobContainerClient, expectedJob);
// Check result files
Assert.Equal(20, await GetResultFileCount(blobContainerClient, "result"));
var schedulerMetadata = await GetSchedulerMetadata(blobContainerClient);
Assert.Empty(schedulerMetadata.FailedJobs);
Assert.Single(schedulerMetadata.ProcessedPatients);
}
finally
{
await blobContainerClient.DeleteIfExistsAsync();
}
}
internal static async Task <Uri> GetOrCreateBlobContainerSasUriForLogAsync(string storageAccountConnectionString)
{
string containerName = GetAzureBlobContainerNameForLog();
var containerClient = new BlobContainerClient(storageAccountConnectionString, containerName);
if (!await containerClient.ExistsAsync())
{
await containerClient.CreateAsync();
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageAccountConnectionString);
var container = new CloudBlobContainer(containerClient.Uri, storageAccount.Credentials);
return(GetContainerSasUri(container));
}
public async Task DoDownload(CancellationToken cancellationToken)
{
BlobServiceClient blobServiceClient = new BlobServiceClient(Options.ConnectionString);
BlobContainerClient blobContainerClient = blobServiceClient.GetBlobContainerClient(Options.ContainerName);
if (!await blobContainerClient.ExistsAsync(cancellationToken))
{
throw new Exception($"Container {Options.ContainerName} doesn't exist!");
}
var blobs = blobContainerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: Options.BlobNamePrefix, cancellationToken: cancellationToken);
List <BlobDownloadWorkItem> items = new List <BlobDownloadWorkItem>();
await foreach (BlobItem b in blobs)
{
items.Add(new BlobDownloadWorkItem()
{
Id = b.Name,
Size = b.Properties.ContentLength.Value
});
}
var orderedItems = items.OrderBy(b => b.Id);
var fileSize = items.Sum(b => b.Size);
using (FileStream fileStream = File.Open(_targetFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Write))
{
fileStream.SetLength(fileSize);
}
List <Task> tasks = new List <Task>();
SemaphoreSlim throttle = new SemaphoreSlim(Options.NumOfChannels, Options.NumOfChannels);
long startOffset = 0;
foreach (var blobItem in orderedItems)
{
await throttle.WaitAsync(cancellationToken);
long offset = startOffset;
Task t = Task.Run(() => this.DownloadWorker(blobContainerClient, throttle, blobItem.Id, offset, blobItem.Size, cancellationToken));
tasks.Add(t);
startOffset += blobItem.Size;
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
public async Task InitializeAsync()
{
// Create the container and return a container client object
BlobContainerClient containerClient = _blobServiceClient.GetBlobContainerClient(_containerName);
_containerExists = await containerClient.ExistsAsync();
if (!_containerExists)
{
_containerClient = await _blobServiceClient.CreateBlobContainerAsync(_containerName);
}
else
{
_containerClient = containerClient;
}
}
public BlobTriggerEndToEndTests()
{
_nameResolver = new RandomNameResolver();
// pull from a default host
var host = new HostBuilder()
.ConfigureDefaultTestHost(b =>
{
b.AddAzureStorageBlobs().AddAzureStorageQueues();
})
.Build();
_blobServiceClient = new BlobServiceClient(TestEnvironment.PrimaryStorageAccountConnectionString);
_testContainer = _blobServiceClient.GetBlobContainerClient(_nameResolver.ResolveInString(SingleTriggerContainerName));
Assert.False(_testContainer.ExistsAsync().Result);
_testContainer.CreateAsync().Wait();
}
