/// <inheritdoc />
public async Task <AuditBlob[]> GetUnprocessedAudits(ScannerContainer container)
{
var uri = new Uri($"{this.config.AzureBlob.BasePath}/{container.Name}?{this.config.AzureBlob.Sas}");
var client = new Azure.Storage.Blobs.BlobContainerClient(uri);
var blobs = new List <AuditBlob>();
await foreach (var blob in client.GetBlobsAsync(BlobTraits.Metadata))
{
// skip all not metadata file.
if (!blob.Name.EndsWith("meta"))
{
continue;
}
if (blob.Metadata == null || !blob.Metadata.ContainsKey(ProcessedMetadataKey))
{
blobs.Add(new AuditBlob
{
Name = blob.Name,
ParentContainer = container,
});
}
}
return(blobs.ToArray());
}
/// <summary>
/// The method returns all audit files in the container.
/// </summary>
/// <param name="container">The container with audits.</param>
/// <returns>Array of Audit blobs.</returns>
public async Task MarkAllAuditsAsUnprocessed(ScannerContainer container)
{
var uri = new Uri($"{this.config.AzureBlob.BasePath}/{container.Name}?{this.config.AzureBlob.Sas}");
var client = new Azure.Storage.Blobs.BlobContainerClient(uri);
await foreach (var blob in client.GetBlobsAsync())
{
// skip all not metadata file.
if (!blob.Name.EndsWith("meta"))
{
continue;
}
await client.GetBlobClient(blob.Name).SetMetadataAsync(new Dictionary <string, string>());
}
}
/// <inheritdoc />
public async Task <int> MoveProcessedBlobsToArchive(CancellationToken cancellation)
{
Logger.Information("Archive processed audits was started");
var archiveContainerClient = this.storageClient.GetBlobContainerClient(ArchiveContainerName);
await archiveContainerClient.CreateIfNotExistsAsync(cancellationToken : cancellation);
var blobsToCopy = new List <(AuditBlob blob, CopyFromUriOperation copyOperation)>();
await foreach (var container in this.storageClient.GetBlobContainersAsync(cancellationToken: cancellation))
{
// skip "system" containers
if (container.Name.StartsWith("0-"))
{
continue;
}
var scannerContainerClient = this.storageClient.GetBlobContainerClient(container.Name);
var scannerContainer = new ScannerContainer(container.Name);
var regularBlobs = new List <string>();
var auditsToArchive = new Dictionary <string, List <string> >();
// Azure Storage stores all files inside container without any hierarchy. Thus joseki has to build the hierarchy here:
// - Find processed audits by presence of Processed tag on metadata file
// - All blobs, that have the same prefix as processed metadata file are moved to the same dictionary item
// - one dictionary item - is one audit folder
await foreach (var blob in scannerContainerClient.GetBlobsAsync(BlobTraits.Metadata, cancellationToken: cancellation))
{
// not metadata files are stored in memory to be placed to corresponding dictionary cells later
if (!blob.Name.EndsWith("meta") && blob.Name != container.Name)
{
regularBlobs.Add(blob.Name);
continue;
}
// if metadata file has processed tag - create a dictionary key-value pair
if (blob.Metadata == null || blob.Metadata.ContainsKey(AzureBlobStorageProcessor.ProcessedMetadataKey))
{
// the folder name is file_path without "/meta" at the end.
var auditFolderName = blob.Name[..^ 5];