public IAsyncEnumerable <AlphaNumericString> GetTags()
{
return(new AsyncEnumerable <AlphaNumericString>(async yield =>
{
var containers = await _client.ListContainersAsync();
var tags = containers
.Select(c => c.Name == EmptyContainer ? AlphaNumericString.Empty : new AlphaNumericString(c.Name));
foreach (var tag in tags)
{
var file = await GetFiles(tag).FirstOrDefaultAsync(yield.CancellationToken);
if (file != null)
{
await yield.ReturnAsync(tag);
}
}
}));
}
private async Task <List <ScoreRecord> > InspectProcessingOrchestrations()
{
var results = new List <ScoreRecord>();
log.LogTrace($"Checking processing orchestrations");
var containers = await orchestrationBlobClient.ListContainersAsync();
int score = containers.Count();
log.LogTrace($"{score} orchestration containers were present");
if (score > 500)
{
score = 500;
}
results.Add(new ScoreRecord {
Type = $"Processing", Notes = $"{containers.Count()} processing orchestration containers were detected (1 point each, max 500)", Score = score
});
return(results);
}
private async Task <List <ScoreRecord> > InspectTrainingStorage()
{
var results = new List <ScoreRecord>();
// Check containers have been created for each document type where a model needs to be trained
log.LogTrace($"Checking that processing containers have been created in training account");
var containers = await trainingBlobClient.ListContainersAsync();
var documentTypesForChallenge = Environment.GetEnvironmentVariable("DocumentTypesForChallenge").Split(',').ToList();
foreach (var item in documentTypesForChallenge)
{
log.LogTrace($"Checking container {item}");
if (containers.Where(c => c.Name == item).Count() == 1)
{
log.LogTrace($"Container {item} has been created");
results.Add(new ScoreRecord {
Type = $"Training", Notes = $"Container for document type {item} was detected", Score = 150 / documentTypesForChallenge.Count
});
}
}
// Check that training documents have been uploaded
log.LogTrace($"Checking that training documents have been uploaded to containers");
foreach (var item in containers)
{
log.LogTrace($"Container {item.Name}");
int i = 0; int j = 0;
var allBlobs = await item.ListBlobsAsync();
foreach (IListBlobItem blob in allBlobs)
{
string name = blob.Uri.Segments.Last();
if (name.ToLower().EndsWith(".pdf"))
{
log.LogTrace($"Document {name} detected");
if (i < 10)
{
i++;
}
}
if (name.ToLower().EndsWith(".pdf.labels.json"))
{
log.LogTrace($"Document {name} detected");
if (j < 10)
{
j++;
}
}
if (name.ToLower().EndsWith(".fott"))
{
log.LogTrace($"Document {name} detected");
results.Add(new ScoreRecord {
Type = $"Training", Notes = $"Recognizer labelling project has been created {name}", Score = 500
});
}
}
if (i > 0)
{
results.Add(new ScoreRecord {
Type = $"Training", Notes = $"{i} raw documents for document type {item.Name} are present (10 points each)", Score = 10 * i
});
}
if (j > 0)
{
results.Add(new ScoreRecord {
Type = $"Training", Notes = $"{j} labelled documents for document type {item.Name} are present (25 points each)", Score = 25 * j
});
}
}
return(results);
}