public async Task ProcessFileAsync(
[QueueTrigger("COIVD19FilesToProcess", Connection = "AzureWebJobsStorage")] string queueItem
, [Table("SyncStatus", Connection = "AzureWebJobsStorage")] CloudTable SyncStatusTable
, [Table("ErrorLogs", Connection = "AzureWebJobsStorage")] CloudTable ErrorLogsTable
, [Table("Covid19", Connection = "AzureWebJobsStorage")] CloudTable Covid19Table
, ILogger log)
{
await Covid19Table.CreateIfNotExistsAsync();
await ErrorLogsTable.CreateIfNotExistsAsync();
int count = 0;
List <ErrorLogEntity> Errors = new List <ErrorLogEntity>();
GitCommitFile file = JsonConvert.DeserializeObject <GitCommitFile>(queueItem);
var covidEntities = await _COVIDService.GetFileContentAsEntityListAsync(file);
log.LogInformation($"Start: Processing {file.raw_url}");
foreach (var entity in covidEntities)
{
try
{
var coividInserOperation = TableOperation.InsertOrMerge(entity);
var covidResult = await Covid19Table.ExecuteAsync(coividInserOperation);
count++;
}
catch (Exception ex)
{
var errorMsg = $"Source:{file.raw_url} -- Error inserting into Azure Table Storage {ex.Message}, Data: {JsonConvert.SerializeObject(entity)}";
log.LogError(errorMsg);
Errors.Add(new ErrorLogEntity(file.raw_url, errorMsg));
}
}
if (Errors.Any())
{
TableBatchOperation batchOperation = new TableBatchOperation();
Errors.ForEach(batchOperation.Insert);
await ErrorLogsTable.ExecuteBatchAsync(batchOperation);
}
log.LogInformation($"Done: Processing {file.raw_url}");
var operation = TableOperation.InsertOrMerge(new SyncStatusEntity("Process_File", count));
var result = await SyncStatusTable.ExecuteAsync(operation);
}
public async Task <IEnumerable <COVID19Entity> > GetFileContentAsEntityListAsync(GitCommitFile item)
{
var fileDataRes = await _client.GetAsync(item.raw_url);
var fileData = await fileDataRes.Content.ReadAsStringAsync();
var standardiseData = StandardiseFile(fileData);
return(ConvertDT2COVID19Entities(item.filename, standardiseData));
}
