private static async Task <bool> ShouldProceedAsync(CloudTable bottlerFilesTable, string prefix, string filePrefix, ILogger log) { try { var lockRecord = await LockTableEntity.GetLockRecordAsync(filePrefix, bottlerFilesTable); if (lockRecord?.State == LockTableEntity.BatchState.Waiting) { // Update the lock record to mark it as in progress lockRecord.State = LockTableEntity.BatchState.InProgress; await bottlerFilesTable.ExecuteAsync(TableOperation.Replace(lockRecord)); return(true); } else { log.LogInformation($@"Validate for {prefix} skipped. State was {lockRecord?.State.ToString() ?? @"[null]"}."); } } catch (StorageException) { log.LogInformation($@"Validate for {prefix} skipped (StorageException. Somebody else picked it up already."); } return(false); }
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, @"post")] HttpRequestMessage req, ILogger log) { var payloadFromEventGrid = JToken.ReadFrom(new JsonTextReader(new StreamReader(await req.Content.ReadAsStreamAsync()))); dynamic eventGridSoleItem = (payloadFromEventGrid as JArray)?.SingleOrDefault(); if (eventGridSoleItem == null) { return(req.CreateErrorResponse(HttpStatusCode.BadRequest, $@"Expecting only one item in the Event Grid message")); } if (eventGridSoleItem.eventType == @"Microsoft.EventGrid.SubscriptionValidationEvent") { log.LogTrace(@"Event Grid Validation event received."); return(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent($"{{ \"validationResponse\" : \"{((dynamic)payloadFromEventGrid)[0].data.validationCode}\" }}") }); } var newCustomerFile = Helpers.ParseEventGridPayload(eventGridSoleItem, log); if (newCustomerFile == null) { // The request either wasn't valid (filename couldn't be parsed) or not applicable (put in to a folder other than /inbound) return(req.CreateResponse(System.Net.HttpStatusCode.NoContent)); } // get the prefix for the name so we can check for others in the same container with in the customer blob storage account var prefix = newCustomerFile.BatchPrefix; if (!CloudStorageAccount.TryParse(Environment.GetEnvironmentVariable(@"CustomerBlobStorage"), out var blobStorage)) { throw new Exception(@"Can't create a storage account accessor from app setting connection string, sorry!"); } var blobClient = blobStorage.CreateCloudBlobClient(); var matches = await blobClient.ListBlobsAsync(prefix : $@"{newCustomerFile.ContainerName}/inbound/{prefix}"); var matchNames = matches.Select(m => Path.GetFileNameWithoutExtension(blobClient.GetBlobReferenceFromServerAsync(m.StorageUri.PrimaryUri).GetAwaiter().GetResult().Name).Split('_').Last()).ToList(); var expectedFiles = Helpers.GetExpectedFilesForCustomer(); var filesStillWaitingFor = expectedFiles.Except(matchNames, new BlobFilenameVsDatabaseFileMaskComparer()); if (!filesStillWaitingFor.Any()) { // Verify that this prefix isn't already in the lock table for processings var lockTable = await Helpers.GetLockTableAsync(); var entriesMatchingPrefix = await LockTableEntity.GetLockRecordAsync(prefix, lockTable); if (entriesMatchingPrefix != null) { log.LogInformation($@"Skipping. We've already queued the batch with prefix '{prefix}' for processing"); return(req.CreateResponse(HttpStatusCode.NoContent)); } log.LogInformation(@"Got all the files! Moving on..."); try { await lockTable.ExecuteAsync(TableOperation.Insert(new LockTableEntity(prefix))); } catch (StorageException) { log.LogInformation($@"Skipping. We've already queued the batch with prefix '{prefix}' for processing"); return(req.CreateResponse(HttpStatusCode.NoContent)); } using (var c = new HttpClient()) { var jsonObjectForValidator = $@"{{ ""prefix"" : ""{newCustomerFile.ContainerName}/inbound/{prefix}"", ""fileTypes"" : [ {string.Join(", ", expectedFiles.Select(e => $@"""{e}"""))} ] }}"; // call next step in functions with the prefix so it knows what to go grab await c.PostAsync($@"{Environment.GetEnvironmentVariable(@"ValidateFunctionUrl")}", new StringContent(jsonObjectForValidator)); return(req.CreateResponse(HttpStatusCode.OK)); } } else { log.LogInformation($@"Still waiting for more files... Have {matches.Count()} file(s) from this customer ({newCustomerFile.CustomerName}) for batch {newCustomerFile.BatchPrefix}. Still need {string.Join(", ", filesStillWaitingFor)}"); return(req.CreateResponse(HttpStatusCode.NoContent)); } }