public static async System.Threading.Tasks.Task <HttpResponseMessage> RunAsync([HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequestMessage req, [DurableClient] IDurableClient starter, ILogger log) { var inputToFunction = JToken.ReadFrom(new JsonTextReader(new StreamReader(await req.Content.ReadAsStreamAsync()))); dynamic eventGridSoleItem = (inputToFunction as JArray)?.SingleOrDefault(); if (eventGridSoleItem == null) { return(req.CreateResponse(HttpStatusCode.BadRequest, @"Expecting only one item in the Event Grid message")); } if (eventGridSoleItem.eventType == @"Microsoft.EventGrid.SubscriptionValidationEvent") { log.LogTrace(@"Event Grid Validation event received."); return(req.CreateCompatibleResponse(HttpStatusCode.OK, $"{{ \"validationResponse\" : \"{((dynamic)inputToFunction)[0].data.validationCode}\" }}")); } CustomerBlobAttributes newCustomerFile = Helpers.ParseEventGridPayload(eventGridSoleItem, log); if (newCustomerFile == null) { // The request either wasn't valid (filename couldn't be parsed) or not applicable (put in to a folder other than /inbound) return(req.CreateResponse(HttpStatusCode.NoContent)); } string customerName = newCustomerFile.CustomerName, name = newCustomerFile.Filename, containerName = newCustomerFile.ContainerName; log.LogInformation($@"Processing new file. customer: {customerName}, filename: {name}"); // get the prefix for the name so we can check for others in the same container with in the customer blob storage account var prefix = newCustomerFile.BatchPrefix; await starter.SignalEntityAsync <IBatchEntity>(prefix, b => b.NewFile(newCustomerFile.FullUrl)); return(req.CreateResponse(HttpStatusCode.Accepted)); }
private static async Task <IEnumerable <string> > ValidateCsvStructureAsync(ICloudBlob blob, uint requiredNumberOfColumnsPerLine, string filetypeDescription) { var errs = new List <string>(); try { using (var blobReader = new StreamReader(await blob.OpenReadAsync(new AccessCondition(), new BlobRequestOptions(), new OperationContext()))) { var fileAttributes = CustomerBlobAttributes.Parse(blob.Uri.AbsolutePath); for (var lineNumber = 0; !blobReader.EndOfStream; lineNumber++) { var errorPrefix = $@"{filetypeDescription} file '{fileAttributes.Filename}' Record {lineNumber}"; var line = blobReader.ReadLine(); var fields = line.Split(','); if (fields.Length != requiredNumberOfColumnsPerLine) { errs.Add($@"{errorPrefix} is malformed. Should have {requiredNumberOfColumnsPerLine} values; has {fields.Length}"); continue; } for (var i = 0; i < fields.Length; i++) { errorPrefix = $@"{errorPrefix} Field {i}"; var field = fields[i]; // each field must be enclosed in double quotes if (field[0] != '"' || field.Last() != '"') { errs.Add($@"{errorPrefix}: value ({field}) is not enclosed in double quotes ("")"); continue; } } } // Validate file is UTF-8 encoded if (!blobReader.CurrentEncoding.BodyName.Equals("utf-8", StringComparison.OrdinalIgnoreCase)) { errs.Add($@"{blob.Name} is not UTF-8 encoded"); } } } catch (StorageException storEx) { SwallowStorage404(storEx); } return(errs); }
public static async Task Run([OrchestrationTrigger] IDurableOrchestrationContext context, ILogger log) #endif { if (!context.IsReplaying) { context.Log(log, $@"EnsureAllFiles STARTING - InstanceId: {context.InstanceId}"); } else { context.Log(log, $@"EnsureAllFiles REPLAYING"); } dynamic eventGridSoleItem = context.GetInputAsJson(); CustomerBlobAttributes newCustomerFile = Helpers.ParseEventGridPayload(eventGridSoleItem, log); if (newCustomerFile == null) { // The request either wasn't valid (filename couldn't be parsed) or not applicable (put in to a folder other than /inbound) return; } var expectedFiles = Helpers.GetExpectedFilesForCustomer(); var filesStillWaitingFor = new HashSet <string>(expectedFiles); var filename = newCustomerFile.Filename; while (filesStillWaitingFor.Any()) { filesStillWaitingFor.Remove(Path.GetFileNameWithoutExtension(filename).Split('_').Last()); if (filesStillWaitingFor.Count == 0) { break; } context.Log(log, $@"Still waiting for more files... Still need {string.Join(", ", filesStillWaitingFor)} for customer {newCustomerFile.CustomerName}, batch {newCustomerFile.BatchPrefix}"); filename = await context.WaitForExternalEvent <string>(@"newfile"); context.Log(log, $@"Got new file via event: {filename}"); } // Verify that this prefix isn't already in the lock table for processings context.Log(log, @"Got all the files! Moving on..."); // call next step in functions with the prefix so it knows what to go grab await context.CallActivityAsync(@"ValidateFileSet", new { prefix = $@"{newCustomerFile.ContainerName}/inbound/{newCustomerFile.BatchPrefix}", fileTypes = expectedFiles }); }
public async Task NewFile(string fileUri) { var newCustomerFile = CustomerBlobAttributes.Parse(fileUri); _logger.LogInformation($@"Got new file via event: {newCustomerFile.Filename}"); this.ReceivedFileTypes.Add(newCustomerFile.Filetype); _logger.LogTrace($@"Actor '{_id}' got file '{newCustomerFile.Filetype}'"); var filesStillWaitingFor = Helpers.GetExpectedFilesForCustomer().Except(this.ReceivedFileTypes); if (filesStillWaitingFor.Any()) { _logger.LogInformation($@"Still waiting for more files... Still need {string.Join(", ", filesStillWaitingFor)} for customer {newCustomerFile.CustomerName}, batch {newCustomerFile.BatchPrefix}"); } else { _logger.LogInformation(@"Got all the files! Moving on..."); // call next step in functions with the prefix so it knows what to go grab await Helpers.DoValidationAsync($@"{newCustomerFile.ContainerName}/inbound/{newCustomerFile.BatchPrefix}", _logger); } }
public static CustomerBlobAttributes ParseEventGridPayload(dynamic eventGridItem, ILogger log) { if (eventGridItem.eventType == @"Microsoft.Storage.BlobCreated" && eventGridItem.data.api == @"PutBlob" && eventGridItem.data.contentType == @"text/csv") { try { var retVal = CustomerBlobAttributes.Parse((string)eventGridItem.data.url); if (retVal != null && !retVal.ContainerName.Equals(retVal.CustomerName)) { throw new ArgumentException($@"File '{retVal.Filename}' uploaded to container '{retVal.ContainerName}' doesn't have the right prefix: the first token in the filename ({retVal.CustomerName}) must be the customer name, which should match the container name", nameof(eventGridItem)); } return(retVal); } catch (Exception ex) { log.LogError(@"Error parsing Event Grid payload", ex); } } return(null); }
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, @"post", Route = @"Validate")] HttpRequestMessage req, ILogger log) { log.LogTrace(@"ValidateFileSet run."); if (!CloudStorageAccount.TryParse(Environment.GetEnvironmentVariable(@"CustomerBlobStorage"), out var storageAccount)) { throw new Exception(@"Can't create a storage account accessor from app setting connection string, sorry!"); } var payload = JObject.Parse(await req.Content.ReadAsStringAsync()); var prefix = payload["prefix"].ToString(); // This is the entire path w/ prefix for the file set log.LogTrace($@"prefix: {prefix}"); var filePrefix = prefix.Substring(prefix.LastIndexOf('/') + 1); log.LogTrace($@"filePrefix: {filePrefix}"); var lockTable = await Helpers.GetLockTableAsync(); if (!await ShouldProceedAsync(lockTable, prefix, filePrefix, log)) { return(req.CreateResponse(HttpStatusCode.OK)); } var blobClient = storageAccount.CreateCloudBlobClient(); var targetBlobs = await blobClient.ListBlobsAsync(WebUtility.UrlDecode(prefix)); var customerName = filePrefix.Split('_').First().Split('-').Last(); var errors = new List <string>(); var filesToProcess = payload["fileTypes"].Values <string>(); foreach (var blobDetails in targetBlobs) { var blob = await blobClient.GetBlobReferenceFromServerAsync(blobDetails.StorageUri.PrimaryUri); var fileParts = CustomerBlobAttributes.Parse(blob.Uri.AbsolutePath); if (!filesToProcess.Contains(fileParts.Filetype, StringComparer.OrdinalIgnoreCase)) { log.LogTrace($@"{blob.Name} skipped. Isn't in the list of file types to process ({string.Join(", ", filesToProcess)}) for bottler '{customerName}'"); continue; } var lowerFileType = fileParts.Filetype.ToLowerInvariant(); log.LogInformation($@"Validating {lowerFileType}..."); uint numColumns = 0; switch (lowerFileType) { case @"type5": // salestype numColumns = 2; break; case @"type10": // mixedpack case @"type4": // shipfrom numColumns = 3; break; case @"type1": // channel case @"type2": // customer numColumns = 4; break; case @"type9": // itemdetail case @"type3": // shipto numColumns = 14; break; case @"type6": // salesdetail numColumns = 15; break; case @"type8": // product numColumns = 21; break; case @"type7": // sales numColumns = 23; break; default: throw new ArgumentOutOfRangeException(nameof(prefix), $@"Unhandled file type: {fileParts.Filetype}"); } errors.AddRange(await ValidateCsvStructureAsync(blob, numColumns, lowerFileType)); } try { await LockTableEntity.UpdateAsync(filePrefix, LockTableEntity.BatchState.Done, lockTable); } catch (StorageException) { log.LogWarning($@"That's weird. The lock for prefix {prefix} wasn't there. Shouldn't happen!"); return(req.CreateResponse(HttpStatusCode.OK)); } if (errors.Any()) { log.LogError($@"Errors found in batch {filePrefix}: {string.Join(@", ", errors)}"); // move files to 'invalid-set' folder await MoveBlobsAsync(log, blobClient, targetBlobs, @"invalid-set"); return(req.CreateErrorResponse(HttpStatusCode.BadRequest, string.Join(@", ", errors))); } else { // move these files to 'valid-set' folder await MoveBlobsAsync(log, blobClient, targetBlobs, @"valid-set"); log.LogInformation($@"Set {filePrefix} successfully validated and queued for further processing."); return(req.CreateResponse(HttpStatusCode.OK)); } }
public static async Task <bool> DoValidationAsync(string prefix, ILogger logger = null) { logger?.LogTrace(@"ValidateFileSet run."); if (!CloudStorageAccount.TryParse(Environment.GetEnvironmentVariable(@"CustomerBlobStorage"), out var storageAccount)) { throw new Exception(@"Can't create a storage account accessor from app setting connection string, sorry!"); } logger?.LogTrace($@"prefix: {prefix}"); var filePrefix = prefix.Substring(prefix.LastIndexOf('/') + 1); logger?.LogTrace($@"filePrefix: {filePrefix}"); var blobClient = storageAccount.CreateCloudBlobClient(); var targetBlobs = await blobClient.ListBlobsAsync(WebUtility.UrlDecode(prefix)); var customerName = filePrefix.Split('_').First().Split('-').Last(); var errors = new List <string>(); var expectedFiles = Helpers.GetExpectedFilesForCustomer(); foreach (var blobDetails in targetBlobs) { var blob = await blobClient.GetBlobReferenceFromServerAsync(blobDetails.StorageUri.PrimaryUri); var fileParts = CustomerBlobAttributes.Parse(blob.Uri.AbsolutePath); if (!expectedFiles.Contains(fileParts.Filetype, StringComparer.OrdinalIgnoreCase)) { logger?.LogTrace($@"{blob.Name} skipped. Isn't in the list of file types to process ({string.Join(", ", expectedFiles)}) for customer '{customerName}'"); continue; } var lowerFileType = fileParts.Filetype.ToLowerInvariant(); uint numColumns = 0; switch (lowerFileType) { case @"type5": // salestype numColumns = 2; break; case @"type10": // mixed case @"type4": // shipfrom numColumns = 3; break; case @"type1": // channel case @"type2": // customer numColumns = 4; break; case @"type9": // itemdetail numColumns = 5; break; case @"type3": // shipto numColumns = 14; break; case @"type6": // salesdetail numColumns = 15; break; case @"type8": // product numColumns = 21; break; case @"type7": // sales numColumns = 23; break; default: throw new ArgumentOutOfRangeException(nameof(prefix), $@"Unhandled file type: {fileParts.Filetype}"); } errors.AddRange(await ValidateCsvStructureAsync(blob, numColumns, lowerFileType)); } if (errors.Any()) { logger.LogError($@"Errors found in batch {filePrefix}: {string.Join(@", ", errors)}"); // move files to 'invalid-set' folder await Helpers.MoveBlobsAsync(blobClient, targetBlobs, @"invalid-set", logger); return(false); } else { // move these files to 'valid-set' folder await Helpers.MoveBlobsAsync(blobClient, targetBlobs, @"valid-set", logger); logger.LogInformation($@"Set {filePrefix} successfully validated and queued for further processing."); return(true); } }
public static async System.Threading.Tasks.Task <HttpResponseMessage> RunAsync( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequestMessage req, [DurableClient] IDurableClient starter, ILogger log) #endif { var inputToFunction = JToken.ReadFrom(new JsonTextReader(new StreamReader(await req.Content.ReadAsStreamAsync()))); dynamic eventGridSoleItem = (inputToFunction as JArray)?.SingleOrDefault(); if (eventGridSoleItem == null) { return(req.CreateCompatibleResponse(HttpStatusCode.BadRequest, @"Expecting only one item in the Event Grid message")); } if (eventGridSoleItem.eventType == @"Microsoft.EventGrid.SubscriptionValidationEvent") { log.LogTrace(@"Event Grid Validation event received."); return(req.CreateCompatibleResponse(HttpStatusCode.OK, $"{{ \"validationResponse\" : \"{((dynamic)inputToFunction)[0].data.validationCode}\" }}")); } CustomerBlobAttributes newCustomerFile = Helpers.ParseEventGridPayload(eventGridSoleItem, log); if (newCustomerFile == null) { // The request either wasn't valid (filename couldn't be parsed) or not applicable (put in to a folder other than /inbound) return(req.CreateCompatibleResponse(HttpStatusCode.NoContent)); } string customerName = newCustomerFile.CustomerName, name = newCustomerFile.Filename; starter.Log(log, $@"Processing new file. customer: {customerName}, filename: {name}"); // get the prefix for the name so we can check for others in the same container with in the customer blob storage account var prefix = newCustomerFile.BatchPrefix; var instanceForPrefix = await starter.GetStatusAsync(prefix); if (instanceForPrefix == null) { starter.Log(log, $@"New instance needed for prefix '{prefix}'. Starting..."); var retval = await starter.StartNewAsync(@"EnsureAllFiles", prefix, eventGridSoleItem); starter.Log(log, $@"Started. {retval}"); } else { starter.Log(log, $@"Instance already waiting. Current status: {instanceForPrefix.RuntimeStatus}. Firing 'newfile' event..."); if (instanceForPrefix.RuntimeStatus != OrchestrationRuntimeStatus.Running) { await starter.TerminateAsync(prefix, @"bounce"); var retval = await starter.StartNewAsync(@"EnsureAllFiles", prefix, eventGridSoleItem); starter.Log(log, $@"Restarted listener for {prefix}. {retval}"); } else { await starter.RaiseEventAsync(prefix, @"newfile", newCustomerFile.Filename); } } return(starter.CreateCheckStatusResponse(req, prefix)); }