private static async Task <bool> ShouldProceedAsync(CloudTable bottlerFilesTable, string prefix, string filePrefix, ILogger log)
{
try
{
var lockRecord = await LockTableEntity.GetLockRecordAsync(filePrefix, bottlerFilesTable);
if (lockRecord?.State == LockTableEntity.BatchState.Waiting)
{
// Update the lock record to mark it as in progress
lockRecord.State = LockTableEntity.BatchState.InProgress;
await bottlerFilesTable.ExecuteAsync(TableOperation.Replace(lockRecord));
return(true);
}
else
{
log.LogInformation($@"Validate for {prefix} skipped. State was {lockRecord?.State.ToString() ?? @"[null]"}.");
}
}
catch (StorageException)
{
log.LogInformation($@"Validate for {prefix} skipped (StorageException. Somebody else picked it up already.");
}
return(false);
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, @"post", Route = @"Validate")] HttpRequestMessage req, ILogger log)
{
log.LogTrace(@"ValidateFileSet run.");
if (!CloudStorageAccount.TryParse(Environment.GetEnvironmentVariable(@"CustomerBlobStorage"), out var storageAccount))
{
throw new Exception(@"Can't create a storage account accessor from app setting connection string, sorry!");
}
var payload = JObject.Parse(await req.Content.ReadAsStringAsync());
var prefix = payload["prefix"].ToString(); // This is the entire path w/ prefix for the file set
log.LogTrace($@"prefix: {prefix}");
var filePrefix = prefix.Substring(prefix.LastIndexOf('/') + 1);
log.LogTrace($@"filePrefix: {filePrefix}");
var lockTable = await Helpers.GetLockTableAsync();
if (!await ShouldProceedAsync(lockTable, prefix, filePrefix, log))
{
return(req.CreateResponse(HttpStatusCode.OK));
}
var blobClient = storageAccount.CreateCloudBlobClient();
var targetBlobs = await blobClient.ListBlobsAsync(WebUtility.UrlDecode(prefix));
var customerName = filePrefix.Split('_').First().Split('-').Last();
var errors = new List <string>();
var filesToProcess = payload["fileTypes"].Values <string>();
foreach (var blobDetails in targetBlobs)
{
var blob = await blobClient.GetBlobReferenceFromServerAsync(blobDetails.StorageUri.PrimaryUri);
var fileParts = CustomerBlobAttributes.Parse(blob.Uri.AbsolutePath);
if (!filesToProcess.Contains(fileParts.Filetype, StringComparer.OrdinalIgnoreCase))
{
log.LogTrace($@"{blob.Name} skipped. Isn't in the list of file types to process ({string.Join(", ", filesToProcess)}) for bottler '{customerName}'");
continue;
}
var lowerFileType = fileParts.Filetype.ToLowerInvariant();
log.LogInformation($@"Validating {lowerFileType}...");
uint numColumns = 0;
switch (lowerFileType)
{
case @"type5": // salestype
numColumns = 2;
break;
case @"type10": // mixedpack
case @"type4": // shipfrom
numColumns = 3;
break;
case @"type1": // channel
case @"type2": // customer
numColumns = 4;
break;
case @"type9": // itemdetail
case @"type3": // shipto
numColumns = 14;
break;
case @"type6": // salesdetail
numColumns = 15;
break;
case @"type8": // product
numColumns = 21;
break;
case @"type7": // sales
numColumns = 23;
break;
default:
throw new ArgumentOutOfRangeException(nameof(prefix), $@"Unhandled file type: {fileParts.Filetype}");
}
errors.AddRange(await ValidateCsvStructureAsync(blob, numColumns, lowerFileType));
}
try
{
await LockTableEntity.UpdateAsync(filePrefix, LockTableEntity.BatchState.Done, lockTable);
}
catch (StorageException)
{
log.LogWarning($@"That's weird. The lock for prefix {prefix} wasn't there. Shouldn't happen!");
return(req.CreateResponse(HttpStatusCode.OK));
}
if (errors.Any())
{
log.LogError($@"Errors found in batch {filePrefix}: {string.Join(@", ", errors)}");
// move files to 'invalid-set' folder
await MoveBlobsAsync(log, blobClient, targetBlobs, @"invalid-set");
return(req.CreateErrorResponse(HttpStatusCode.BadRequest, string.Join(@", ", errors)));
}
else
{
// move these files to 'valid-set' folder
await MoveBlobsAsync(log, blobClient, targetBlobs, @"valid-set");
log.LogInformation($@"Set {filePrefix} successfully validated and queued for further processing.");
return(req.CreateResponse(HttpStatusCode.OK));
}
}
public static async Task <HttpResponseMessage> Run([HttpTrigger(AuthorizationLevel.Function, @"post")] HttpRequestMessage req, ILogger log)
{
var payloadFromEventGrid = JToken.ReadFrom(new JsonTextReader(new StreamReader(await req.Content.ReadAsStreamAsync())));
dynamic eventGridSoleItem = (payloadFromEventGrid as JArray)?.SingleOrDefault();
if (eventGridSoleItem == null)
{
return(req.CreateErrorResponse(HttpStatusCode.BadRequest, $@"Expecting only one item in the Event Grid message"));
}
if (eventGridSoleItem.eventType == @"Microsoft.EventGrid.SubscriptionValidationEvent")
{
log.LogTrace(@"Event Grid Validation event received.");
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent($"{{ \"validationResponse\" : \"{((dynamic)payloadFromEventGrid)[0].data.validationCode}\" }}")
});
}
var newCustomerFile = Helpers.ParseEventGridPayload(eventGridSoleItem, log);
if (newCustomerFile == null)
{ // The request either wasn't valid (filename couldn't be parsed) or not applicable (put in to a folder other than /inbound)
return(req.CreateResponse(System.Net.HttpStatusCode.NoContent));
}
// get the prefix for the name so we can check for others in the same container with in the customer blob storage account
var prefix = newCustomerFile.BatchPrefix;
if (!CloudStorageAccount.TryParse(Environment.GetEnvironmentVariable(@"CustomerBlobStorage"), out var blobStorage))
{
throw new Exception(@"Can't create a storage account accessor from app setting connection string, sorry!");
}
var blobClient = blobStorage.CreateCloudBlobClient();
var matches = await blobClient.ListBlobsAsync(prefix : $@"{newCustomerFile.ContainerName}/inbound/{prefix}");
var matchNames = matches.Select(m => Path.GetFileNameWithoutExtension(blobClient.GetBlobReferenceFromServerAsync(m.StorageUri.PrimaryUri).GetAwaiter().GetResult().Name).Split('_').Last()).ToList();
var expectedFiles = Helpers.GetExpectedFilesForCustomer();
var filesStillWaitingFor = expectedFiles.Except(matchNames, new BlobFilenameVsDatabaseFileMaskComparer());
if (!filesStillWaitingFor.Any())
{
// Verify that this prefix isn't already in the lock table for processings
var lockTable = await Helpers.GetLockTableAsync();
var entriesMatchingPrefix = await LockTableEntity.GetLockRecordAsync(prefix, lockTable);
if (entriesMatchingPrefix != null)
{
log.LogInformation($@"Skipping. We've already queued the batch with prefix '{prefix}' for processing");
return(req.CreateResponse(HttpStatusCode.NoContent));
}
log.LogInformation(@"Got all the files! Moving on...");
try
{
await lockTable.ExecuteAsync(TableOperation.Insert(new LockTableEntity(prefix)));
}
catch (StorageException)
{
log.LogInformation($@"Skipping. We've already queued the batch with prefix '{prefix}' for processing");
return(req.CreateResponse(HttpStatusCode.NoContent));
}
using (var c = new HttpClient())
{
var jsonObjectForValidator =
$@"{{
""prefix"" : ""{newCustomerFile.ContainerName}/inbound/{prefix}"",
""fileTypes"" : [
{string.Join(", ", expectedFiles.Select(e => $@"""{e}"""))}
]
}}";
// call next step in functions with the prefix so it knows what to go grab
await c.PostAsync($@"{Environment.GetEnvironmentVariable(@"ValidateFunctionUrl")}", new StringContent(jsonObjectForValidator));
return(req.CreateResponse(HttpStatusCode.OK));
}
}
else
{
log.LogInformation($@"Still waiting for more files... Have {matches.Count()} file(s) from this customer ({newCustomerFile.CustomerName}) for batch {newCustomerFile.BatchPrefix}. Still need {string.Join(", ", filesStillWaitingFor)}");
return(req.CreateResponse(HttpStatusCode.NoContent));
}
}
