public static async Task Run([EventGridTrigger] EventGridEvent blobCreatedEvent, [Blob("{data.url}", FileAccess.Read, Connection = "FBI-STORAGEACCT")] Stream myBlob, ILogger log) { bool trbundles = true; string strbundles = System.Environment.GetEnvironmentVariable("FBI-TRANSFORMBUNDLES"); if (!string.IsNullOrEmpty(strbundles) && (strbundles.ToLower().Equals("no") || strbundles.ToLower().Equals("false"))) { trbundles = false; } StorageBlobCreatedEventData createdEvent = ((JObject)blobCreatedEvent.Data).ToObject <StorageBlobCreatedEventData>(); string name = createdEvent.Url.Substring(createdEvent.Url.LastIndexOf('/') + 1); if (myBlob == null) { return; } log.LogInformation($"ImportFHIRBUndles: Processing file Name:{name} \n Size: {myBlob.Length}"); var cbclient = StorageUtils.GetCloudBlobClient(System.Environment.GetEnvironmentVariable("FBI-STORAGEACCT")); StreamReader reader = new StreamReader(myBlob); var text = await reader.ReadToEndAsync(); var trtext = (trbundles ? FHIRUtils.TransformBundle(text, log) : text); var fhirbundle = await FHIRUtils.CallFHIRServer("", trtext, HttpMethod.Post, log); var result = LoadErrorsDetected(trtext, fhirbundle, name, log); //Bundle Post was Throttled we can retry if (!fhirbundle.Success && fhirbundle.Status == System.Net.HttpStatusCode.TooManyRequests) { //Currently cannot use retry hints with EventGrid Trigger function bindings so we will throw and exception to enter eventgrid retry logic for FHIR Server throttling and do //our own dead letter for internal errors or unrecoverable conditions log.LogInformation($"ImportFHIRBUndles File Name:{name} is throttled..."); throw new Exception($"ImportFHIRBUndles: Transient Error File: {name}...Entering eventgrid retry process until success or ultimate failure to dead letter if configured."); } //No Errors move to processed container if (fhirbundle.Success && ((JArray)result["errors"]).Count == 0 && ((JArray)result["throttled"]).Count == 0) { await StorageUtils.MoveTo(cbclient, "bundles", "bundlesprocessed", name, $"{name}.processed", log); await StorageUtils.WriteStringToBlob(cbclient, "bundlesprocessed", $"{name}.processed.result", fhirbundle.Content, log); log.LogInformation($"ImportFHIRBUndles Processed file Name:{name}"); } //Handle Errors from FHIR Server of proxy if (!fhirbundle.Success || ((JArray)result["errors"]).Count > 0) { await StorageUtils.MoveTo(cbclient, "bundles", "bundleserr", name, $"{name}.err", log); await StorageUtils.WriteStringToBlob(cbclient, "bundleserr", $"{name}.err.response", fhirbundle.Content, log); await StorageUtils.WriteStringToBlob(cbclient, "bundleserr", $"{name}.err.actionneeded", result.ToString(), log); log.LogInformation($"ImportFHIRBUndles File Name:{name} had errors. Moved to deadletter bundleserr directory"); } //Handle Throttled Requests inside of bundle so we will create a new bundle to retry if (fhirbundle.Success && ((JArray)result["throttled"]).Count > 0) { var nb = NDJSONConverter.initBundle(); nb["entry"] = result["throttled"]; string fn = $"retry{Guid.NewGuid().ToString().Replace("-","")}.json"; await StorageUtils.MoveTo(cbclient, "bundles", "bundlesprocessed", name, $"{name}.processed", log); await StorageUtils.WriteStringToBlob(cbclient, "bundlesprocessed", $"{name}.processed.result", fhirbundle.Content, log); await StorageUtils.WriteStringToBlob(cbclient, "bundles", fn, nb.ToString(), log); log.LogInformation($"ImportFHIRBUndles File Name:{name} had throttled resources in response bundle. Moved to processed..Created retry bunde {fn}"); } }
public static async Task Run([EventGridTrigger] EventGridEvent blobCreatedEvent, [Blob("{data.url}", FileAccess.Read, Connection = "FBI-STORAGEACCT")] Stream myBlob, ILogger log) { int maxresourcesperbundle = 200; var cbclient = StorageUtils.GetCloudBlobClient(System.Environment.GetEnvironmentVariable("FBI-STORAGEACCT")); string mrbundlemax = System.Environment.GetEnvironmentVariable("FBI-MAXRESOURCESPERBUNDLE"); if (!string.IsNullOrEmpty(mrbundlemax)) { if (!int.TryParse(mrbundlemax, out maxresourcesperbundle)) { maxresourcesperbundle = 200; } } StorageBlobCreatedEventData createdEvent = ((JObject)blobCreatedEvent.Data).ToObject <StorageBlobCreatedEventData>(); log.LogInformation($"NDJSONConverter: Processing blob at {createdEvent.Url}..."); string name = createdEvent.Url.Substring(createdEvent.Url.LastIndexOf('/') + 1); JObject rv = initBundle(); int linecnt = 0; int total = 0; int bundlecnt = 0; int errcnt = 0; int fileno = 1; //Stream myBlob = null; StringBuilder errsb = new StringBuilder(); using (StreamReader reader = new StreamReader(myBlob)) { string line; while ((line = reader.ReadLine()) != null) { linecnt++; JObject res = null; try { res = JObject.Parse(line); addResource(rv, res); bundlecnt++; total++; } catch (Exception e) { log.LogError($"NDJSONConverter: File {name} is in error or contains invalid JSON at line number {linecnt}:{e.Message}"); errsb.Append($"{line}\n"); errcnt++; } if (bundlecnt >= maxresourcesperbundle) { await StorageUtils.WriteStringToBlob(cbclient, "bundles", $"{name}-{fileno++}.json", rv.ToString(), log); bundlecnt = 0; rv = null; rv = initBundle(); } } if (bundlecnt > 0) { await StorageUtils.WriteStringToBlob(cbclient, "bundles", $"{name}-{fileno++}.json", rv.ToString(), log); } await StorageUtils.MoveTo(cbclient, "ndjson", "ndjsonprocessed", name, $"{name}.processed", log); if (errcnt > 0) { await StorageUtils.WriteStringToBlob(cbclient, "ndjsonerr", $"{name}.err", errsb.ToString(), log); } log.LogInformation($"NDJSONConverter: Processing file {name} completed with {total} resources created in {fileno-1} bundles with {errcnt} errors..."); } }