Exemple #1
0
        public static void Run([CosmosDBTrigger(
                                    databaseName: "%CosmosDBNAME%",
                                    collectionName: "%CosmosHL7Collection%",
                                    ConnectionStringSetting = "CosmosDBConnection",
                                    CreateLeaseCollectionIfNotExists = true,
                                    LeaseCollectionPrefix = "srchupd",
                                    LeaseCollectionName = "leases")] IReadOnlyList <Document> input, ILogger log)
        {
            log.LogInformation("UpdateSearchIndexDiagReports function triggered");
            if (input != null && input.Count > 0)
            {
                SearchUtilities search = new SearchUtilities(log);
                log.LogInformation($"There were {input.Count} documents modified in DB..Running NLP/Seach pipeline for modified docs...");
                List <MedicalEntities> searcharr = new List <MedicalEntities>();
                foreach (Document d in input)
                {
                    string        json    = d.ToString();
                    StringBuilder builder = new StringBuilder();
                    var           obj     = JObject.Parse(json);
                    string        msgtype = Utilities.getFirstField(obj["hl7message"]["MSH"]["MSH.9"]);
                    if (msgtype.ToLower().Equals("oru") || msgtype.ToLower().Equals("mdm"))
                    {
                        if (obj["hl7message"]["OBX"] != null)
                        {
                            if (obj["hl7message"]["OBX"].Type == JTokenType.Array)
                            {
                                foreach (var obx in obj["hl7message"]["OBX"])
                                {
                                    if (Utilities.getFirstField(obx["OBX.2"]).Equals("TX") || Utilities.getFirstField(obx["OBX.2"]).Equals("FT"))
                                    {
                                        builder.Append(Utilities.getFirstField(obx["OBX.5"]));
                                    }
                                }
                            }
                            else
                            {
                                var obx = obj["hl7message"]["OBX"];
                                if (Utilities.getFirstField(obx["OBX.2"]).Equals("TX") || Utilities.getFirstField(obx["OBX.2"]).Equals("FT"))
                                {
                                    builder.Append(Utilities.getFirstField(obx["OBX.5"]));
                                }
                            }
                            string report = builder.ToString();
                            report = report.UnEscapeHL7();
                            report = report.Replace(@"\\", @"\");

                            string cogurl             = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
                            string responseFromServer = NLPUtilities.ExtractTextUsingCogServices(Encoding.UTF8.GetBytes(report), cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
                            if (string.IsNullOrEmpty(responseFromServer))
                            {
                                responseFromServer = NLPUtilities.ExtractTextUsingTIKA(Encoding.UTF8.GetBytes(report), Utilities.GetEnvironmentVariable("TIKAServerurl"));
                            }
                            if (responseFromServer.StartsWith("TIMEOUT~"))
                            {
                                log.LogTrace("{\"id\":\"" + Utilities.getFirstField(obj["id"]) + "\",\"status\":\"Timeout\",\"readresulturl\":\"" + responseFromServer.Split("~")[1] + "\"}");
                            }
                            if (string.IsNullOrEmpty(responseFromServer) || responseFromServer.Length < 3)
                            {
                                log.LogError($"TIKA Server may have failed to parse content {(string)obj["id"]}");
                            }
                            //Send Report to NLP
                            CTakesRequest creq = new CTakesRequest()
                            {
                                Content      = responseFromServer,
                                CTAKESUrl    = Utilities.GetEnvironmentVariable("CTAKESServerURL"),
                                UMLSUser     = Utilities.GetEnvironmentVariable("CTAKESUMLSUser"),
                                UMLSPassword = Utilities.GetEnvironmentVariable("CTAKESUMLSPassword"),
                                Format       = Utilities.GetEnvironmentVariable("CTAKESFormat"),
                            };
                            var result = NLPUtilities.ExtractMedicalEntities(creq);
                            result.Id       = (string)obj["id"];
                            result.Location = (string)obj["rhm"];
                            if (string.IsNullOrEmpty(result.ParsedText))
                            {
                                result.ParsedText = responseFromServer;
                            }
                            string doctype = "";
                            if (msgtype.Equals("MDM"))
                            {
                                doctype = Utilities.getFirstField(obj["hl7message"]["TXA"]["TXA.2"]);
                            }
                            if (doctype == null)
                            {
                                doctype = "";
                            }
                            result.DocumentType = doctype;
                            searcharr.Add(result);
                        }
                    }
                }
                if (searcharr.Count > 0)
                {
                    search.UploadMedicalEntities(searcharr.ToArray());
                }
            }
            log.LogInformation("UpdateSearchIndexDiagReports function completed");
        }
        public static void Run([BlobTrigger("%StorageAccountBlob%/ingest/documents/{name}", Connection = "StorageAccount")] Stream myBlob, string name, ILogger log)
        {
            log.LogInformation("NLP Extract Entities File triggered by ingest/documents/" + name);
            try
            {
                string coid   = name;
                int    dirend = coid.LastIndexOf("/");
                if (dirend > -1)
                {
                    coid = coid.Substring(dirend + 1);
                }
                int extbegin = coid.LastIndexOf(".");
                if (extbegin > -1)
                {
                    coid = coid.Substring(0, extbegin);
                }
                string loc       = "ingest/documents/" + name;
                byte[] byteArray = null;
                using (MemoryStream ms = new MemoryStream())
                {
                    myBlob.CopyTo(ms);
                    byteArray = ms.ToArray();
                }
                log.LogInformation("Calling CogServices/TIKA to Extract Text from hl7json/ingest/documents/" + name);
                string cogurl = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
                log.LogInformation("Trying CogServices...");
                string responseFromServer = NLPUtilities.ExtractTextUsingCogServices(byteArray, cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
                if (string.IsNullOrEmpty(responseFromServer))
                {
                    log.LogInformation("No extract Trying TIKA...");
                    responseFromServer = NLPUtilities.ExtractTextUsingTIKA(byteArray, Utilities.GetEnvironmentVariable("TIKAServerurl"));
                }
                if (responseFromServer.StartsWith("TIMEOUT~"))
                {
                    log.LogTrace("CogServiceExtract Timeout: {\"id\":\"" + coid + "\",\"status\":\"Timeout\",\"readresulturl\":\"" + responseFromServer.Split("~")[1] + "\"}");
                }

                //string responseFromServer = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length);
                log.LogInformation("Extracting Medical Reports from hl7json/ingest/documents/" + name);
                //Extract Reports From Content (Auto-Detect Medical Exchange Formats (CDA, HL7, FHIR))
                List <string>          medreports = NLPUtilities.ExtractMedicalReportData(responseFromServer, log);
                List <MedicalEntities> retVal     = new List <MedicalEntities>();
                foreach (string medreport in medreports)
                {
                    CTakesRequest creq = new CTakesRequest()
                    {
                        Content      = medreport,
                        CTAKESUrl    = Utilities.GetEnvironmentVariable("CTAKESServerURL"),
                        UMLSUser     = Utilities.GetEnvironmentVariable("CTAKESUMLSUser"),
                        UMLSPassword = Utilities.GetEnvironmentVariable("CTAKESUMLSPassword"),
                        Format       = Utilities.GetEnvironmentVariable("CTAKESFormat"),
                    };
                    log.LogInformation("Calling CTAKES to extract medical entities from hl7json/ingest/documents/" + name);
                    var result = NLPUtilities.ExtractMedicalEntities(creq);
                    result.Id           = coid;
                    result.Location     = loc;
                    result.DocumentType = name;
                    retVal.Add(result);
                }
                log.LogInformation("Updateing search index with content and medical entities from hl7json/ingest/documents/" + name);
                SearchUtilities su = new SearchUtilities(log);
                su.UploadMedicalEntities(retVal.ToArray());
                log.LogInformation("Succesfully Completed processing of hl7json/ingest/documents/" + name);
            }
            catch (System.Exception e)
            {
                log.LogError(e, e.Message);
            }
        }
Exemple #3
0
        public static List <string> ExtractMedicalReportData(string report, ILogger log)
        {
            List <string> retVal = new List <string>();

            if (report.TrimStart().StartsWith("MSH|"))
            {
                string[]      segs = report.Replace("\r\n", "\r").Split("\r");
                StringBuilder sb   = new StringBuilder();
                sb.Append(segs[0]).Append("\r");
                for (int i = 1; i < segs.Length; i++)
                {
                    if (segs[i].StartsWith("MSH"))
                    {
                        string s = ExtractReportFromHL7(sb.ToString());
                        if (!String.IsNullOrEmpty(s))
                        {
                            retVal.Add(s);
                        }
                        sb.Clear();
                    }
                    sb.Append(segs[i]).Append("\r");
                }
                if (sb.Length > 0)
                {
                    string s = ExtractReportFromHL7(sb.ToString());
                    if (!String.IsNullOrEmpty(s))
                    {
                        string cogurl     = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
                        string extractstr = NLPUtilities.ExtractTextUsingCogServices(Encoding.UTF8.GetBytes(s), cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
                        if (string.IsNullOrEmpty(extractstr))
                        {
                            extractstr = NLPUtilities.ExtractTextUsingTIKA(Encoding.UTF8.GetBytes(s), Utilities.GetEnvironmentVariable("TIKAServerurl"));
                        }
                        retVal.Add(extractstr);
                    }
                }

                //FHIR HTML
            }
            else if (report.TrimStart().StartsWith("{") || report.TrimStart().StartsWith("["))
            {
                try
                {
                    if (report.TrimStart().StartsWith("{"))
                    {
                        report = "[" + report + "]";
                    }
                    var dr = JArray.Parse(report);
                    foreach (JObject obj in dr.Children())
                    {
                        if (obj["text"] != null)
                        {
                            retVal.Add(obj["text"]["div"].GetFirstField());
                        }
                    }
                }
                catch (Exception e)
                {
                    log.LogError(e, e.Message);
                }
            }
            else
            {
                retVal.Add(report);
            }
            return(retVal);
        }
        public static async Task <IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
            ILogger log)
        {
            log.LogInformation("NLP Extract Entities Http called");
            try
            {
                string doctype = req.Query["doctype"];
                if (doctype == null)
                {
                    doctype = "Unkown Doc Type";
                }
                string handwritten = req.Query["handwritten"];
                string coid        = req.Query["id"];
                if (coid == null)
                {
                    coid = Guid.NewGuid().ToString();
                }
                string loc = req.Query["location"];
                if (loc == null)
                {
                    loc = "Adhoc Document Addition";
                }
                string updatesearch = req.Query["updatesearch"];
                byte[] byteArray    = null;
                // Read the post data into byte array
                using (var stream = new MemoryStream())
                {
                    await req.Body.CopyToAsync(stream);

                    byteArray = stream.ToArray();
                }
                string cogurl = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
                if (handwritten != null)
                {
                    cogurl += "?mode=Handwritten";
                }
                string responseFromServer = NLPUtilities.ExtractTextUsingCogServices(byteArray, cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
                if (string.IsNullOrEmpty(responseFromServer))
                {
                    responseFromServer = NLPUtilities.ExtractTextUsingTIKA(byteArray, Utilities.GetEnvironmentVariable("TIKAServerurl"));
                }
                if (responseFromServer.StartsWith("TIMEOUT~"))
                {
                    return(new JsonResult(JObject.Parse("{\"id\":\"" + coid + "\",\"status\":\"Timeout\",\"readresulturl\":\"" + responseFromServer.Split("~")[1] + "\"}")));
                }

                //Extract Reports From Content (Auto-Detect Medical Exchange Formats (CDA, HL7, FHIR))
                List <string>          medreports = NLPUtilities.ExtractMedicalReportData(responseFromServer, log);
                List <MedicalEntities> retVal     = new List <MedicalEntities>();
                foreach (string medreport in medreports)
                {
                    CTakesRequest creq = new CTakesRequest()
                    {
                        Content      = medreport,
                        CTAKESUrl    = Utilities.GetEnvironmentVariable("CTAKESServerURL"),
                        UMLSUser     = Utilities.GetEnvironmentVariable("CTAKESUMLSUser"),
                        UMLSPassword = Utilities.GetEnvironmentVariable("CTAKESUMLSPassword"),
                        Format       = Utilities.GetEnvironmentVariable("CTAKESFormat"),
                    };

                    var result = NLPUtilities.ExtractMedicalEntities(creq);
                    result.Id           = coid;
                    result.Location     = loc;
                    result.DocumentType = doctype;
                    retVal.Add(result);
                }
                if (updatesearch != null)
                {
                    SearchUtilities su = new SearchUtilities(log);
                    su.UploadMedicalEntities(retVal.ToArray());
                }

                return(new JsonResult(retVal));
            }
            catch (System.Exception e)
            {
                log.LogError(e, e.Message);
                return(new System.Web.Http.InternalServerErrorResult());
            }
        }