public static void Run([CosmosDBTrigger(
databaseName: "%CosmosDBNAME%",
collectionName: "%CosmosHL7Collection%",
ConnectionStringSetting = "CosmosDBConnection",
CreateLeaseCollectionIfNotExists = true,
LeaseCollectionPrefix = "srchupd",
LeaseCollectionName = "leases")] IReadOnlyList <Document> input, ILogger log)
{
log.LogInformation("UpdateSearchIndexDiagReports function triggered");
if (input != null && input.Count > 0)
{
SearchUtilities search = new SearchUtilities(log);
log.LogInformation($"There were {input.Count} documents modified in DB..Running NLP/Seach pipeline for modified docs...");
List <MedicalEntities> searcharr = new List <MedicalEntities>();
foreach (Document d in input)
{
string json = d.ToString();
StringBuilder builder = new StringBuilder();
var obj = JObject.Parse(json);
string msgtype = Utilities.getFirstField(obj["hl7message"]["MSH"]["MSH.9"]);
if (msgtype.ToLower().Equals("oru") || msgtype.ToLower().Equals("mdm"))
{
if (obj["hl7message"]["OBX"] != null)
{
if (obj["hl7message"]["OBX"].Type == JTokenType.Array)
{
foreach (var obx in obj["hl7message"]["OBX"])
{
if (Utilities.getFirstField(obx["OBX.2"]).Equals("TX") || Utilities.getFirstField(obx["OBX.2"]).Equals("FT"))
{
builder.Append(Utilities.getFirstField(obx["OBX.5"]));
}
}
}
else
{
var obx = obj["hl7message"]["OBX"];
if (Utilities.getFirstField(obx["OBX.2"]).Equals("TX") || Utilities.getFirstField(obx["OBX.2"]).Equals("FT"))
{
builder.Append(Utilities.getFirstField(obx["OBX.5"]));
}
}
string report = builder.ToString();
report = report.UnEscapeHL7();
report = report.Replace(@"\\", @"\");
string cogurl = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
string responseFromServer = NLPUtilities.ExtractTextUsingCogServices(Encoding.UTF8.GetBytes(report), cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
if (string.IsNullOrEmpty(responseFromServer))
{
responseFromServer = NLPUtilities.ExtractTextUsingTIKA(Encoding.UTF8.GetBytes(report), Utilities.GetEnvironmentVariable("TIKAServerurl"));
}
if (responseFromServer.StartsWith("TIMEOUT~"))
{
log.LogTrace("{\"id\":\"" + Utilities.getFirstField(obj["id"]) + "\",\"status\":\"Timeout\",\"readresulturl\":\"" + responseFromServer.Split("~")[1] + "\"}");
}
if (string.IsNullOrEmpty(responseFromServer) || responseFromServer.Length < 3)
{
log.LogError($"TIKA Server may have failed to parse content {(string)obj["id"]}");
}
//Send Report to NLP
CTakesRequest creq = new CTakesRequest()
{
Content = responseFromServer,
CTAKESUrl = Utilities.GetEnvironmentVariable("CTAKESServerURL"),
UMLSUser = Utilities.GetEnvironmentVariable("CTAKESUMLSUser"),
UMLSPassword = Utilities.GetEnvironmentVariable("CTAKESUMLSPassword"),
Format = Utilities.GetEnvironmentVariable("CTAKESFormat"),
};
var result = NLPUtilities.ExtractMedicalEntities(creq);
result.Id = (string)obj["id"];
result.Location = (string)obj["rhm"];
if (string.IsNullOrEmpty(result.ParsedText))
{
result.ParsedText = responseFromServer;
}
string doctype = "";
if (msgtype.Equals("MDM"))
{
doctype = Utilities.getFirstField(obj["hl7message"]["TXA"]["TXA.2"]);
}
if (doctype == null)
{
doctype = "";
}
result.DocumentType = doctype;
searcharr.Add(result);
}
}
}
if (searcharr.Count > 0)
{
search.UploadMedicalEntities(searcharr.ToArray());
}
}
log.LogInformation("UpdateSearchIndexDiagReports function completed");
}
public static void Run([BlobTrigger("%StorageAccountBlob%/ingest/documents/{name}", Connection = "StorageAccount")] Stream myBlob, string name, ILogger log)
{
log.LogInformation("NLP Extract Entities File triggered by ingest/documents/" + name);
try
{
string coid = name;
int dirend = coid.LastIndexOf("/");
if (dirend > -1)
{
coid = coid.Substring(dirend + 1);
}
int extbegin = coid.LastIndexOf(".");
if (extbegin > -1)
{
coid = coid.Substring(0, extbegin);
}
string loc = "ingest/documents/" + name;
byte[] byteArray = null;
using (MemoryStream ms = new MemoryStream())
{
myBlob.CopyTo(ms);
byteArray = ms.ToArray();
}
log.LogInformation("Calling CogServices/TIKA to Extract Text from hl7json/ingest/documents/" + name);
string cogurl = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
log.LogInformation("Trying CogServices...");
string responseFromServer = NLPUtilities.ExtractTextUsingCogServices(byteArray, cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
if (string.IsNullOrEmpty(responseFromServer))
{
log.LogInformation("No extract Trying TIKA...");
responseFromServer = NLPUtilities.ExtractTextUsingTIKA(byteArray, Utilities.GetEnvironmentVariable("TIKAServerurl"));
}
if (responseFromServer.StartsWith("TIMEOUT~"))
{
log.LogTrace("CogServiceExtract Timeout: {\"id\":\"" + coid + "\",\"status\":\"Timeout\",\"readresulturl\":\"" + responseFromServer.Split("~")[1] + "\"}");
}
//string responseFromServer = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length);
log.LogInformation("Extracting Medical Reports from hl7json/ingest/documents/" + name);
//Extract Reports From Content (Auto-Detect Medical Exchange Formats (CDA, HL7, FHIR))
List <string> medreports = NLPUtilities.ExtractMedicalReportData(responseFromServer, log);
List <MedicalEntities> retVal = new List <MedicalEntities>();
foreach (string medreport in medreports)
{
CTakesRequest creq = new CTakesRequest()
{
Content = medreport,
CTAKESUrl = Utilities.GetEnvironmentVariable("CTAKESServerURL"),
UMLSUser = Utilities.GetEnvironmentVariable("CTAKESUMLSUser"),
UMLSPassword = Utilities.GetEnvironmentVariable("CTAKESUMLSPassword"),
Format = Utilities.GetEnvironmentVariable("CTAKESFormat"),
};
log.LogInformation("Calling CTAKES to extract medical entities from hl7json/ingest/documents/" + name);
var result = NLPUtilities.ExtractMedicalEntities(creq);
result.Id = coid;
result.Location = loc;
result.DocumentType = name;
retVal.Add(result);
}
log.LogInformation("Updateing search index with content and medical entities from hl7json/ingest/documents/" + name);
SearchUtilities su = new SearchUtilities(log);
su.UploadMedicalEntities(retVal.ToArray());
log.LogInformation("Succesfully Completed processing of hl7json/ingest/documents/" + name);
}
catch (System.Exception e)
{
log.LogError(e, e.Message);
}
}
public static List <string> ExtractMedicalReportData(string report, ILogger log)
{
List <string> retVal = new List <string>();
if (report.TrimStart().StartsWith("MSH|"))
{
string[] segs = report.Replace("\r\n", "\r").Split("\r");
StringBuilder sb = new StringBuilder();
sb.Append(segs[0]).Append("\r");
for (int i = 1; i < segs.Length; i++)
{
if (segs[i].StartsWith("MSH"))
{
string s = ExtractReportFromHL7(sb.ToString());
if (!String.IsNullOrEmpty(s))
{
retVal.Add(s);
}
sb.Clear();
}
sb.Append(segs[i]).Append("\r");
}
if (sb.Length > 0)
{
string s = ExtractReportFromHL7(sb.ToString());
if (!String.IsNullOrEmpty(s))
{
string cogurl = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
string extractstr = NLPUtilities.ExtractTextUsingCogServices(Encoding.UTF8.GetBytes(s), cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
if (string.IsNullOrEmpty(extractstr))
{
extractstr = NLPUtilities.ExtractTextUsingTIKA(Encoding.UTF8.GetBytes(s), Utilities.GetEnvironmentVariable("TIKAServerurl"));
}
retVal.Add(extractstr);
}
}
//FHIR HTML
}
else if (report.TrimStart().StartsWith("{") || report.TrimStart().StartsWith("["))
{
try
{
if (report.TrimStart().StartsWith("{"))
{
report = "[" + report + "]";
}
var dr = JArray.Parse(report);
foreach (JObject obj in dr.Children())
{
if (obj["text"] != null)
{
retVal.Add(obj["text"]["div"].GetFirstField());
}
}
}
catch (Exception e)
{
log.LogError(e, e.Message);
}
}
else
{
retVal.Add(report);
}
return(retVal);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("NLP Extract Entities Http called");
try
{
string doctype = req.Query["doctype"];
if (doctype == null)
{
doctype = "Unkown Doc Type";
}
string handwritten = req.Query["handwritten"];
string coid = req.Query["id"];
if (coid == null)
{
coid = Guid.NewGuid().ToString();
}
string loc = req.Query["location"];
if (loc == null)
{
loc = "Adhoc Document Addition";
}
string updatesearch = req.Query["updatesearch"];
byte[] byteArray = null;
// Read the post data into byte array
using (var stream = new MemoryStream())
{
await req.Body.CopyToAsync(stream);
byteArray = stream.ToArray();
}
string cogurl = Utilities.GetEnvironmentVariable("CogServicesOCRURL");
if (handwritten != null)
{
cogurl += "?mode=Handwritten";
}
string responseFromServer = NLPUtilities.ExtractTextUsingCogServices(byteArray, cogurl, Utilities.GetEnvironmentVariable("CogServicesKey"));
if (string.IsNullOrEmpty(responseFromServer))
{
responseFromServer = NLPUtilities.ExtractTextUsingTIKA(byteArray, Utilities.GetEnvironmentVariable("TIKAServerurl"));
}
if (responseFromServer.StartsWith("TIMEOUT~"))
{
return(new JsonResult(JObject.Parse("{\"id\":\"" + coid + "\",\"status\":\"Timeout\",\"readresulturl\":\"" + responseFromServer.Split("~")[1] + "\"}")));
}
//Extract Reports From Content (Auto-Detect Medical Exchange Formats (CDA, HL7, FHIR))
List <string> medreports = NLPUtilities.ExtractMedicalReportData(responseFromServer, log);
List <MedicalEntities> retVal = new List <MedicalEntities>();
foreach (string medreport in medreports)
{
CTakesRequest creq = new CTakesRequest()
{
Content = medreport,
CTAKESUrl = Utilities.GetEnvironmentVariable("CTAKESServerURL"),
UMLSUser = Utilities.GetEnvironmentVariable("CTAKESUMLSUser"),
UMLSPassword = Utilities.GetEnvironmentVariable("CTAKESUMLSPassword"),
Format = Utilities.GetEnvironmentVariable("CTAKESFormat"),
};
var result = NLPUtilities.ExtractMedicalEntities(creq);
result.Id = coid;
result.Location = loc;
result.DocumentType = doctype;
retVal.Add(result);
}
if (updatesearch != null)
{
SearchUtilities su = new SearchUtilities(log);
su.UploadMedicalEntities(retVal.ToArray());
}
return(new JsonResult(retVal));
}
catch (System.Exception e)
{
log.LogError(e, e.Message);
return(new System.Web.Http.InternalServerErrorResult());
}
}