private JObject ProcessStructObject(JToken structItem, FhirParquetSchemaNode schemaNode) { if (structItem is not JObject fhirJObject) { _logger.LogError($"Current FHIR object is not a valid JObject: {schemaNode.GetNodePath()}."); throw new ParquetDataProcessorException($"Current FHIR object is not a valid JObject: {schemaNode.GetNodePath()}."); } JObject processedObject = new JObject(); foreach (var subItem in fhirJObject) { JToken subObject = subItem.Value; // Process choice type FHIR resource. if (schemaNode.ContainsChoiceDataType(subItem.Key)) { var choiceTypeName = schemaNode.ChoiceTypeNodes[subItem.Key].Item1; var choiceTypeDataType = schemaNode.ChoiceTypeNodes[subItem.Key].Item2; if (!schemaNode.SubNodes[choiceTypeName].SubNodes.ContainsKey(choiceTypeDataType)) { _logger.LogError($"Data type \"{choiceTypeDataType}\" cannot be found in choice type property, {schemaNode.GetNodePath()}."); throw new ParquetDataProcessorException($"Data type \"{choiceTypeDataType}\" cannot be found in choice type property, {schemaNode.GetNodePath()}."); } var dataTypeNode = schemaNode.SubNodes[choiceTypeName].SubNodes[choiceTypeDataType]; processedObject.Add(choiceTypeName, ProcessChoiceTypeObject(subObject, dataTypeNode)); } else { // Ignore FHIR data node if it doesn't exist in schema. if (schemaNode.SubNodes == null || !schemaNode.SubNodes.ContainsKey(subItem.Key)) { continue; } FhirParquetSchemaNode subNode = schemaNode.SubNodes[subItem.Key]; if (subNode.IsRepeated) { // Process array FHIR resource. processedObject.Add(subNode.Name, ProcessArrayObject(subObject, subNode)); } else if (subNode.IsLeaf) { // Process leaf FHIR resource. processedObject.Add(subNode.Name, ProcessLeafObject(subObject, subNode)); } else { // Process struct FHIR resource. processedObject.Add(subNode.Name, ProcessStructObject(subObject, subNode)); } } } return(processedObject); }
private static Dictionary <string, FhirParquetSchemaNode> CreateSchemaMap() { var schemaMap = new Dictionary <string, FhirParquetSchemaNode>(); // Mock Patient schema node var idSchemaNode = new FhirParquetSchemaNode { ChoiceTypeNodes = null, Depth = 1, Name = "id", NodePaths = new List <string> { "Patient", "id" }, IsRepeated = false, SubNodes = null, Type = "id", }; schemaMap["Patient"] = new FhirParquetSchemaNode { ChoiceTypeNodes = new Dictionary <string, Tuple <string, string> > { { "deceasedBoolean", new Tuple <string, string>("deceased", "boolean") }, { "deceasedDateTime", new Tuple <string, string>("deceased", "dateTime") }, }, Depth = 0, Name = "Patient", IsRepeated = false, SubNodes = new Dictionary <string, FhirParquetSchemaNode> { { "id", idSchemaNode }, }, Type = "Patient", }; // Mock Observation schema node idSchemaNode.NodePaths[0] = "Observation"; schemaMap["Observation"] = new FhirParquetSchemaNode { ChoiceTypeNodes = new Dictionary <string, Tuple <string, string> > { { "effectiveDateTime", new Tuple <string, string>("effective", "dateTime") }, { "effectivePeriod", new Tuple <string, string>("effective", "period") }, }, Depth = 0, Name = "Observation", NodePaths = new List <string> { "Observation" }, IsRepeated = false, SubNodes = new Dictionary <string, FhirParquetSchemaNode> { { "id", idSchemaNode }, }, Type = "Observation", }; return(schemaMap); }
/// <summary> /// The input json schema itself "type" should be object. /// And the "type" for each property of json schema should be basic types, which means the Json data must be one layer. /// </summary> /// <param name="schemaType">The schema type.</param> /// <param name="jsonSchema">The json schema.</param> /// <returns>A FhirParquetSchemaNode instance.</returns> public static FhirParquetSchemaNode ParseJSchema(string schemaType, JSchema jsonSchema) { if (jsonSchema.Type == null || jsonSchema.Type == JSchemaType.Null) { throw new GenerateFhirParquetSchemaNodeException(string.Format("The \"{0}\" customized schema have no \"type\" keyword or \"type\" is null.", schemaType)); } if (jsonSchema.Type != JSchemaType.Object) { throw new GenerateFhirParquetSchemaNodeException(string.Format("The \"{0}\" customized schema type \"{1}\" should be \"object\".", schemaType, jsonSchema.Type)); } var fhirPath = new List <string>() { schemaType }; var customizedSchemaNode = new FhirParquetSchemaNode() { Name = schemaType, Type = schemaType, Depth = 0, NodePaths = new List <string>(fhirPath), SubNodes = new Dictionary <string, FhirParquetSchemaNode>(), }; foreach (var property in jsonSchema.Properties) { fhirPath.Add(property.Key); if (property.Value.Type == null || property.Value.Type == JSchemaType.Null) { throw new GenerateFhirParquetSchemaNodeException(string.Format("Property \"{0}\" for \"{1}\" customized schema have no \"type\" keyword or \"type\" is null.", property.Key, schemaType)); } if (!FhirParquetSchemaConstants.BasicJSchemaTypeMap.ContainsKey(property.Value.Type.Value)) { throw new GenerateFhirParquetSchemaNodeException(string.Format("Property \"{0}\" type \"{1}\" for \"{2}\" customized schema is not basic type.", property.Key, property.Value.Type.Value, schemaType)); } customizedSchemaNode.SubNodes.Add( property.Key, BuildLeafNode(property.Key, FhirParquetSchemaConstants.BasicJSchemaTypeMap[property.Value.Type.Value], 1, fhirPath)); fhirPath.RemoveAt(fhirPath.Count - 1); } return(customizedSchemaNode); }
private JObject ProcessChoiceTypeObject(JToken fhirObject, FhirParquetSchemaNode schemaNode) { JObject choiceRootObject = new JObject(); if (schemaNode.IsLeaf) { choiceRootObject.Add(schemaNode.Name, ProcessLeafObject(fhirObject, schemaNode)); } else { choiceRootObject.Add(schemaNode.Name, ProcessStructObject(fhirObject, schemaNode)); } return(choiceRootObject); }
private JValue ProcessLeafObject(JToken fhirObject, FhirParquetSchemaNode schemaNode) { if (schemaNode.Type == FhirParquetSchemaConstants.JsonStringType) { return(new JValue(fhirObject.ToString(Formatting.None))); } if (fhirObject is not JValue fhirLeafObject) { _logger.LogError($"Invalid data: complex object found in leaf schema node {schemaNode.GetNodePath()}."); throw new ParquetDataProcessorException($"Invalid data: complex object found in leaf schema node {schemaNode.GetNodePath()}."); } return(fhirLeafObject); }
public void GivenArrowSchemaManager_WhenAddSchema_SchemaShouldBeAddedWithoutException() { // Mock Organization schema node var organizationSchemaNode = new FhirParquetSchemaNode { ChoiceTypeNodes = new Dictionary <string, Tuple <string, string> >(), Depth = 0, Name = "Organization", NodePaths = new List <string> { "Organization" }, IsRepeated = false, SubNodes = new Dictionary <string, FhirParquetSchemaNode>(), Type = "Organization", }; var arrowSchemaManager = new ArrowSchemaManager(TestConstants.TestSchemaMap); arrowSchemaManager.AddOrUpdateArrowSchema("Organization", organizationSchemaNode); }
private JArray ProcessArrayObject(JToken arrayItem, FhirParquetSchemaNode schemaNode) { if (arrayItem is not JArray fhirArrayObject) { _logger.LogError($"Current FHIR object is not a valid JArray: {schemaNode.GetNodePath()}."); throw new ParquetDataProcessorException($"Current FHIR object is not a valid JArray: {schemaNode.GetNodePath()}."); } JArray arrayObject = new JArray(); foreach (var item in fhirArrayObject) { if (schemaNode.IsLeaf) { arrayObject.Add(ProcessLeafObject(item, schemaNode)); } else { arrayObject.Add(ProcessStructObject(item, schemaNode)); } } return(arrayObject); }