private JObject ProcessStructObject(JToken structItem, FhirParquetSchemaNode schemaNode)
{
if (structItem is not JObject fhirJObject)
{
_logger.LogError($"Current FHIR object is not a valid JObject: {schemaNode.GetNodePath()}.");
throw new ParquetDataProcessorException($"Current FHIR object is not a valid JObject: {schemaNode.GetNodePath()}.");
}
JObject processedObject = new JObject();
foreach (var subItem in fhirJObject)
{
JToken subObject = subItem.Value;
// Process choice type FHIR resource.
if (schemaNode.ContainsChoiceDataType(subItem.Key))
{
var choiceTypeName = schemaNode.ChoiceTypeNodes[subItem.Key].Item1;
var choiceTypeDataType = schemaNode.ChoiceTypeNodes[subItem.Key].Item2;
if (!schemaNode.SubNodes[choiceTypeName].SubNodes.ContainsKey(choiceTypeDataType))
{
_logger.LogError($"Data type \"{choiceTypeDataType}\" cannot be found in choice type property, {schemaNode.GetNodePath()}.");
throw new ParquetDataProcessorException($"Data type \"{choiceTypeDataType}\" cannot be found in choice type property, {schemaNode.GetNodePath()}.");
}
var dataTypeNode = schemaNode.SubNodes[choiceTypeName].SubNodes[choiceTypeDataType];
processedObject.Add(choiceTypeName, ProcessChoiceTypeObject(subObject, dataTypeNode));
}
else
{
// Ignore FHIR data node if it doesn't exist in schema.
if (schemaNode.SubNodes == null || !schemaNode.SubNodes.ContainsKey(subItem.Key))
{
continue;
}
FhirParquetSchemaNode subNode = schemaNode.SubNodes[subItem.Key];
if (subNode.IsRepeated)
{
// Process array FHIR resource.
processedObject.Add(subNode.Name, ProcessArrayObject(subObject, subNode));
}
else if (subNode.IsLeaf)
{
// Process leaf FHIR resource.
processedObject.Add(subNode.Name, ProcessLeafObject(subObject, subNode));
}
else
{
// Process struct FHIR resource.
processedObject.Add(subNode.Name, ProcessStructObject(subObject, subNode));
}
}
}
return(processedObject);
}
private static Dictionary <string, FhirParquetSchemaNode> CreateSchemaMap()
{
var schemaMap = new Dictionary <string, FhirParquetSchemaNode>();
// Mock Patient schema node
var idSchemaNode = new FhirParquetSchemaNode
{
ChoiceTypeNodes = null,
Depth = 1,
Name = "id",
NodePaths = new List <string> {
"Patient", "id"
},
IsRepeated = false,
SubNodes = null,
Type = "id",
};
schemaMap["Patient"] = new FhirParquetSchemaNode
{
ChoiceTypeNodes = new Dictionary <string, Tuple <string, string> >
{
{ "deceasedBoolean", new Tuple <string, string>("deceased", "boolean") },
{ "deceasedDateTime", new Tuple <string, string>("deceased", "dateTime") },
},
Depth = 0,
Name = "Patient",
IsRepeated = false,
SubNodes = new Dictionary <string, FhirParquetSchemaNode>
{
{ "id", idSchemaNode },
},
Type = "Patient",
};
// Mock Observation schema node
idSchemaNode.NodePaths[0] = "Observation";
schemaMap["Observation"] = new FhirParquetSchemaNode
{
ChoiceTypeNodes = new Dictionary <string, Tuple <string, string> >
{
{ "effectiveDateTime", new Tuple <string, string>("effective", "dateTime") },
{ "effectivePeriod", new Tuple <string, string>("effective", "period") },
},
Depth = 0,
Name = "Observation",
NodePaths = new List <string> {
"Observation"
},
IsRepeated = false,
SubNodes = new Dictionary <string, FhirParquetSchemaNode>
{
{ "id", idSchemaNode },
},
Type = "Observation",
};
return(schemaMap);
}
/// <summary>
/// The input json schema itself "type" should be object.
/// And the "type" for each property of json schema should be basic types, which means the Json data must be one layer.
/// </summary>
/// <param name="schemaType">The schema type.</param>
/// <param name="jsonSchema">The json schema.</param>
/// <returns>A FhirParquetSchemaNode instance.</returns>
public static FhirParquetSchemaNode ParseJSchema(string schemaType, JSchema jsonSchema)
{
if (jsonSchema.Type == null || jsonSchema.Type == JSchemaType.Null)
{
throw new GenerateFhirParquetSchemaNodeException(string.Format("The \"{0}\" customized schema have no \"type\" keyword or \"type\" is null.", schemaType));
}
if (jsonSchema.Type != JSchemaType.Object)
{
throw new GenerateFhirParquetSchemaNodeException(string.Format("The \"{0}\" customized schema type \"{1}\" should be \"object\".", schemaType, jsonSchema.Type));
}
var fhirPath = new List <string>()
{
schemaType
};
var customizedSchemaNode = new FhirParquetSchemaNode()
{
Name = schemaType,
Type = schemaType,
Depth = 0,
NodePaths = new List <string>(fhirPath),
SubNodes = new Dictionary <string, FhirParquetSchemaNode>(),
};
foreach (var property in jsonSchema.Properties)
{
fhirPath.Add(property.Key);
if (property.Value.Type == null || property.Value.Type == JSchemaType.Null)
{
throw new GenerateFhirParquetSchemaNodeException(string.Format("Property \"{0}\" for \"{1}\" customized schema have no \"type\" keyword or \"type\" is null.", property.Key, schemaType));
}
if (!FhirParquetSchemaConstants.BasicJSchemaTypeMap.ContainsKey(property.Value.Type.Value))
{
throw new GenerateFhirParquetSchemaNodeException(string.Format("Property \"{0}\" type \"{1}\" for \"{2}\" customized schema is not basic type.", property.Key, property.Value.Type.Value, schemaType));
}
customizedSchemaNode.SubNodes.Add(
property.Key,
BuildLeafNode(property.Key, FhirParquetSchemaConstants.BasicJSchemaTypeMap[property.Value.Type.Value], 1, fhirPath));
fhirPath.RemoveAt(fhirPath.Count - 1);
}
return(customizedSchemaNode);
}
private JObject ProcessChoiceTypeObject(JToken fhirObject, FhirParquetSchemaNode schemaNode)
{
JObject choiceRootObject = new JObject();
if (schemaNode.IsLeaf)
{
choiceRootObject.Add(schemaNode.Name, ProcessLeafObject(fhirObject, schemaNode));
}
else
{
choiceRootObject.Add(schemaNode.Name, ProcessStructObject(fhirObject, schemaNode));
}
return(choiceRootObject);
}
private JValue ProcessLeafObject(JToken fhirObject, FhirParquetSchemaNode schemaNode)
{
if (schemaNode.Type == FhirParquetSchemaConstants.JsonStringType)
{
return(new JValue(fhirObject.ToString(Formatting.None)));
}
if (fhirObject is not JValue fhirLeafObject)
{
_logger.LogError($"Invalid data: complex object found in leaf schema node {schemaNode.GetNodePath()}.");
throw new ParquetDataProcessorException($"Invalid data: complex object found in leaf schema node {schemaNode.GetNodePath()}.");
}
return(fhirLeafObject);
}
public void GivenArrowSchemaManager_WhenAddSchema_SchemaShouldBeAddedWithoutException()
{
// Mock Organization schema node
var organizationSchemaNode = new FhirParquetSchemaNode
{
ChoiceTypeNodes = new Dictionary <string, Tuple <string, string> >(),
Depth = 0,
Name = "Organization",
NodePaths = new List <string> {
"Organization"
},
IsRepeated = false,
SubNodes = new Dictionary <string, FhirParquetSchemaNode>(),
Type = "Organization",
};
var arrowSchemaManager = new ArrowSchemaManager(TestConstants.TestSchemaMap);
arrowSchemaManager.AddOrUpdateArrowSchema("Organization", organizationSchemaNode);
}
private JArray ProcessArrayObject(JToken arrayItem, FhirParquetSchemaNode schemaNode)
{
if (arrayItem is not JArray fhirArrayObject)
{
_logger.LogError($"Current FHIR object is not a valid JArray: {schemaNode.GetNodePath()}.");
throw new ParquetDataProcessorException($"Current FHIR object is not a valid JArray: {schemaNode.GetNodePath()}.");
}
JArray arrayObject = new JArray();
foreach (var item in fhirArrayObject)
{
if (schemaNode.IsLeaf)
{
arrayObject.Add(ProcessLeafObject(item, schemaNode));
}
else
{
arrayObject.Add(ProcessStructObject(item, schemaNode));
}
}
return(arrayObject);
}
