public async Task GivenConfigurationSetAndFhirResources_WhenTransform_CdmFolderShouldBeGenerated()
{
string cdmFolder = Guid.NewGuid().ToString("N");
BaseMappingDefinitionLoader loader = new LocalMappingDefinitionLoader("TestResource\\testSchema");
IEnumerable <TabularMappingDefinition> tabularMappings = loader.Load();
CdmCorpusDefinition defination = CdmSchemaGenerator.InitLocalcdmCorpusDefinition(cdmFolder);
CdmSchemaGenerator cdmSchemaGenerator = new CdmSchemaGenerator(defination);
await cdmSchemaGenerator.InitializeCdmFolderAsync(tabularMappings);
ISource source = new LocalNdjsonSource(Path.Combine("TestResource", "FhirResource"));
ISink sink = new LocalCsvSink(cdmFolder)
{
CsvFilePath = (string tableName) =>
{
return($"data/Local{tableName}/{tableName}-partition-data.csv");
}
};
var transformer = new BasicFhirElementTabularTransformer();
FhirElementTabularTransformer.IdGenerator = () =>
{
return("0000");
};
TransformationExecutor executor = new TransformationExecutor(source, sink, tabularMappings, transformer);
await executor.ExecuteAsync();
// Whether generated CDM schema
Assert.IsTrue(File.Exists(Path.Combine(cdmFolder, "default.manifest.cdm.json")));
Assert.IsTrue(File.Exists(Path.Combine(cdmFolder, "LocalPatient.cdm.json")));
Assert.IsTrue(File.Exists(Path.Combine(cdmFolder, "LocalPatientName.cdm.json")));
// If generated flatten data
Assert.IsTrue(Directory.Exists(Path.Combine(cdmFolder, "data", "LocalPatient")));
Assert.IsTrue(Directory.Exists(Path.Combine(cdmFolder, "data", "LocalPatientName")));
// If generated data are same with ground truth
string[] tableNames =
{
"Patient",
"AllergyIntolerance",
"CarePlan",
"Encounter",
"Location",
"Observation",
"PatientName",
"PatientFlattenJson",
"EncounterClass",
"CarePlanPeriod"
};
foreach (var tableName in tableNames)
{
var sourceFilePath = Path.Combine("TestResource", "TestOutput", $"Local{tableName}", $"{tableName}-partition-data.csv");
var targetFilePath = Path.Combine(@$ "{cdmFolder}", "data", $"Local{tableName}", $"{tableName}-partition-data.csv");
Assert.IsTrue(CheckSameContent(sourceFilePath, targetFilePath), $"{sourceFilePath} and {targetFilePath} are different.");
}
}
static async Task Main(string[] args)
{
TransformationLogging.LoggerFactory = LoggerFactory.Create(builder => {
builder.AddFilter("Microsoft", LogLevel.Warning)
.AddFilter("System", LogLevel.Warning)
.AddFilter("Microsoft.Health.Fhir.Transformation", LogLevel.Information)
.AddConsole();
});
ILogger logger = TransformationLogging.CreateLogger <Program>();
var rootCommand = new RootCommand()
{
new Option <string>("--config"),
new Option <string>("--input"),
new Option <string>("--output"),
new Option <int>("--maxDepth", getDefaultValue: () => 3),
};
rootCommand.Handler = CommandHandler.Create(
new Func <string, string, string, int, Task>(async(config, input, output, maxDepth) =>
{
LocalMappingDefinitionLoader configLoader = new LocalMappingDefinitionLoader(config, maxDepth);
TabularMappingDefinition[] mappings = configLoader.Load();
FhirElementTabularTransformer transformer = new BasicFhirElementTabularTransformer();
logger.LogInformation("Start to generate CDM schema.");
CdmCorpusDefinition defination = CdmSchemaGenerator.InitLocalcdmCorpusDefinition(output);
CdmSchemaGenerator cdmSchemaGenerator = new CdmSchemaGenerator(defination);
cdmSchemaGenerator.InitializeCdmFolderAsync(mappings).Wait();
logger.LogInformation("Generate CDM schema completed.");
string operationId = Guid.NewGuid().ToString("N");
ISource source = new LocalNdjsonSource(input);
ISink sink = new LocalCsvSink(output)
{
CsvFilePath = (string tableName) =>
{
return($"data/Local{tableName}/partition-data-{operationId}.csv");
}
};
logger.LogInformation("Start to transform data.");
IProgress <(int, int)> progressHandler = new Progress <(int, int)>(progress =>
{
if (progress.Item1 % 100 == 0 || progress.Item2 % 100 == 0)
{
logger.LogInformation($"({progress.Item1} loaded, {progress.Item2} transformed) to CDM folder. {DateTime.UtcNow.ToLongTimeString()}");
}
});
TransformationExecutor executor = new TransformationExecutor(source, sink, mappings, transformer);
await executor.ExecuteAsync(progressHandler);
logger.LogInformation("Complete to transform data.");
}));
await rootCommand.InvokeAsync(args);
}
public void GivenDefinitionFolder_WhenLoadConfiguration_TabularMappingDefinitionShouldBeReturn()
{
BaseMappingDefinitionLoader loader = new LocalMappingDefinitionLoader(".\\TestResource\\TestSchemaSimple");
var tabularMappingDefinitions = loader.Load().ToList();
var columns = tabularMappingDefinitions[0].Columns;
Assert.AreEqual(33, columns.Count());
Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.ResourceId)));
Assert.IsFalse(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.FhirPath)));
Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals("NameMulti_seg_path")));
}
public void GivenDefinitionFolderWithResourceUnroll_WhenLoadConfiguration_TabularMappingDefinitionShouldBeReturn()
{
LocalMappingDefinitionLoader loader = new LocalMappingDefinitionLoader(".\\TestResource\\TestSchemaUnroll");
var tabularMappingDefinitions = loader.Load().ToList();
var columns = tabularMappingDefinitions[0].Columns;
Assert.AreEqual(8, columns.Count());
Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.ResourceId)));
Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.FhirPath)));
Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.RowId)));
Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.ParentPath)));
}