Exemplo n.º 1
0
        public async Task GivenConfigurationSetAndFhirResources_WhenTransform_CdmFolderShouldBeGenerated()
        {
            string cdmFolder = Guid.NewGuid().ToString("N");
            BaseMappingDefinitionLoader            loader          = new LocalMappingDefinitionLoader("TestResource\\testSchema");
            IEnumerable <TabularMappingDefinition> tabularMappings = loader.Load();
            CdmCorpusDefinition defination = CdmSchemaGenerator.InitLocalcdmCorpusDefinition(cdmFolder);

            CdmSchemaGenerator cdmSchemaGenerator = new CdmSchemaGenerator(defination);
            await cdmSchemaGenerator.InitializeCdmFolderAsync(tabularMappings);

            ISource source = new LocalNdjsonSource(Path.Combine("TestResource", "FhirResource"));
            ISink   sink   = new LocalCsvSink(cdmFolder)
            {
                CsvFilePath = (string tableName) =>
                {
                    return($"data/Local{tableName}/{tableName}-partition-data.csv");
                }
            };
            var transformer = new BasicFhirElementTabularTransformer();

            FhirElementTabularTransformer.IdGenerator = () =>
            {
                return("0000");
            };
            TransformationExecutor executor = new TransformationExecutor(source, sink, tabularMappings, transformer);

            await executor.ExecuteAsync();

            // Whether generated CDM schema
            Assert.IsTrue(File.Exists(Path.Combine(cdmFolder, "default.manifest.cdm.json")));
            Assert.IsTrue(File.Exists(Path.Combine(cdmFolder, "LocalPatient.cdm.json")));
            Assert.IsTrue(File.Exists(Path.Combine(cdmFolder, "LocalPatientName.cdm.json")));

            // If generated flatten data
            Assert.IsTrue(Directory.Exists(Path.Combine(cdmFolder, "data", "LocalPatient")));
            Assert.IsTrue(Directory.Exists(Path.Combine(cdmFolder, "data", "LocalPatientName")));

            // If generated data are same with ground truth
            string[] tableNames =
            {
                "Patient",
                "AllergyIntolerance",
                "CarePlan",
                "Encounter",
                "Location",
                "Observation",

                "PatientName",
                "PatientFlattenJson",
                "EncounterClass",
                "CarePlanPeriod"
            };

            foreach (var tableName in tableNames)
            {
                var sourceFilePath = Path.Combine("TestResource", "TestOutput", $"Local{tableName}", $"{tableName}-partition-data.csv");
                var targetFilePath = Path.Combine(@$ "{cdmFolder}", "data", $"Local{tableName}", $"{tableName}-partition-data.csv");
                Assert.IsTrue(CheckSameContent(sourceFilePath, targetFilePath), $"{sourceFilePath} and {targetFilePath} are different.");
            }
        }
Exemplo n.º 2
0
        static async Task Main(string[] args)
        {
            TransformationLogging.LoggerFactory = LoggerFactory.Create(builder => {
                builder.AddFilter("Microsoft", LogLevel.Warning)
                .AddFilter("System", LogLevel.Warning)
                .AddFilter("Microsoft.Health.Fhir.Transformation", LogLevel.Information)
                .AddConsole();
            });

            ILogger logger = TransformationLogging.CreateLogger <Program>();

            var rootCommand = new RootCommand()
            {
                new Option <string>("--config"),
                new Option <string>("--input"),
                new Option <string>("--output"),
                new Option <int>("--maxDepth", getDefaultValue: () => 3),
            };

            rootCommand.Handler = CommandHandler.Create(
                new Func <string, string, string, int, Task>(async(config, input, output, maxDepth) =>
            {
                LocalMappingDefinitionLoader configLoader = new LocalMappingDefinitionLoader(config, maxDepth);
                TabularMappingDefinition[] mappings       = configLoader.Load();
                FhirElementTabularTransformer transformer = new BasicFhirElementTabularTransformer();

                logger.LogInformation("Start to generate CDM schema.");
                CdmCorpusDefinition defination        = CdmSchemaGenerator.InitLocalcdmCorpusDefinition(output);
                CdmSchemaGenerator cdmSchemaGenerator = new CdmSchemaGenerator(defination);
                cdmSchemaGenerator.InitializeCdmFolderAsync(mappings).Wait();
                logger.LogInformation("Generate CDM schema completed.");

                string operationId = Guid.NewGuid().ToString("N");
                ISource source     = new LocalNdjsonSource(input);
                ISink sink         = new LocalCsvSink(output)
                {
                    CsvFilePath = (string tableName) =>
                    {
                        return($"data/Local{tableName}/partition-data-{operationId}.csv");
                    }
                };

                logger.LogInformation("Start to transform data.");
                IProgress <(int, int)> progressHandler = new Progress <(int, int)>(progress =>
                {
                    if (progress.Item1 % 100 == 0 || progress.Item2 % 100 == 0)
                    {
                        logger.LogInformation($"({progress.Item1} loaded, {progress.Item2} transformed) to CDM folder. {DateTime.UtcNow.ToLongTimeString()}");
                    }
                });

                TransformationExecutor executor = new TransformationExecutor(source, sink, mappings, transformer);
                await executor.ExecuteAsync(progressHandler);
                logger.LogInformation("Complete to transform data.");
            }));

            await rootCommand.InvokeAsync(args);
        }
        public void GivenDefinitionFolder_WhenLoadConfiguration_TabularMappingDefinitionShouldBeReturn()
        {
            BaseMappingDefinitionLoader loader = new LocalMappingDefinitionLoader(".\\TestResource\\TestSchemaSimple");
            var tabularMappingDefinitions      = loader.Load().ToList();

            var columns = tabularMappingDefinitions[0].Columns;

            Assert.AreEqual(33, columns.Count());
            Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.ResourceId)));
            Assert.IsFalse(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.FhirPath)));
            Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals("NameMulti_seg_path")));
        }
        public void GivenDefinitionFolderWithResourceUnroll_WhenLoadConfiguration_TabularMappingDefinitionShouldBeReturn()
        {
            LocalMappingDefinitionLoader loader = new LocalMappingDefinitionLoader(".\\TestResource\\TestSchemaUnroll");
            var tabularMappingDefinitions       = loader.Load().ToList();

            var columns = tabularMappingDefinitions[0].Columns;

            Assert.AreEqual(8, columns.Count());
            Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.ResourceId)));
            Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.FhirPath)));
            Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.RowId)));
            Assert.IsTrue(condition: columns.Any(c => c.columnName.Equals(ReservedColumnName.ParentPath)));
        }