static async Task ExploreManifest(CdmCorpusDefinition cdmCorpus, string manifestPath) { Console.WriteLine($"\nLoading manifest {manifestPath} ..."); CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestPath); if (manifest == null) { Console.WriteLine($"Unable to load manifest {manifestPath}. Please inspect error log for additional details."); return; } // ------------------------------------------------------------------------------------------------------------ // List all the entities found in the manifest and allow the user to choose which entity to explore. while (true) { int index = 1; if (manifest.Entities.Count > 0) { Console.WriteLine("List of all entities:"); foreach (var entDec in manifest.Entities) { // Print entity declarations. // Assume there are only local entities in this manifest for simplicity. Console.Write(" " + index.ToString().PadRight(3)); Console.Write(" " + entDec.EntityName.PadRight(35)); Console.WriteLine(" " + entDec.EntityPath); index++; } } if (manifest.SubManifests.Count > 0) { Console.WriteLine("List of all sub-manifests:"); foreach (var manifestDecl in manifest.SubManifests) { // Print sub-manifest declarations. Console.Write(" " + index.ToString().PadRight(3)); Console.Write(" " + manifestDecl.ManifestName.PadRight(35)); Console.WriteLine(" " + manifestDecl.Definition); index++; } } if (index == 1) { Console.Write("No Entities or Sub-manifest found. Press [enter] to exit."); Console.ReadLine(); break; } Console.Write("Enter a number to show details for that Entity or Sub-manifest (press [enter] to exit): "); // Get the user's choice. string input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { break; } // Make sure the user's input is a number. int num = 0; if (!int.TryParse(input, out num)) { Console.WriteLine("\nEnter a number."); Console.WriteLine(); continue; } // User can select an entry that is a sub-manifest if (num > manifest.Entities.Count) { int subNum = num - manifest.Entities.Count - 1; // Re-enter this method supplying the absolute path of the submanifest definition (relative to the current manifest) await ExploreManifest(cdmCorpus, cdmCorpus.Storage.CreateAbsoluteCorpusPath(manifest.SubManifests[subNum].Definition, manifest)); continue; } index = 1; foreach (var entityDec in manifest.Entities) { if (index == num) { Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ..."); // From the path to the entity, get the actual schema description. // Take the local relative path in this doc and make sure it works. var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc while (true) { // List all the metadata properties of this entity that can be explored. Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:"); Console.WriteLine(" 1: Attributes"); Console.WriteLine(" 2: Traits"); Console.WriteLine(" 3: Properties"); Console.WriteLine(" 4: Data partition locations"); Console.WriteLine(" 5: Relationships"); Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): "); // Get the user's choice. input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { Console.WriteLine(); break; } // Make sure the user's input is a number. int choice = 0; if (int.TryParse(input, out choice)) { switch (choice) { // List the entity's attributes. case 1: ListAttributes(entSelected); break; // List the entity's traits. case 2: ListTraits(entSelected); break; // List the entity's properties. case 3: ListProperties(entSelected, entityDec); break; // List the entity's data partition locations. case 4: ListDataPartitionLocations(cdmCorpus, entityDec); break; // List the entity's relationships. case 5: if (manifest.Relationships != null && manifest.Relationships.Count > 0) { // The manifest file contains pre-calculated entity relationships, so we can read them directly. ListRelationshipsFromManifest(manifest, entSelected); } else { // The manifest file doesn't contain relationships, so we have to compute the relationships first. await cdmCorpus.CalculateEntityGraphAsync(manifest); ListRelationships(cdmCorpus, entSelected); } break; default: Console.WriteLine("\nEnter a number between 1-5."); break; } } else { Console.WriteLine("\nEnter a number."); } } } index++; } } }
static async Task Main(string[] args) { // Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths var corpus = new CdmCorpusDefinition(); // set callback to receive error and warning logs. corpus.SetEventCallback(new EventCallback { Invoke = (level, message) => { Console.WriteLine(message); } }, CdmStatusLevel.Warning); Console.WriteLine("Configure storage adapters."); // Configure storage adapters to point at the target local manifest location and at the fake public standards string pathFromExeToExampleRoot = "../../../../../../"; corpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "8-logical-manipulation-using-projections/sample-data")); corpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local // Fake cdm, normaly use the CDM Standards adapter // Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there corpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); Console.WriteLine("Create logical entity definition."); var logicalFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("local:/"); var logicalDoc = logicalFolder.Documents.Add("Person.cdm.json"); logicalDoc.Imports.Add("Address.cdm.json"); var entity = logicalDoc.Definitions.Add("Person"); // Add "name" data typed attribute. var nameAttr = entity.Attributes.Add("name") as CdmTypeAttributeDefinition; nameAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true); // Add "age" data typed attribute. var ageAttr = entity.Attributes.Add("age") as CdmTypeAttributeDefinition; ageAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true); // Add "address" entity typed attribute. var entityAttr = new CdmEntityAttributeDefinition(corpus.Ctx, "address") { Entity = new CdmEntityReference(corpus.Ctx, "Address", true) }; ApplyArrayExpansion(entityAttr, 1, 3, "{m}{A}{o}", "countAttribute"); ApplyDefaultBehavior(entityAttr, "addressFK", "address"); entity.Attributes.Add(entityAttr); // Add "email" data typed attribute. var emailAttr = entity.Attributes.Add("email") as CdmTypeAttributeDefinition; emailAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true); // Save the logical definition of Person. await entity.InDocument.SaveAsAsync("Person.cdm.json"); Console.WriteLine("Get \"resolved\" folder where the resolved entities will be saved."); var resolvedFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("local:/resolved/"); var resOpt = new ResolveOptions(entity); // To get more information about directives and their meaning refer to // https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes // We will start by resolving this entity with the "normalized" direcitve. // This directive will be used on this and the next two examples so we can analize the resolved entity // without the array expansion. Console.WriteLine("Resolving logical entity with normalized directive."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized" }); var resNormalizedEntity = await entity.CreateResolvedEntityAsync($"normalized_{entity.EntityName}", resOpt, resolvedFolder); await resNormalizedEntity.InDocument.SaveAsAsync($"{resNormalizedEntity.EntityName}.cdm.json"); // Another common scenario is to resolve an entity using the "referenceOnly" directive. // This directives is used to replace the relationships with a foreign key. Console.WriteLine("Resolving logical entity with referenceOnly directive."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "referenceOnly" }); var resReferenceOnlyEntity = await entity.CreateResolvedEntityAsync($"referenceOnly_{entity.EntityName}", resOpt, resolvedFolder); await resReferenceOnlyEntity.InDocument.SaveAsAsync($"{resReferenceOnlyEntity.EntityName}.cdm.json"); // When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that // a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the // entity attribute and we use the "structured" directive to resolve the entity attribute as an attribute group. Console.WriteLine("Resolving logical entity with structured directive."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "structured" }); var resStructuredEntity = await entity.CreateResolvedEntityAsync($"structured_{entity.EntityName}", resOpt, resolvedFolder); await resStructuredEntity.InDocument.SaveAsAsync($"{resStructuredEntity.EntityName}.cdm.json"); // Now let us remove the "normalized" directive so the array expansion operation can run. Console.WriteLine("Resolving logical entity without directives (array expansion)."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { }); var resArrayEntity = await entity.CreateResolvedEntityAsync($"array_expansion_{entity.EntityName}", resOpt, resolvedFolder); await resArrayEntity.InDocument.SaveAsAsync($"{resArrayEntity.EntityName}.cdm.json"); }
public async Task TestCalculateRelationshipsOnResolvedEntities() { var expectedResolvedManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedResolvedSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedSubManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedResolvedExcManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedExcManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedResolvedExcSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedExcSubManifestRels.json")).ToObject <List <E2ERelationship> >(); CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities"); var rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json"); var resolvedManifest = await LoadAndResolveManifest(corpus, rootManifest, "-resolved"); string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(resolvedManifest.SubManifests[0].Definition); CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition; // using createResolvedManifest will only populate exclusive relationships VerifyRelationships(resolvedManifest, expectedResolvedExcManifestRels); VerifyRelationships(subManifest, expectedResolvedExcSubManifestRels); // check that each relationship has been created correctly with the all flag await resolvedManifest.PopulateManifestRelationshipsAsync(); await subManifest.PopulateManifestRelationshipsAsync(); VerifyRelationships(resolvedManifest, expectedResolvedManifestRels); VerifyRelationships(subManifest, expectedResolvedSubManifestRels); // it is not enough to check if the relationships are correct. // We need to check if the incoming and outgoing relationships are // correct as well. One being correct can cover up the other being wrong // A var aEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[0].EntityPath, resolvedManifest); var aInRels = corpus.FetchIncomingRelationships(aEnt); var aOutRels = corpus.FetchOutgoingRelationships(aEnt); Assert.AreEqual(0, aInRels.Count); Assert.AreEqual(1, aOutRels.Count); Assert.AreEqual("local:/A-resolved.cdm.json/A", aOutRels[0].FromEntity); Assert.AreEqual("local:/B-resolved.cdm.json/B", aOutRels[0].ToEntity); // B var bEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[1].EntityPath, resolvedManifest); var bInRels = corpus.FetchIncomingRelationships(bEnt); var bOutRels = corpus.FetchOutgoingRelationships(bEnt); Assert.AreEqual(2, bInRels.Count); Assert.AreEqual("local:/A-resolved.cdm.json/A", bInRels[0].FromEntity); Assert.AreEqual("local:/B-resolved.cdm.json/B", bInRels[0].ToEntity); Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", bInRels[1].FromEntity); Assert.AreEqual("local:/B-resolved.cdm.json/B", bInRels[1].ToEntity); Assert.AreEqual(0, bOutRels.Count); // C var cEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[0].EntityPath, subManifest); var cInRels = corpus.FetchIncomingRelationships(cEnt); var cOutRels = corpus.FetchOutgoingRelationships(cEnt); Assert.AreEqual(0, cInRels.Count); Assert.AreEqual(2, cOutRels.Count); Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", cOutRels[0].FromEntity); Assert.AreEqual("local:/B-resolved.cdm.json/B", cOutRels[0].ToEntity); Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", cOutRels[1].FromEntity); Assert.AreEqual("local:/sub/D-resolved.cdm.json/D", cOutRels[1].ToEntity); // D var dEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[1].EntityPath, subManifest); var dInRels = corpus.FetchIncomingRelationships(dEnt); var dOutRels = corpus.FetchOutgoingRelationships(dEnt); Assert.AreEqual(1, dInRels.Count); Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", dInRels[0].FromEntity); Assert.AreEqual("local:/sub/D-resolved.cdm.json/D", dInRels[0].ToEntity); Assert.AreEqual(0, dOutRels.Count); // E var eEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[2].EntityPath, resolvedManifest); var eInRels = corpus.FetchIncomingRelationships(eEnt); var eOutRels = corpus.FetchOutgoingRelationships(eEnt); Assert.AreEqual(1, eInRels.Count); Assert.AreEqual(0, eOutRels.Count); Assert.AreEqual("local:/sub/F-resolved.cdm.json/F", eInRels[0].FromEntity); Assert.AreEqual("local:/E-resolved.cdm.json/E", eInRels[0].ToEntity); // F var fEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[2].EntityPath, subManifest); var fInRels = corpus.FetchIncomingRelationships(fEnt); var fOutRels = corpus.FetchOutgoingRelationships(fEnt); Assert.AreEqual(0, fInRels.Count); Assert.AreEqual(1, fOutRels.Count); Assert.AreEqual("local:/sub/F-resolved.cdm.json/F", fOutRels[0].FromEntity); Assert.AreEqual("local:/E-resolved.cdm.json/E", fOutRels[0].ToEntity); }
public async Task TestRefreshesDataPartitionPatterns() { var inputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestRefreshDataPartitionPatterns"); var actualLastModTime = new FileInfo(Path.GetFullPath(inputPath)).LastWriteTimeUtc; var cdmCorpus = new CdmCorpusDefinition(); cdmCorpus.SetEventCallback(new EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); cdmCorpus.Storage.Mount("local", new LocalAdapter(inputPath)); cdmCorpus.Storage.DefaultNamespace = "local"; var cdmManifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/patternManifest.manifest.cdm.json"); var partitionEntity = cdmManifest.Entities.AllItems[0]; Assert.AreEqual(partitionEntity.DataPartitions.Count, 1); var timeBeforeLoad = DateTime.Now; await cdmManifest.FileStatusCheckAsync(); // file status check should check patterns and add two more partitions that match the pattern // should not re-add already existing partitions // Mac and Windows behave differently when listing file content, so we don't want to be strict about partition file order int totalExpectedPartitionsFound = 0; foreach (CdmDataPartitionDefinition partition in partitionEntity.DataPartitions.AllItems) { switch (partition.Location) { case "partitions/existingPartition.csv": totalExpectedPartitionsFound++; break; case "partitions/someSubFolder/someSubPartition.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.SpecializedSchema, "test special schema"); Assert.IsTrue(partition.LastFileStatusCheckTime > timeBeforeLoad); // inherits the exhibited traits from pattern Assert.AreEqual(partition.ExhibitsTraits.Count, 1); Assert.AreEqual(partition.ExhibitsTraits.AllItems[0].NamedReference, "is"); Assert.AreEqual(partition.Arguments.Count, 1); Assert.IsTrue(partition.Arguments.ContainsKey("testParam1")); List <string> argArray = partition.Arguments["testParam1"]; Assert.AreEqual(argArray.Count, 1); Assert.AreEqual(argArray[0], "/someSubFolder/someSub"); break; case "partitions/newPartition.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.Arguments.Count, 1); break; case "partitions/2018/folderCapture.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.Arguments.Count, 1); Assert.AreEqual(partition.Arguments.ContainsKey("year"), true); Assert.AreEqual(partition.Arguments["year"][0], "2018"); break; case "partitions/2018/8/15/folderCapture.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.Arguments.Count, 3); Assert.AreEqual(partition.Arguments.ContainsKey("year"), true); Assert.AreEqual(partition.Arguments["year"][0], "2018"); Assert.AreEqual(partition.Arguments.ContainsKey("month"), true); Assert.AreEqual(partition.Arguments["month"][0], "8"); Assert.AreEqual(partition.Arguments.ContainsKey("day"), true); Assert.AreEqual(partition.Arguments["day"][0], "15"); break; case "partitions/2018/8/15/folderCaptureRepeatedGroup.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.Arguments.Count, 1); Assert.AreEqual(partition.Arguments.ContainsKey("day"), true); Assert.AreEqual(partition.Arguments["day"][0], "15"); break; case "partitions/testTooFew.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.Arguments.Count, 0); break; case "partitions/testTooMany.csv": totalExpectedPartitionsFound++; Assert.AreEqual(partition.Arguments.Count, 0); break; } } Assert.AreEqual(totalExpectedPartitionsFound, 8); }
private async Task CreateManifest(CdmCorpusDefinition cdmCorpus) { Console.WriteLine("Make placeholder manifest"); // Make the temp manifest and add it to the root of the local documents in the corpus CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract"); // Add each declaration, this example is about medical appointments and care plans manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account"); manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address"); manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan"); manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept"); manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact"); manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device"); manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment"); manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter"); manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare"); manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location"); // Add the temp manifest to the root of the local documents in the corpus. var localRoot = cdmCorpus.Storage.FetchRootFolder("local"); localRoot.Documents.Add(manifestAbstract); // Create the resolved version of everything in the root folder too Console.WriteLine("Resolve the placeholder"); var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", ""); // Add an import to the foundations doc so the traits about partitons will resolve nicely manifestResolved.Imports.Add("cdm:/foundations.cdm.json"); Console.WriteLine("Save the documents"); foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities) { // Get the entity being pointed at var localEDef = eDef; var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved); // Make a fake partition, just to demo that var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description"); localEDef.DataPartitions.Add(part); part.Explanation = "not real data, just for demo"; // Define the location of the partition, relative to the manifest var location = $"local:/{entDef.EntityName}/partition-data.csv"; part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved); // Add trait to partition for csv params var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false); csvTrait.Arguments.Add("columnHeaders", "true"); csvTrait.Arguments.Add("delimiter", ","); // Get the actual location of the partition file from the corpus string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location); // Make a fake file with nothing but header for columns string header = ""; foreach (CdmTypeAttributeDefinition att in entDef.Attributes) { if (header != "") { header += ","; } header += att.Name; } Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{entDef.EntityName}")); File.WriteAllText(partPath, header); } await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true); }
public async Task TestPartitionPatternWithGlob() { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestPartitionPatternWithGlob"); int patternWithGlobAndRegex = 0; corpus.SetEventCallback(new EventCallback { Invoke = (CdmStatusLevel statusLevel, string message) => { if (message.Equals("CdmDataPartitionPatternDefinition | The Data Partition Pattern contains both a glob pattern (/testfile.csv) and a regular expression (/subFolder/testSubFile.csv) set, the glob pattern will be used. | FileStatusCheckAsync")) { patternWithGlobAndRegex++; } } }, CdmStatusLevel.Warning); CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("pattern.manifest.cdm.json"); await manifest.FileStatusCheckAsync(); // one pattern object contains both glob and regex Assert.AreEqual(1, patternWithGlobAndRegex); // make sure '.' in glob is not converted to '.' in regex CdmLocalEntityDeclarationDefinition dotIsEscaped = (CdmLocalEntityDeclarationDefinition)manifest.Entities[0]; Assert.AreEqual(dotIsEscaped.DataPartitionPatterns[0].GlobPattern, "test.ile.csv"); Assert.AreEqual(dotIsEscaped.DataPartitions.Count, 0); // star pattern should not match anything CdmLocalEntityDeclarationDefinition onlyStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[1]; Assert.AreEqual(onlyStar.DataPartitionPatterns[0].GlobPattern, "*"); Assert.AreEqual(onlyStar.DataPartitions.Count, 0); // star can match nothing CdmLocalEntityDeclarationDefinition starNoMatch = (CdmLocalEntityDeclarationDefinition)manifest.Entities[2]; Assert.AreEqual(starNoMatch.DataPartitionPatterns[0].GlobPattern, "/testfile*.csv"); Assert.AreEqual(starNoMatch.DataPartitions.Count, 1); Assert.AreEqual(starNoMatch.DataPartitions[0].Location, "/partitions/testfile.csv"); // star at root level // this should match any files at root level, none in subfolders CdmLocalEntityDeclarationDefinition starAtRoot = (CdmLocalEntityDeclarationDefinition)manifest.Entities[3]; Assert.AreEqual(starAtRoot.DataPartitionPatterns[0].GlobPattern, "/*.csv"); Assert.AreEqual(starAtRoot.DataPartitions.Count, 1); Assert.AreEqual(starAtRoot.DataPartitions[0].Location, "/partitions/testfile.csv"); // star at deeper level CdmLocalEntityDeclarationDefinition starAtDeeperLevel = (CdmLocalEntityDeclarationDefinition)manifest.Entities[4]; Assert.AreEqual(starAtDeeperLevel.DataPartitionPatterns[0].GlobPattern, "/*/*.csv"); Assert.AreEqual(starAtDeeperLevel.DataPartitions.Count, 1); Assert.AreEqual(starAtDeeperLevel.DataPartitions[0].Location, "/partitions/subFolder/testSubFile.csv"); // pattern that ends with star CdmLocalEntityDeclarationDefinition endsWithStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[5]; Assert.AreEqual(endsWithStar.DataPartitionPatterns[0].GlobPattern, "/testfile*"); Assert.AreEqual(endsWithStar.DataPartitions.Count, 1); Assert.AreEqual(endsWithStar.DataPartitions[0].Location, "/partitions/testfile.csv"); // globstar (**) on its own matches CdmLocalEntityDeclarationDefinition globStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[6]; Assert.AreEqual(globStar.DataPartitionPatterns[0].GlobPattern, "**"); Assert.AreEqual(2, globStar.DataPartitions.Count); Assert.AreEqual(1, globStar.DataPartitions.Where(x => x.Location == "/partitions/testfile.csv" ).ToList().Count); Assert.AreEqual(1, globStar.DataPartitions.Where(x => x.Location == "/partitions/subFolder/testSubFile.csv" ).ToList().Count); // globstar at the beginning of the pattern CdmLocalEntityDeclarationDefinition beginsWithGlobstar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[7]; Assert.AreEqual(beginsWithGlobstar.DataPartitionPatterns[0].GlobPattern, "/**.csv"); Assert.AreEqual(1, beginsWithGlobstar.DataPartitions.Count); Assert.AreEqual(beginsWithGlobstar.DataPartitions[0].Location, "/partitions/testfile.csv"); // globstar at the end of the pattern CdmLocalEntityDeclarationDefinition endsWithGlobstar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[8]; Assert.AreEqual(endsWithGlobstar.DataPartitionPatterns[0].GlobPattern, "/**"); Assert.AreEqual(endsWithGlobstar.DataPartitions.Count, 2); Assert.AreEqual(1, endsWithGlobstar.DataPartitions.Where(x => x.Location == "/partitions/testfile.csv" ).ToList().Count); Assert.AreEqual(1, endsWithGlobstar.DataPartitions.Where(x => x.Location == "/partitions/subFolder/testSubFile.csv" ).ToList().Count); // globstar matches zero or more folders CdmLocalEntityDeclarationDefinition zeroOrMoreFolders = (CdmLocalEntityDeclarationDefinition)manifest.Entities[9]; Assert.AreEqual(zeroOrMoreFolders.DataPartitionPatterns[0].GlobPattern, "/**/*.csv"); Assert.AreEqual(2, zeroOrMoreFolders.DataPartitions.Count); Assert.AreEqual(1, zeroOrMoreFolders.DataPartitions.Where(x => x.Location == "/partitions/testfile.csv" ).ToList().Count); Assert.AreEqual(1, zeroOrMoreFolders.DataPartitions.Where(x => x.Location == "/partitions/subFolder/testSubFile.csv" ).ToList().Count); // globstar matches zero or more folders without starting slash CdmLocalEntityDeclarationDefinition zeroOrMoreNoStartingSlash = (CdmLocalEntityDeclarationDefinition)manifest.Entities[10]; Assert.AreEqual(zeroOrMoreNoStartingSlash.DataPartitionPatterns[0].GlobPattern, "/**/*.csv"); Assert.AreEqual(2, zeroOrMoreNoStartingSlash.DataPartitions.Count); Assert.AreEqual(1, zeroOrMoreNoStartingSlash.DataPartitions.Where(x => x.Location == "/partitions/testfile.csv" ).ToList().Count); Assert.AreEqual(1, zeroOrMoreNoStartingSlash.DataPartitions.Where(x => x.Location == "/partitions/subFolder/testSubFile.csv" ).ToList().Count); // question mark in the middle of a pattern CdmLocalEntityDeclarationDefinition questionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[11]; Assert.AreEqual(questionMark.DataPartitionPatterns[0].GlobPattern, "/test?ile.csv"); Assert.AreEqual(1, questionMark.DataPartitions.Count); Assert.AreEqual(questionMark.DataPartitions[0].Location, "/partitions/testfile.csv"); // question mark at the beginning of a pattern CdmLocalEntityDeclarationDefinition beginsWithQuestionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[12]; Assert.AreEqual(beginsWithQuestionMark.DataPartitionPatterns[0].GlobPattern, "/?estfile.csv"); Assert.AreEqual(1, beginsWithQuestionMark.DataPartitions.Count); Assert.AreEqual(beginsWithQuestionMark.DataPartitions[0].Location, "/partitions/testfile.csv"); // question mark at the end of a pattern CdmLocalEntityDeclarationDefinition endsWithQuestionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[13]; Assert.AreEqual(endsWithQuestionMark.DataPartitionPatterns[0].GlobPattern, "/testfile.cs?"); Assert.AreEqual(1, endsWithQuestionMark.DataPartitions.Count); Assert.AreEqual(endsWithQuestionMark.DataPartitions[0].Location, "/partitions/testfile.csv"); // backslash in glob can match slash CdmLocalEntityDeclarationDefinition backslashInPattern = (CdmLocalEntityDeclarationDefinition)manifest.Entities[14]; Assert.AreEqual(backslashInPattern.DataPartitionPatterns[0].GlobPattern, "\\testfile.csv"); Assert.AreEqual(1, backslashInPattern.DataPartitions.Count); Assert.AreEqual(backslashInPattern.DataPartitions[0].Location, "/partitions/testfile.csv"); // pattern object includes glob pattern and regular expression CdmLocalEntityDeclarationDefinition globAndRegex = (CdmLocalEntityDeclarationDefinition)manifest.Entities[15]; Assert.AreEqual(globAndRegex.DataPartitionPatterns[0].GlobPattern, "/testfile.csv"); Assert.AreEqual(globAndRegex.DataPartitionPatterns[0].RegularExpression, "/subFolder/testSubFile.csv"); Assert.AreEqual(1, globAndRegex.DataPartitions.Count); // matching this file means the glob pattern was (correctly) used Assert.AreEqual(globAndRegex.DataPartitions[0].Location, "/partitions/testfile.csv"); }
/// <summary> /// Get the text version of all the resolved entities. /// </summary> /// <param name="cdmCorpus"> The CDM corpus. </param> /// <param name="directives"> The directives to use while getting the resolved entities. </param> /// <param name="manifest"> The manifest to be resolved. </param> /// <param name="spew"> The object used to store the text to be returned. </param> /// <returns> The text version of the resolved entities. (it's in a form that facilitates debugging) </returns> internal static async Task <string> ListAllResolved(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CdmManifestDefinition manifest, StringSpewCatcher spew = null) { // make sure the corpus has a set of default artifact attributes await cdmCorpus.PrepareArtifactAttributesAsync(); var seen = new HashSet <string>(); Func <CdmManifestDefinition, Task> seekEntities = null; seekEntities = async(CdmManifestDefinition f) => { if (f.Entities != null) { if (spew != null) { spew.SpewLine(f.FolderPath); } foreach (CdmEntityDeclarationDefinition entity in f.Entities) { string corpusPath; CdmEntityDeclarationDefinition ent = entity; CdmObject currentFile = f; while (ent is CdmReferencedEntityDeclarationDefinition) { corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(ent.EntityPath, currentFile); ent = await cdmCorpus.FetchObjectAsync <CdmReferencedEntityDeclarationDefinition>(corpusPath); currentFile = ent; } corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(((CdmLocalEntityDeclarationDefinition)ent).EntityPath, currentFile); ResolveOptions resOpt = new ResolveOptions() { ImportsLoadStrategy = ImportsLoadStrategy.Load }; CdmEntityDefinition newEnt = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(corpusPath, null, resOpt); resOpt.WrtDoc = newEnt.InDocument; resOpt.Directives = directives; ResolvedEntity resEnt = new ResolvedEntity(resOpt, newEnt); if (spew != null) { resEnt.Spew(resOpt, spew, " ", true); } } } if (f.SubManifests != null) { // folder.SubManifests.ForEach(async f => foreach (CdmManifestDeclarationDefinition subManifest in f.SubManifests) { string corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(subManifest.Definition, f); await seekEntities(await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(corpusPath)); } } }; await seekEntities(manifest); if (spew != null) { return(spew.GetContent()); } return(""); }
private static async Task RunTest(string testName, string sourceEntityName) { try { string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName); string testExpectedOutputPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName); string testActualOutputPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName); CdmCorpusDefinition corpus = new CdmCorpusDefinition(); corpus.SetEventCallback(new EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); corpus.Storage.Mount("localInput", new LocalAdapter(testInputPath)); corpus.Storage.Mount("localExpectedOutput", new LocalAdapter(testExpectedOutputPath)); corpus.Storage.Mount("localActualOutput", new LocalAdapter(testActualOutputPath)); corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath)); corpus.Storage.DefaultNamespace = "localInput"; CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"localInput:/{sourceEntityName}.cdm.json/{sourceEntityName}") as CdmEntityDefinition; Assert.IsTrue(srcEntityDef != null); var resOpt = new ResolveOptions { WrtDoc = srcEntityDef.InDocument, Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { }) }; CdmFolderDefinition actualOutputFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("localActualOutput:/"); CdmEntityDefinition resolvedEntityDef = null; string outputEntityFileName = string.Empty; string entityFileName = string.Empty; entityFileName = "default"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "referenceOnly"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "normalized"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "structured"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "structured" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "referenceOnly_normalized"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly", "normalized" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "referenceOnly_structured"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly", "structured" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "normalized_structured"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "structured" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } entityFileName = "referenceOnly_normalized_structured"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly", "normalized", "structured" }); outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder); if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions())) { ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath); } } catch (Exception e) { Assert.Fail(e.Message); } }
public async Task TestCalculateRelationshipsAndPopulateManifests() { var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests"); var expectedAllManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedAllManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedAllSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedAllSubManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedExclusiveManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedExclusiveManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedExclusiveSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedExclusiveSubManifestRels.json")).ToObject <List <E2ERelationship> >(); CdmCorpusDefinition corpus = new CdmCorpusDefinition(); corpus.SetEventCallback(new Utilities.EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); corpus.Storage.Mount("local", new LocalAdapter(testInputPath)); corpus.Storage.DefaultNamespace = "local"; CdmManifestDefinition rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json"); string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(rootManifest.SubManifests[0].Definition); CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition; await corpus.CalculateEntityGraphAsync(rootManifest); await rootManifest.PopulateManifestRelationshipsAsync(); Assert.AreEqual(rootManifest.Relationships.Count, 5); Assert.AreEqual(subManifest.Relationships.Count, 7); // check that each relationship has been created correctly foreach (E2ERelationship expectedRel in expectedAllManifestRels) { List <CdmE2ERelationship> found = rootManifest.Relationships.Where(x => x.FromEntity == expectedRel.FromEntity && x.FromEntityAttribute == expectedRel.FromEntityAttribute && x.ToEntity == expectedRel.ToEntity && x.ToEntityAttribute == expectedRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } foreach (E2ERelationship expectedSubRel in expectedAllSubManifestRels) { List <CdmE2ERelationship> found = subManifest.Relationships.Where(x => x.FromEntity == expectedSubRel.FromEntity && x.FromEntityAttribute == expectedSubRel.FromEntityAttribute && x.ToEntity == expectedSubRel.ToEntity && x.ToEntityAttribute == expectedSubRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } // make sure only relationships where to and from entities are in the manifest are found with the "exclusive" option is passed in await rootManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.Exclusive); Assert.AreEqual(rootManifest.Relationships.Count, 3); Assert.AreEqual(subManifest.Relationships.Count, 3); // check that each relationship has been created correctly foreach (E2ERelationship expectedRel in expectedExclusiveManifestRels) { List <CdmE2ERelationship> found = rootManifest.Relationships.Where(x => x.FromEntity == expectedRel.FromEntity && x.FromEntityAttribute == expectedRel.FromEntityAttribute && x.ToEntity == expectedRel.ToEntity && x.ToEntityAttribute == expectedRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } foreach (E2ERelationship expectedSubRel in expectedExclusiveSubManifestRels) { List <CdmE2ERelationship> found = subManifest.Relationships.Where(x => x.FromEntity == expectedSubRel.FromEntity && x.FromEntityAttribute == expectedSubRel.FromEntityAttribute && x.ToEntity == expectedSubRel.ToEntity && x.ToEntityAttribute == expectedSubRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } // make sure no relationships are added when "none" relationship option is passed in await rootManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.None); Assert.AreEqual(rootManifest.Relationships.Count, 0); Assert.AreEqual(subManifest.Relationships.Count, 0); }
static async Task Main(string[] args) { // ------------------------------------------------------------------------------------------------------------ // Instantiate a corpus. The corpus is the collection of all documents and folders created or discovered // while navigating objects and paths. var cdmCorpus = new CdmCorpusDefinition(); // ------------------------------------------------------------------------------------------------------------ // Configure storage adapters and mount them to the corpus. // We want our storage adapters to point at the local manifest location and at the example public standards. string pathFromExeToExampleRoot = "../../../../../../"; // Storage adapter pointing to the target local manifest location. cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "1-read-manifest")); // 'local' is our default namespace. // Any paths that start navigating without a device tag (ex. 'cdm') will just default to the 'local' namepace. cdmCorpus.Storage.DefaultNamespace = "local"; // Storage adapter pointing to the example public standards. // This is a fake 'cdm'; normally the Github adapter would be used to point at the real public standards. // Mount it as the 'cdm' device, not the default, so that we must use "cdm:<folder-path>" to get there. cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); // Example how to mount to the ADLS: // cdmCorpus.Storage.Mount("adls", // new ADLSAdapter( // "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. // "/<FILESYSTEM-NAME>", // Root. // "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. // "<CLIENT-ID>", // Client ID. // "<CLIENT-SECRET>" // Client secret. // ) // ); // ------------------------------------------------------------------------------------------------------------ // Open the default manifest file at the root. var manifestFile = "default.manifest.cdm.json"; // This method turns relative corpus paths into absolute paths in case we are in some sub-folders // and don't know it. var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestFile); // ------------------------------------------------------------------------------------------------------------ // List all the entities found in the manifest and allow the user to choose which entity to explore. while (true) { Console.WriteLine($"List of all entities found in {manifestFile}:"); // Loop over any entity declarations. int iEnt = 1; foreach (var entDec in manifest.Entities) { // Print it out. // Assume there are only local entities in this manifest for simplicity. Console.Write(" " + iEnt.ToString().PadRight(3)); Console.Write(" " + entDec.EntityName.PadRight(35)); Console.WriteLine(" " + entDec.EntityPath); iEnt++; } Console.Write("Enter a number to show details for that Entity (press [enter] to exit): "); // Get the user's choice. string input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { break; } // Make sure the user's input is a number. int num = 0; if (int.TryParse(input, out num)) { iEnt = 1; foreach (var entityDec in manifest.Entities) { if (iEnt == num) { Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ..."); // From the path to the entity, get the actual schema description. // Take the local relative path in this doc and make sure it works. var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc while (true) { // List all the metadata properties of this entity that can be explored. Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:"); Console.WriteLine(" 1: Attributes"); Console.WriteLine(" 2: Traits"); Console.WriteLine(" 3: Properties"); Console.WriteLine(" 4: Data partition locations"); Console.WriteLine(" 5: Relationships"); Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): "); // Get the user's choice. input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { Console.WriteLine(); break; } // Make sure the user's input is a number. int choice = 0; if (int.TryParse(input, out choice)) { switch (choice) { // List the entity's attributes. case 1: ListAttributes(entSelected); break; // List the entity's traits. case 2: ListTraits(entSelected); break; // List the entity's properties. case 3: ListProperties(entSelected, entityDec); break; // List the entity's data partition locations. case 4: ListDataPartitionLocations(entityDec); break; // List the entity's relationships. case 5: if (manifest.Relationships != null && manifest.Relationships.Count > 0) { // The manifest file contains pre-calculated entity relationships, so we can read them directly. ListRelationshipsFromManifest(manifest, entSelected); } else { // The manifest file doesn't contain relationships, so we have to compute the relationships first. await cdmCorpus.CalculateEntityGraphAsync(manifest); ListRelationships(cdmCorpus, entSelected); } break; default: Console.WriteLine("\nEnter a number between 1-5."); break; } } else { Console.WriteLine("\nEnter a number."); } } } iEnt++; } } else { Console.WriteLine("\nEnter a number."); Console.WriteLine(); } } }
public async Task TestCalculateRelationshipsOnResolvedEntities() { var expectedResolvedManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedResolvedSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedSubManifestRels.json")).ToObject <List <E2ERelationship> >(); var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities"); CdmCorpusDefinition corpus = new CdmCorpusDefinition(); corpus.SetEventCallback(new Utilities.EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); corpus.Storage.Mount("local", new LocalAdapter(testInputPath)); corpus.Storage.DefaultNamespace = "local"; var rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json"); var resolvedManifest = await LoadAndResolveManifest(corpus, rootManifest, "-resolved"); string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(resolvedManifest.SubManifests[0].Definition); CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition; // using createResolvedManifest will only populate exclusive relationships Assert.AreEqual(resolvedManifest.Relationships.Count, expectedResolvedManifestRels.Count); Assert.AreEqual(subManifest.Relationships.Count, expectedResolvedSubManifestRels.Count); // check that each relationship has been created correctly foreach (E2ERelationship expectedRel in expectedResolvedManifestRels) { List <CdmE2ERelationship> found = resolvedManifest.Relationships.Where(x => x.FromEntity == expectedRel.FromEntity && x.FromEntityAttribute == expectedRel.FromEntityAttribute && x.ToEntity == expectedRel.ToEntity && x.ToEntityAttribute == expectedRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } foreach (E2ERelationship expectedSubRel in expectedResolvedSubManifestRels) { List <CdmE2ERelationship> found = subManifest.Relationships.Where(x => x.FromEntity == expectedSubRel.FromEntity && x.FromEntityAttribute == expectedSubRel.FromEntityAttribute && x.ToEntity == expectedSubRel.ToEntity && x.ToEntityAttribute == expectedSubRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } // it is not enough to check if the relationships are correct. // We need to check if the incoming and outgoing relationships are // correct as well. One being correct can cover up the other being wrong // A var aEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[0].EntityPath, resolvedManifest); var aInRels = corpus.FetchIncomingRelationships(aEnt); var aOutRels = corpus.FetchOutgoingRelationships(aEnt); Assert.AreEqual(aInRels.Count, 0); Assert.AreEqual(aOutRels.Count, 1); Assert.AreEqual(aOutRels[0].FromEntity, "local:/A-resolved.cdm.json/A"); Assert.AreEqual(aOutRels[0].ToEntity, "local:/B-resolved.cdm.json/B"); // B var bEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[1].EntityPath, resolvedManifest); var bInRels = corpus.FetchIncomingRelationships(bEnt); var bOutRels = corpus.FetchOutgoingRelationships(bEnt); Assert.AreEqual(bInRels.Count, 1); Assert.AreEqual(bInRels[0].FromEntity, "local:/A-resolved.cdm.json/A"); Assert.AreEqual(bInRels[0].ToEntity, "local:/B-resolved.cdm.json/B"); Assert.AreEqual(bOutRels.Count, 0); // C var cEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[0].EntityPath, subManifest); var cInRels = corpus.FetchIncomingRelationships(cEnt); var cOutRels = corpus.FetchOutgoingRelationships(cEnt); Assert.AreEqual(cInRels.Count, 0); Assert.AreEqual(cOutRels.Count, 2); Assert.AreEqual(cOutRels[0].FromEntity, "local:/sub/C-resolved.cdm.json/C"); // TODO: this should point to the resolved entity, currently an open bug Assert.AreEqual(cOutRels[0].ToEntity, "local:/B.cdm.json/B"); Assert.AreEqual(cOutRels[1].FromEntity, "local:/sub/C-resolved.cdm.json/C"); Assert.AreEqual(cOutRels[1].ToEntity, "local:/sub/D-resolved.cdm.json/D"); // D var dEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[1].EntityPath, subManifest); var dInRels = corpus.FetchIncomingRelationships(dEnt); var dOutRels = corpus.FetchOutgoingRelationships(dEnt); Assert.AreEqual(dInRels.Count, 1); Assert.AreEqual(dInRels[0].FromEntity, "local:/sub/C-resolved.cdm.json/C"); Assert.AreEqual(dInRels[0].ToEntity, "local:/sub/D-resolved.cdm.json/D"); Assert.AreEqual(dOutRels.Count, 0); }
static async Task Main(string[] args) { var pathFromExeToExampleRoot = "../../../../../../"; // ------------------------------------------------------------------------------------------------------------ // Instantiate corpus and set up the default namespace to be local var cdmCorpus = new CdmCorpusDefinition(); cdmCorpus.Storage.DefaultNamespace = "local"; // ------------------------------------------------------------------------------------------------------------ // Set up adapters for managing access to different files-system locations // Fake cdm, normaly use the github adapter // Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); cdmCorpus.Storage.Mount("local", new LocalAdapter(modelJsonRoot)); // Example how to mount to the ADLS - make sure the hostname and root entered here are also changed // in the example.model.json file we load in the next section cdmCorpus.Storage.Mount("adls", new ADLSAdapter( "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. "/<FILESYSTEM-NAME>", // Root. "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. "<CLIENT-ID>", // Client ID. "<CLIENT-SECRET>" // Client secret. )); // ------------------------------------------------------------------------------------------------------------ // Load a model.json file from local FS var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/example.model.json"); // ------------------------------------------------------------------------------------------------------------ // Explore entities and partitions defined in the model Console.WriteLine("Listing entity declarations:"); foreach (CdmEntityDeclarationDefinition decl in manifest.Entities) { Console.WriteLine(" " + decl.EntityName); // TODO: This can be rewritten in a different way since data partition gives null for referenced entities, suggestions are welcome. if (decl.ObjectType == CdmObjectType.LocalEntityDeclarationDef) { foreach (CdmDataPartitionDefinition dataPart in decl.DataPartitions) { Console.WriteLine(" " + dataPart.Location); } } } // ------------------------------------------------------------------------------------------------------------ // Make changes to the model // Create a new document where the new entity's definition will be stored var newEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "NewEntity.cdm.json", false); newEntityDoc.Imports.Add("cdm:/foundations.cdm.json"); cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(newEntityDoc); var newEntity = newEntityDoc.Definitions.Add(CdmObjectType.EntityDef, "NewEntity") as CdmEntityDefinition; // Define new string attribute and add it to the entity definition var newAttribute = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "NewAttribute", false); newAttribute.DataFormat = CdmDataFormat.String; newEntity.Attributes.Add(newAttribute); // Call will create EntityDeclarationDefinition based on entity definition and add it to manifest.Entities var newEntityDecl = manifest.Entities.Add(newEntity); // Define a partition and add it to the local declaration var newPartition = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "NewPartition", false); newPartition.Location = "adls:/NewPartition.csv"; newEntityDecl.DataPartitions.Add(newPartition); // ------------------------------------------------------------------------------------------------------------ // Save the file to ADLSg2 - we achieve that by adding the manifest to the root folder of // the ADLS file-system and performing a save on the manifest CdmFolderDefinition adlsFolder = cdmCorpus.Storage.FetchRootFolder("adls"); adlsFolder.Documents.Add(manifest); await manifest.SaveAsAsync("example.model.json", true); }
public async Task TestLoadProjection() { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestLoadProjection"); CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json"); string expected = "TestSource"; string actual = null; CdmObjectType actualType = CdmObjectType.Error; #region TestEntityStringReference.cdm.json CdmEntityDefinition entTestEntityStringReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityStringReference.cdm.json/TestEntityStringReference", manifest); Assert.IsNotNull(entTestEntityStringReference); actual = ((CdmEntityReference)entTestEntityStringReference.ExtendsEntity).NamedReference; actualType = ((CdmEntityReference)entTestEntityStringReference.ExtendsEntity).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.EntityRef, actualType); #endregion // TestEntityStringReference.cdm.json #region TestEntityEntityReference.cdm.json CdmEntityDefinition entTestEntityEntityReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityEntityReference.cdm.json/TestEntityEntityReference", manifest); Assert.IsNotNull(entTestEntityEntityReference); actual = ((CdmEntityReference)entTestEntityEntityReference.ExtendsEntity).NamedReference; actualType = ((CdmEntityReference)entTestEntityEntityReference.ExtendsEntity).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.EntityRef, actualType); #endregion // TestEntityEntityReference.cdm.json #region TestEntityProjection.cdm.json CdmEntityDefinition entTestEntityProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityProjection.cdm.json/TestEntityProjection", manifest); Assert.IsNotNull(entTestEntityProjection); actual = ((CdmEntityReference)((CdmProjection)entTestEntityProjection.ExtendsEntity.ExplicitReference).Source).NamedReference; actualType = ((CdmProjection)entTestEntityProjection.ExtendsEntity.ExplicitReference).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.ProjectionDef, actualType); #endregion // TestEntityProjection.cdm.json #region TestEntityNestedProjection.cdm.json CdmEntityDefinition entTestEntityNestedProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityNestedProjection.cdm.json/TestEntityNestedProjection", manifest); Assert.IsNotNull(entTestEntityNestedProjection); actual = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entTestEntityNestedProjection.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).Source.NamedReference; actualType = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entTestEntityNestedProjection.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.ProjectionDef, actualType); #endregion // TestEntityNestedProjection.cdm.json #region TestEntityAttributeStringReference.cdm.json CdmEntityDefinition entTestEntityAttributeStringReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeStringReference.cdm.json/TestEntityAttributeStringReference", manifest); Assert.IsNotNull(entTestEntityAttributeStringReference); actual = ((CdmEntityAttributeDefinition)entTestEntityAttributeStringReference.Attributes[0]).Entity.NamedReference; actualType = ((CdmEntityAttributeDefinition)entTestEntityAttributeStringReference.Attributes[0]).Entity.ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.EntityRef, actualType); #endregion // TestEntityAttributeStringReference.cdm.json #region TestEntityAttributeEntityReference.cdm.json CdmEntityDefinition entTestEntityAttributeEntityReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeEntityReference.cdm.json/TestEntityAttributeEntityReference", manifest); Assert.IsNotNull(entTestEntityAttributeEntityReference); actual = ((CdmEntityAttributeDefinition)entTestEntityAttributeEntityReference.Attributes[0]).Entity.NamedReference; actualType = ((CdmEntityAttributeDefinition)entTestEntityAttributeEntityReference.Attributes[0]).Entity.ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.EntityRef, actualType); #endregion // TestEntityAttributeEntityReference.cdm.json #region TestEntityAttributeProjection.cdm.json CdmEntityDefinition entTestEntityAttributeProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeProjection.cdm.json/TestEntityAttributeProjection", manifest); Assert.IsNotNull(entTestEntityAttributeProjection); actual = ((CdmProjection)((CdmEntityAttributeDefinition)entTestEntityAttributeProjection.Attributes[0]).Entity.ExplicitReference).Source.NamedReference; actualType = ((CdmEntityAttributeDefinition)entTestEntityAttributeProjection.Attributes[0]).Entity.ExplicitReference.ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.ProjectionDef, actualType); #endregion // TestEntityAttributeProjection.cdm.json #region TestEntityAttributeNestedProjection.cdm.json CdmEntityDefinition entTestEntityAttributeNestedProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeNestedProjection.cdm.json/TestEntityAttributeNestedProjection", manifest); Assert.IsNotNull(entTestEntityAttributeNestedProjection); actual = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entTestEntityAttributeNestedProjection.Attributes[0]).Entity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).Source.NamedReference; actualType = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entTestEntityAttributeNestedProjection.Attributes[0]).Entity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(CdmObjectType.ProjectionDef, actualType); #endregion // TestEntityAttributeNestedProjection.cdm.json #region TestOperationCollection.cdm.json CdmEntityDefinition entTestOperationCollection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestOperationCollection.cdm.json/TestOperationCollection", manifest); Assert.IsNotNull(entTestOperationCollection); int actualOperationCount = ((CdmProjection)entTestOperationCollection.ExtendsEntity.ExplicitReference).Operations.Count; Assert.AreEqual(9, actualOperationCount); CdmOperationCollection operations = ((CdmProjection)entTestOperationCollection.ExtendsEntity.ExplicitReference).Operations; Assert.AreEqual(CdmOperationType.AddCountAttribute, operations[0].Type); Assert.AreEqual(CdmOperationType.AddSupportingAttribute, operations[1].Type); Assert.AreEqual(CdmOperationType.AddTypeAttribute, operations[2].Type); Assert.AreEqual(CdmOperationType.ExcludeAttributes, operations[3].Type); Assert.AreEqual(CdmOperationType.ArrayExpansion, operations[4].Type); Assert.AreEqual(CdmOperationType.CombineAttributes, operations[5].Type); Assert.AreEqual(CdmOperationType.RenameAttributes, operations[6].Type); Assert.AreEqual(CdmOperationType.ReplaceAsForeignKey, operations[7].Type); Assert.AreEqual(CdmOperationType.IncludeAttributes, operations[8].Type); #endregion // TestOperationCollection.cdm.json #region TestEntityTrait.cdm.json CdmEntityDefinition entTestEntityTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityTrait.cdm.json/TestEntityTrait", manifest); Assert.IsNotNull(entTestEntityTrait); Assert.AreEqual("TestAttribute", ((CdmTypeAttributeDefinition)entTestEntityTrait.Attributes[0]).Name); Assert.AreEqual("testDataType", ((CdmTypeAttributeDefinition)entTestEntityTrait.Attributes[0]).DataType.NamedReference); #endregion // TestEntityTrait.cdm.json #region TestEntityExtendsTrait.cdm.json CdmEntityDefinition entTestEntityExtendsTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityExtendsTrait.cdm.json/TestEntityExtendsTrait", manifest); Assert.IsNotNull(entTestEntityExtendsTrait); Assert.AreEqual("TestExtendsTraitAttribute", ((CdmTypeAttributeDefinition)entTestEntityExtendsTrait.Attributes[0]).Name); Assert.AreEqual("testDerivedDataType", ((CdmTypeAttributeDefinition)entTestEntityExtendsTrait.Attributes[0]).DataType.NamedReference); #endregion // TestEntityExtendsTrait.cdm.json #region TestProjectionTrait.cdm.json CdmEntityDefinition entTestProjectionTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestProjectionTrait.cdm.json/TestProjectionTrait", manifest); Assert.IsNotNull(entTestProjectionTrait); Assert.AreEqual("TestProjectionAttribute", ((CdmTypeAttributeDefinition)entTestProjectionTrait.Attributes[0]).Name); Assert.AreEqual("testDataType", ((CdmTypeAttributeDefinition)entTestProjectionTrait.Attributes[0]).DataType.NamedReference); #endregion // TestProjectionTrait.cdm.json #region TestProjectionExtendsTrait.cdm.json CdmEntityDefinition entTestProjectionExtendsTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestProjectionExtendsTrait.cdm.json/TestProjectionExtendsTrait", manifest); Assert.IsNotNull(entTestProjectionExtendsTrait); Assert.AreEqual("TestProjectionAttributeB", ((CdmTypeAttributeDefinition)entTestProjectionExtendsTrait.Attributes[0]).Name); Assert.AreEqual("testExtendsDataTypeB", ((CdmTypeAttributeDefinition)entTestProjectionExtendsTrait.Attributes[0]).DataType.NamedReference); #endregion // TestProjectionExtendsTrait.cdm.json }
static async Task Main() { // Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths var cdmCorpus = new CdmCorpusDefinition(); Console.WriteLine("Configuring storage adapters"); // Configure storage adapters to point at the target local manifest location and at the fake public standards var pathFromExeToExampleRoot = "../"; cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "output")); cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local // Fake cdm, normaly use the github adapter // Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "base-files")); // Example how to mount to the ADLS. // cdmCorpus.Storage.Mount("adls", // new ADLSAdapter( // "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. // "/<FILESYSTEM-NAME>", // Root. // "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. // "<CLIENT-ID>", // Client ID. // "<CLIENT-SECRET>" // Client secret. // )); Console.WriteLine("Creating placeholder manifest"); // Make the temp manifest and add it to the root of the local documents in the corpus CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract"); // Add the temp manifest to the root of the local documents in the corpus var localRoot = cdmCorpus.Storage.FetchRootFolder("local"); localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json"); //READ CSV - OBSERVE SCHEMA: "entity, atribute, datatype" every row, no headers Console.WriteLine("Loading CDM definition file"); List <string> entitylist = new List <string>(); List <string> attributelist = new List <string>(); List <string> datatypelist = new List <string>(); using (var reader = new System.IO.StreamReader(@"../cdmdefinition.csv")) { while (!reader.EndOfStream) { var line = reader.ReadLine(); var values = line.Split(','); entitylist.Add(values[0]); attributelist.Add(values[1]); datatypelist.Add(values[2]); } } // Create entities and add attributes string entityname; CdmEntityDefinition entity; bool isnew = true; for (int i = 0; i < entitylist.Count; i++) { isnew = true; entityname = entitylist[i]; Console.WriteLine("Creating " + entityname + " entity"); entity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, entityname, false); while (i == 0 || entitylist[i] == entitylist[i - 1] || isnew) { isnew = false; var attribute = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, attributelist[i], "hasA", datatypelist[i]); entity.Attributes.Add(attribute); i++; if (i == entitylist.Count) { break; } } if (i != entitylist.Count) { i--; } // Create the document which contains the entity var entitydoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityname}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely entitydoc.Imports.Add(FoundationJsonPath); entitydoc.Definitions.Add(entity); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(entitydoc, entitydoc.Name); // Add the entity to the manifest manifestAbstract.Entities.Add(entity); } //THIAGO - MANUAL BREAKDOWN BEFORE AUTOMATING /*var entityname = "FixedAssetMaster"; * var entity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, entityname, false); * var attribute1 = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, "Attribute1", "hasA", "string"); * entity.Attributes.Add(attribute1); * var attribute2 = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, "Attribute2", "hasA", "string"); * entity.Attributes.Add(attribute2); * // Create the document which contains the entity * var entitydoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityname}.cdm.json", false); * // Add an import to the foundations doc so the traits about partitons will resolve nicely * entitydoc.Imports.Add(FoundationJsonPath); * entitydoc.Definitions.Add(entity); * // Add the document to the root of the local documents in the corpus * localRoot.Documents.Add(entitydoc, entitydoc.Name); * // Add the entity to the manifest * manifestAbstract.Entities.Add(entity);*/ /* * // Create the simplest entity - CustomPerson * // Create the entity definition instance * var personEntity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false); * // Add type attributes to the entity instance * var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId"); * personEntity.Attributes.Add(personAttributeId); * var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "string"); * personEntity.Attributes.Add(personAttributeName); * // Add properties to the entity instance * personEntity.DisplayName = CustomPersonEntityName; * personEntity.Version = "0.0.1"; * personEntity.Description = "This is a custom entity created for the sample."; * // Create the document which contains the entity * var personEntityDoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false); * // Add an import to the foundations doc so the traits about partitons will resolve nicely * personEntityDoc.Imports.Add(FoundationJsonPath); * personEntityDoc.Definitions.Add(personEntity); * // Add the document to the root of the local documents in the corpus * localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name); * // Add the entity to the manifest * manifestAbstract.Entities.Add(personEntity); * * * // Create an entity - CustomAccount which has a relationship with the entity CustomPerson * // Create the entity definition instance * var accountEntity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false); * // Add type attributes to the entity instance * var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId"); * accountEntity.Attributes.Add(accountAttributeId); * var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "string"); * accountEntity.Attributes.Add(accountAttributeName); * // Add properties to the entity instance * accountEntity.DisplayName = CustomAccountEntityName; * accountEntity.Version = "0.0.1"; * accountEntity.Description = "This is a custom entity created for the sample."; * // In this sample, every account has one person who owns the account * // the relationship is actually an entity attribute * var attributeExplanation = "The owner of the account, which is a person."; * // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show * // more details of how to use resolution guidance to customize your data * var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation); * accountEntity.Attributes.Add(accountOwnerAttribute); * // Create the document which contains the entity * var accountEntityDoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false); * // Add an import to the foundations doc so the traits about partitons will resolve nicely * accountEntityDoc.Imports.Add(FoundationJsonPath); * // the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits, * // the import to the entity reference CustomPerson's doc is required * accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json"); * accountEntityDoc.Definitions.Add(accountEntity); * // Add the document to the root of the local documents in the corpus * localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name); * // Add the entity to the manifest * manifestAbstract.Entities.Add(accountEntity); * * * // Create an entity which extends "Account" from the standard, it contains everything that "Account" has * var extendedStandardAccountEntity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false); * // This function with 'true' will make a simple reference to the base * extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject<CdmEntityReference>(CdmObjectType.EntityRef, "Account", true); * var attrExplanation = "This is a simple custom account for this sample."; * // Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount * // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show * // more details of how to use resolution guidance to customize your data * var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation); * extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute); * var extendedStandardAccountEntityDoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false); * // Add an import to the foundations doc so the traits about partitons will resolve nicely * extendedStandardAccountEntityDoc.Imports.Add(FoundationJsonPath); * // The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required * // it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required * extendedStandardAccountEntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json"); * extendedStandardAccountEntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json"); * // Add the document to the root of the local documents in the corpus * localRoot.Documents.Add(extendedStandardAccountEntityDoc, extendedStandardAccountEntityDoc.Name); * extendedStandardAccountEntityDoc.Definitions.Add(extendedStandardAccountEntity); * // Add the entity to the manifest * manifestAbstract.Entities.Add(extendedStandardAccountEntity); */ // Create the resolved version of everything in the root folder too Console.WriteLine("Resolving the placeholder"); var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null); // Add an import to the foundations doc so the traits about partitons will resolve nicely manifestResolved.Imports.Add(FoundationJsonPath); Console.WriteLine("Saving the documents"); foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities) { // Get the entity being pointed at var localEDef = eDef; var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved); // Make a fake partition, just to demo that var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description"); localEDef.DataPartitions.Add(part); part.Explanation = "not real data, just for demo"; // We have existing partition files for the custom entities, so we need to make the partition point to the file location part.Location = $"local:/{entDef.EntityName}/partition-data.csv"; // Add trait to partition for csv params var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false); csvTrait.Arguments.Add("columnHeaders", "true"); csvTrait.Arguments.Add("delimiter", ","); } // We can save the documents as manifest.cdm.json format or model.json // Save as manifest.cdm.json await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true); // Save as a model.json // await manifestResolved.SaveAsAsync("model.json", true); Console.WriteLine("Finished"); Console.ReadKey(); }
private async Task LogicalManipulationUsingProjections(CdmCorpusDefinition corpus) { Console.WriteLine("Create logical entity definition."); var logicalFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("output:/"); var logicalDoc = logicalFolder.Documents.Add("Person.cdm.json"); logicalDoc.Imports.Add("local:/Address.cdm.json"); var entity = logicalDoc.Definitions.Add("Person"); // Add "name" data typed attribute. var nameAttr = entity.Attributes.Add("name") as CdmTypeAttributeDefinition; nameAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true); // Add "age" data typed attribute. var ageAttr = entity.Attributes.Add("age") as CdmTypeAttributeDefinition; ageAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true); // Add "address" entity typed attribute. var entityAttr = new CdmEntityAttributeDefinition(corpus.Ctx, "address") { Entity = new CdmEntityReference(corpus.Ctx, "Address", true) }; ApplyArrayExpansion(entityAttr, 1, 3, "{m}{A}{o}", "countAttribute"); ApplyDefaultBehavior(entityAttr, "addressFK", "address"); entity.Attributes.Add(entityAttr); // Add "email" data typed attribute. var emailAttr = entity.Attributes.Add("email") as CdmTypeAttributeDefinition; emailAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true); // Save the logical definition of Person. await entity.InDocument.SaveAsAsync("Person.cdm.json"); Console.WriteLine("Get \"resolved\" folder where the resolved entities will be saved."); var resolvedFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("output:/"); var resOpt = new ResolveOptions(entity); // To get more information about directives and their meaning refer to // https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes // We will start by resolving this entity with the "normalized" direcitve. // This directive will be used on this and the next two examples so we can analize the resolved entity // without the array expansion. Console.WriteLine("Resolving logical entity with normalized directive."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized" }); var resNormalizedEntity = await entity.CreateResolvedEntityAsync($"normalized_{entity.EntityName}", resOpt, resolvedFolder); await resNormalizedEntity.InDocument.SaveAsAsync($"{resNormalizedEntity.EntityName}.cdm.json"); // Another common scenario is to resolve an entity using the "referenceOnly" directive. // This directives is used to replace the relationships with a foreign key. Console.WriteLine("Resolving logical entity with referenceOnly directive."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "referenceOnly" }); var resReferenceOnlyEntity = await entity.CreateResolvedEntityAsync($"referenceOnly_{entity.EntityName}", resOpt, resolvedFolder); await resReferenceOnlyEntity.InDocument.SaveAsAsync($"{resReferenceOnlyEntity.EntityName}.cdm.json"); // When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that // a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the // entity attribute and we use the "structured" directive to resolve the entity attribute as an attribute group. Console.WriteLine("Resolving logical entity with structured directive."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "structured" }); var resStructuredEntity = await entity.CreateResolvedEntityAsync($"structured_{entity.EntityName}", resOpt, resolvedFolder); await resStructuredEntity.InDocument.SaveAsAsync($"{resStructuredEntity.EntityName}.cdm.json"); // Now let us remove the "normalized" directive so the array expansion operation can run. Console.WriteLine("Resolving logical entity without directives (array expansion)."); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { }); var resArrayEntity = await entity.CreateResolvedEntityAsync($"array_expansion_{entity.EntityName}", resOpt, resolvedFolder); await resArrayEntity.InDocument.SaveAsAsync($"{resArrayEntity.EntityName}.cdm.json"); }
/// <summary> /// Common test code for these test cases /// </summary> /// <param name="testName"></param> /// <param name="entityName"></param> private async Task TestRun(string testName, string entityName, bool isEntitySet) { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName); string inputFolder = TestHelper.GetInputFolderPath(testsSubpath, testName); string expectedOutputFolder = TestHelper.GetExpectedOutputFolderPath(testsSubpath, testName); string actualOutputFolder = TestHelper.GetActualOutputFolderPath(testsSubpath, testName); if (!Directory.Exists(actualOutputFolder)) { Directory.CreateDirectory(actualOutputFolder); } CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json"); Assert.IsNotNull(manifest); CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}", manifest); Assert.IsNotNull(entity); CdmEntityDefinition resolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> { "referenceOnly" }); AssertEntityShapeInResolvedEntity(resolvedEntity, isEntitySet); await AttributeContextUtil.ValidateAttributeContext(corpus, expectedOutputFolder, entityName, resolvedEntity); await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); string actualRelationshipsString = ListRelationships(corpus, entity, actualOutputFolder, entityName); string relationshipsFilename = $"REL_{entityName}.txt"; File.WriteAllText(Path.Combine(actualOutputFolder, relationshipsFilename), actualRelationshipsString); string expectedRelationshipsStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, relationshipsFilename)); string expectedRelationshipsString = File.ReadAllText(expectedRelationshipsStringFilePath); Assert.AreEqual(expectedRelationshipsString, actualRelationshipsString); CdmFolderDefinition outputFolder = corpus.Storage.FetchRootFolder("output"); outputFolder.Documents.Add(manifest); string manifestFileName = $"saved.manifest.cdm.json"; await manifest.SaveAsAsync(manifestFileName, saveReferenced : true); string actualManifestPath = Path.Combine(actualOutputFolder, manifestFileName); if (!File.Exists(actualManifestPath)) { Assert.Fail("Unable to save manifest with relationship"); } else { CdmManifestDefinition savedManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"output:/{manifestFileName}"); string actualSavedManifestRel = GetRelationshipStrings(savedManifest.Relationships); string manifestRelationshipsFilename = $"MANIFEST_REL_{entityName}.txt"; File.WriteAllText(Path.Combine(actualOutputFolder, manifestRelationshipsFilename), actualSavedManifestRel); string expectedSavedManifestRel = File.ReadAllText(Path.Combine(expectedOutputFolder, manifestRelationshipsFilename)); Assert.AreEqual(expectedSavedManifestRel, actualSavedManifestRel); } }
static async Task Main(string[] args) { Console.WriteLine("Hello World!"); string pathToDocRoot; string docGroup; CdmManifestDefinition manifest; string testEnt; bool testCorpus = false; bool resolveEnt = false; bool spewAll = true; bool rePersist = false; CdmCorpusDefinition cdmCorpus = new CdmCorpusDefinition(); cdmCorpus.Storage.DefaultNamespace = "local"; if (testCorpus) { pathToDocRoot = "../../../../../../../../CDM.Tools.Internal/TestCorpus"; //pathToDocRoot = "../perfTestCorpus"; //docGroup = "E2EResolution"; //docGroup = "POVResolution"; //docGroup = "MiniDyn"; //docGroup = "composites"; //docGroup = "KnowledgeGraph"; //docGroup = "overrides"; docGroup = "webClicks"; //testEnt = "/E2EResolution/E2EArrayOne.cdm.json/E2EArrayOne"; //testEnt = "/MiniDyn/sub/Lead.cdm.json/Lead"; // testEnt = "/POVResolution/sub1/Main.cdm.json/Main"; testEnt = "local:/MiniDyn/Account.cdm.json/Account"; } else { pathToDocRoot = "../../../../../../../../CDM.SchemaDocuments"; testEnt = "local:/core/applicationCommon/foundationCommon/crmCommon/Account.cdm.json/Account"; docGroup = "standards"; } cdmCorpus.Storage.Mount("local", new LocalAdapter(pathToDocRoot)); manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{docGroup}.manifest.cdm.json"); string version = ""; cdmCorpus.SetEventCallback(new EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Progress); Console.WriteLine("reading source files"); if (resolveEnt) { // AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet<string> { "normalized", "xstructured", "referenceOnly" }); AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "referenceOnly" }); var ent = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(testEnt); ResolveOptions resOpt = new ResolveOptions { WrtDoc = ent.InDocument, Directives = directives }; var x = await ent.CreateResolvedEntityAsync("RESOLVED_KILL", resOpt); resOpt.WrtDoc = x.InDocument; CommonDataModelLoader.PersistDocument(cdmCorpus.RootPath, resOpt, new CopyOptions { StringRefs = false, RemoveSingleRowLocalizedTableTraits = true }); } if (spewAll) { Console.WriteLine("list all resolved"); AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "xstructured", "referenceOnly" }); await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher()); } if (rePersist) { Console.WriteLine("persist corpus"); AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "xstructured", "referenceOnly" }); } //ListAllTraits(cdmCorpus); Console.WriteLine("done"); Console.ReadLine(); }
static async Task Main(string[] args) { // Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths var cdmCorpus = new CdmCorpusDefinition(); Console.WriteLine("Configure storage adapters"); // Configure storage adapters to point at the target local manifest location and at the fake public standards string pathFromExeToExampleRoot = "../../../../../../"; // Mount is as a local device. cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "3-customize-entities")); cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local // Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); // Example how to mount to the ADLS. // cdmCorpus.Storage.Mount("adls", // new ADLSAdapter( // "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. // "/<FILESYSTEM-NAME>", // Root. // "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. // "<CLIENT-ID>", // Client ID. // "<CLIENT-SECRET>" // Client secret. // )); // Open the default manifest at the root, used later when done // This method turns relative corpus paths into absolute ones in case we are in some sub-folders and don't know it var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("default.manifest.cdm.json"); Console.WriteLine("Define new extension"); // First we will make a new document right in the same folder as the manifest var docAbs = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "MobileCareTeam.cdm.json"); // Import the cdm description of the original so the symbols will resolve docAbs.Imports.Add("cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CareTeam.cdm.json", null); // We will make a new trait to identify things that are known to be temporary, used later // In theory this would be defined somewhere central so it can be shared var traitTemp = docAbs.Definitions.Add(CdmObjectType.TraitDef, "means.temporary") as CdmTraitDefinition; // Extends the standard 'means' base trait traitTemp.ExtendsTrait = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means", true); // Add a parameter for the expected duration in days var param = cdmCorpus.MakeObject <CdmParameterDefinition>(CdmObjectType.ParameterDef, "estimatedDays"); // By not using "true" on the last arg, this becomes an real reference object in the json. go look at the difference from "means" when this is done param.DataTypeRef = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "integer"); param.DefaultValue = "30"; traitTemp.Parameters.Add(param); // Make an entity definition and add it to the list of definitions in the document. CdmEntityDefinition entAbs = docAbs.Definitions.Add(CdmObjectType.EntityDef, "MobileCareTeam") as CdmEntityDefinition; // This entity extends the standard // This function with 'true' will make a simple reference to the base entAbs.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "CareTeam", true); // and we will add an attribute CdmTypeAttributeDefinition attNew = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "currentCity"); // The attribute is a type is 'City" this is one of the predefined semantic types in meanings.cdm.json attNew.DataType = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "city", true); attNew.Description = "The current city where the mobile care team is working"; // also apply our fancy new 'temporary' trait. they stay in a city for 90 days on average CdmTraitReference tr = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means.temporary"); tr.Arguments.Add("estimatedDays", "90"); attNew.AppliedTraits.Add(tr); // Add attribute to the entity entAbs.Attributes.Add(attNew); // The entity abstract definition is done, add the document to the corpus in the root folder and then save that doc cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(docAbs); // next step is to remove all of the guesswork out of decoding the entity shape by 'resolving' it to a relational by reference shape Console.WriteLine("Make a local 'resolved' copy"); // Now resolve it // Made the entity and document have a different name to avoid conflicts in this folder var entFlat = await entAbs.CreateResolvedEntityAsync("LocalMobileCareTeam"); // Now just add the pointer into our manifest. Console.WriteLine("Add to manifest"); manifest.Entities.Add(entFlat); // This function will update all of the fileStatus times in the manifest // await manifest.RefreshFileStatus(); // Save the manifest along with linked definition files await manifest.SaveAsAsync("default-resolved.manifest.cdm.json", true); }
private async Task CustomizeEntities(CdmCorpusDefinition cdmCorpus) { // Open the default manifest at the root, used later when done // This method turns relative corpus paths into absolute ones in case we are in some sub-folders and don't know it var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("default.manifest.cdm.json"); Console.WriteLine("Define new extension"); // First we will make a new document right in the same folder as the manifest var docAbs = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "MobileCareTeam.cdm.json"); // Import the cdm description of the original so the symbols will resolve docAbs.Imports.Add("cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CareTeam.cdm.json", null); // We will make a new trait to identify things that are known to be temporary, used later // In theory this would be defined somewhere central so it can be shared var traitTemp = docAbs.Definitions.Add(CdmObjectType.TraitDef, "means.temporary") as CdmTraitDefinition; // Extends the standard 'means' base trait traitTemp.ExtendsTrait = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means", true); // Add a parameter for the expected duration in days var param = cdmCorpus.MakeObject <CdmParameterDefinition>(CdmObjectType.ParameterDef, "estimatedDays"); // By not using "true" on the last arg, this becomes an real reference object in the json. go look at the difference from "means" when this is done param.DataTypeRef = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "integer"); param.DefaultValue = "30"; traitTemp.Parameters.Add(param); // Make an entity definition and add it to the list of definitions in the document. CdmEntityDefinition entAbs = docAbs.Definitions.Add(CdmObjectType.EntityDef, "MobileCareTeam") as CdmEntityDefinition; // This entity extends the standard // This function with 'true' will make a simple reference to the base entAbs.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "CareTeam", true); // and we will add an attribute CdmTypeAttributeDefinition attNew = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "currentCity"); // The attribute is a type is 'City" this is one of the predefined semantic types in meanings.cdm.json attNew.DataType = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "city", true); attNew.Description = "The current city where the mobile care team is working."; // also apply our fancy new 'temporary' trait. they stay in a city for 90 days on average CdmTraitReference tr = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means.temporary"); tr.Arguments.Add("estimatedDays", "90"); attNew.AppliedTraits.Add(tr); // Add attribute to the entity entAbs.Attributes.Add(attNew); // The entity abstract definition is done, add the document to the corpus in the root folder and then save that doc cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(docAbs); // next step is to remove all of the guesswork out of decoding the entity shape by 'resolving' it to a relational by reference shape Console.WriteLine("Make a local 'resolved' copy"); // Now resolve it // Made the entity and document have a different name to avoid conflicts in this folder var entFlat = await entAbs.CreateResolvedEntityAsync("LocalMobileCareTeam"); // Now just add the pointer into our manifest. Console.WriteLine("Add to manifest"); manifest.Entities.Add(entFlat); // This function will update all of the fileStatus times in the manifest // await manifest.RefreshAsync(null); // Save the manifest along with linked definition files. await manifest.SaveAsAsync("default-resolved.manifest.cdm.json", true); }
private async Task CreateNetNewEntities(CdmCorpusDefinition cdmCorpus) { Console.WriteLine("Make placeholder manifest"); // Make the temp manifest and add it to the root of the local documents in the corpus CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract"); // Add the temp manifest to the root of the local adapter in the corpus var localRoot = cdmCorpus.Storage.FetchRootFolder("local"); localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json"); // Create two entities from scratch, and add some attributes, traits, properties, and relationships in between Console.WriteLine("Create net new entities"); // Create the simplest entity - CustomPerson // Create the entity definition instance var personEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false); // Add type attributes to the entity instance var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId"); personEntity.Attributes.Add(personAttributeId); var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "name"); personEntity.Attributes.Add(personAttributeName); // Add properties to the entity instance personEntity.DisplayName = CustomPersonEntityName; personEntity.Version = "0.0.1"; personEntity.Description = "This is a custom entity created for the sample."; // Create the document which contains the entity var personEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely personEntityDoc.Imports.Add(FoundationJsonPath); personEntityDoc.Definitions.Add(personEntity); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name); // Add the entity to the manifest manifestAbstract.Entities.Add(personEntity); // Create an entity - CustomAccount which has a relationship with the entity CustomPerson // Create the entity definition instance var accountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false); // Add type attributes to the entity instance var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId"); accountEntity.Attributes.Add(accountAttributeId); var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "name"); accountEntity.Attributes.Add(accountAttributeName); // Add properties to the entity instance accountEntity.DisplayName = CustomAccountEntityName; accountEntity.Version = "0.0.1"; accountEntity.Description = "This is a custom entity created for the sample."; // In this sample, every account has one person who owns the account // the relationship is actually an entity attribute var attributeExplanation = "The owner of the account, which is a person."; // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show // more details of how to use resolution guidance to customize your data var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation); accountEntity.Attributes.Add(accountOwnerAttribute); // Create the document which contains the entity var accountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely accountEntityDoc.Imports.Add(FoundationJsonPath); // the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits, // the import to the entity reference CustomPerson's doc is required accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json"); accountEntityDoc.Definitions.Add(accountEntity); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name); // Add the entity to the manifest manifestAbstract.Entities.Add(accountEntity); // Create an entity which extends "Account" from the standard, it contains everything that "Account" has var extendedStandardAccountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false); // This function with 'true' will make a simple reference to the base extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "Account", true); var attrExplanation = "This is a simple custom account for this sample."; // Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show // more details of how to use resolution guidance to customize your data var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation); extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute); var extendedStandardAccountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely extendedStandardAccountEntityDoc.Imports.Add(FoundationJsonPath); // The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required // it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required extendedStandardAccountEntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json"); extendedStandardAccountEntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json"); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(extendedStandardAccountEntityDoc, extendedStandardAccountEntityDoc.Name); extendedStandardAccountEntityDoc.Definitions.Add(extendedStandardAccountEntity); // Add the entity to the manifest manifestAbstract.Entities.Add(extendedStandardAccountEntity); // Create the resolved version of everything in the root folder too Console.WriteLine("Resolve the placeholder"); var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null); // Add an import to the foundations doc so the traits about partitons will resolve nicely manifestResolved.Imports.Add(FoundationJsonPath); Console.WriteLine("Save the documents"); foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities) { // Get the entity being pointed at var localEDef = eDef; var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved); // Make a fake partition, just to demo that var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description"); localEDef.DataPartitions.Add(part); part.Explanation = "not real data, just for demo"; // We have existing partition files for the custom entities, so we need to make the partition point to the file location part.Location = $"local:/{entDef.EntityName}/partition-data.csv"; // Add trait to partition for csv params var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false); csvTrait.Arguments.Add("columnHeaders", "true"); csvTrait.Arguments.Add("delimiter", ","); } // We can save the documents as manifest.cdm.json format or model.json // Save as manifest.cdm.json await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true); // Save as a model.json // await manifestResolved.SaveAsAsync("model.json", true); }
public async Task TestResolvedAttributeLimit() { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestResolvedAttributeLimit"); CdmEntityDefinition mainEntity = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/mainEntity.cdm.json/mainEntity"); ResolveOptions resOpt = new ResolveOptions { WrtDoc = mainEntity.InDocument, Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "referenceOnly" }) }; // if attribute limit is reached, entity should be null resOpt.ResolvedAttributeLimit = 4; var resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_zeroAtts", resOpt); Assert.IsNull(resEnt); // when the attribute limit is set to null, there should not be a limit on the possible number of attributes resOpt.ResolvedAttributeLimit = null; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "referenceOnly" }); var ras = mainEntity.FetchResolvedAttributes(resOpt); resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_normalized_referenceOnly", resOpt); // there are 5 total attributes Assert.AreEqual(ras.ResolvedAttributeCount, 5); Assert.AreEqual(ras.Set.Count, 5); Assert.AreEqual(mainEntity.Attributes.Count, 3); // there are 2 attributes grouped in an entity attribute // and 2 attributes grouped in an attribute group Assert.AreEqual(((mainEntity.Attributes[2] as CdmAttributeGroupReference).ExplicitReference as CdmAttributeGroupDefinition).Members.Count, 2); // using the default limit number resOpt = new ResolveOptions { WrtDoc = mainEntity.InDocument, Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "referenceOnly" }) }; ras = mainEntity.FetchResolvedAttributes(resOpt); resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_normalized_referenceOnly", resOpt); // there are 5 total attributes Assert.AreEqual(ras.ResolvedAttributeCount, 5); Assert.AreEqual(ras.Set.Count, 5); Assert.AreEqual(mainEntity.Attributes.Count, 3); // there are 2 attributes grouped in an entity attribute // and 2 attributes grouped in an attribute group Assert.AreEqual(((mainEntity.Attributes[2] as CdmAttributeGroupReference).ExplicitReference as CdmAttributeGroupDefinition).Members.Count, 2); resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "structured" }); ras = mainEntity.FetchResolvedAttributes(resOpt); resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_normalized_structured", resOpt); // there are 5 total attributes Assert.AreEqual(ras.ResolvedAttributeCount, 5); // the attribute count is different because one attribute is a group that contains two different attributes Assert.AreEqual(ras.Set.Count, 4); Assert.AreEqual(mainEntity.Attributes.Count, 3); // again there are 2 attributes grouped in an entity attribute // and 2 attributes grouped in an attribute group Assert.AreEqual(((mainEntity.Attributes[2] as CdmAttributeGroupReference).ExplicitReference as CdmAttributeGroupDefinition).Members.Count, 2); }
static async Task Main(string[] args) { // Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths var cdmCorpus = new CdmCorpusDefinition(); Console.WriteLine("Configure storage adapters"); // Configure storage adapters to point at the target local manifest location and at the fake public standards string pathFromExeToExampleRoot = "../../../../../../"; cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "2-create-manifest")); cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local // Fake cdm, normaly use the github adapter // Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); // Example how to mount to the ADLS. // cdmCorpus.Storage.Mount("adls", // new ADLSAdapter( // "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. // "/<FILESYSTEM-NAME>", // Root. // "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. // "<CLIENT-ID>", // Client ID. // "<CLIENT-SECRET>" // Client secret. // )); Console.WriteLine("Make placeholder manifest"); // Make the temp manifest and add it to the root of the local documents in the corpus CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract"); // Add each declaration, this example is about medical appointments and care plans manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account"); manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address"); manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan"); manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept"); manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact"); manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device"); manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment"); manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter"); manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare"); manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location"); // Add the temp manifest to the root of the local documents in the corpus. var localRoot = cdmCorpus.Storage.FetchRootFolder("local"); localRoot.Documents.Add(manifestAbstract); // Create the resolved version of everything in the root folder too Console.WriteLine("Resolve the placeholder"); var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", ""); // Add an import to the foundations doc so the traits about partitons will resolve nicely manifestResolved.Imports.Add("cdm:/foundations.cdm.json"); Console.WriteLine("Save the docs"); foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities) { // Get the entity being pointed at var localEDef = eDef; var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved); // Make a fake partition, just to demo that var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description"); localEDef.DataPartitions.Add(part); part.Explanation = "not real data, just for demo"; // Define the location of the partition, relative to the manifest var location = $"local:/{entDef.EntityName}/partition-data.csv"; part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved); // Add trait to partition for csv params var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false); csvTrait.Arguments.Add("columnHeaders", "true"); csvTrait.Arguments.Add("delimiter", ","); // Get the actual location of the partition file from the corpus string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location); // Make a fake file with nothing but header for columns string header = ""; foreach (CdmTypeAttributeDefinition att in entDef.Attributes) { if (header != "") { header += ","; } header += att.Name; } Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{ entDef.EntityName}")); File.WriteAllText(partPath, header); } await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true); }
public async Task TestWithoutNesting() { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList"); corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId); // Test fetching an object from invalid namespace results in at least one error message in the recorder _ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar"); Assert.IsNotNull(corpus.Ctx.Events, "Ctx.Events should not be null"); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded when fetching object with incorrect path"); Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key"); Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value"); // Test fetching a good object, this should leave event recorder empty _ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json"); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count == 0, "There should have been no events recorded when fetching object with correct path"); // Test saving a manifest to invalid namespace results in at least one error message in the recorder var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy"); await manifest.SaveAsAsync("foo:/bar", true); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded"); Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key"); Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value"); // Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder await manifest.CreateResolvedManifestAsync("new dummy", null); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded"); Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key"); Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value"); // Test resolving an entity without WRT doc, this should yield at least one error message in the recorder var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2"); await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved"); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded"); Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key"); Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value"); // Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder await manifest.FileStatusCheckAsync(); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded"); Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key"); Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value"); // Repeat the same test but with status level 'None', no events should be recorded corpus.Ctx.ReportAtLevel = CdmStatusLevel.None; await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved"); Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call"); Assert.IsTrue(corpus.Ctx.Events.Count == 0, "There should have been no events recorded when fetching object with correct path"); }
public async Task TestUpdateRelationships() { var expectedRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestUpdateRelationships", "expectedRels.json")).ToObject <List <E2ERelationship> >(); string tempFromFilePath = "fromEntTemp.cdm.json"; string tempFromEntityPath = "local:/fromEntTemp.cdm.json/fromEnt"; string tempToEntityPath = "local:/toEnt.cdm.json/toEnt"; // Initialize corpus and entity files CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestUpdateRelationships"); CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/main.manifest.cdm.json"); CdmManifestDefinition manifestNoToEnt = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/mainNoToEnt.manifest.cdm.json"); CdmEntityDefinition fromEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/fromEnt.cdm.json/fromEnt"); await fromEnt.InDocument.SaveAsAsync(tempFromFilePath, options : new CopyOptions() { IsTopLevelDocument = false }); async Task reloadFromEntity() { await fromEnt.InDocument.SaveAsAsync(tempFromFilePath, options : new CopyOptions() { IsTopLevelDocument = false }); // fetch again to reset the cache await corpus.FetchObjectAsync <CdmEntityDefinition>(tempFromEntityPath, null, false, true); } try { // 1. test when entity attribute is removed await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); // check that the relationship has been created correctly VerifyRelationships(manifest, expectedRels); // now remove the entity attribute, which removes the relationship CdmAttributeItem removedAttribute = fromEnt.Attributes[0]; fromEnt.Attributes.RemoveAt(0); await reloadFromEntity(); await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); // check that the relationship has been removed VerifyRelationships(manifest, new List <E2ERelationship>()); // 2. test when the to entity is removed // restore the entity to the original state fromEnt.Attributes.Add(removedAttribute); await reloadFromEntity(); await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); // check that the relationship has been created correctly VerifyRelationships(manifest, expectedRels); // remove the to entity fromEnt.Attributes.RemoveAt(0); await reloadFromEntity(); // fetch again to reset the cache await corpus.FetchObjectAsync <CdmEntityDefinition>(tempToEntityPath, null, false, true); await corpus.CalculateEntityGraphAsync(manifestNoToEnt); await manifestNoToEnt.PopulateManifestRelationshipsAsync(); // check that the relationship has been removed VerifyRelationships(manifestNoToEnt, new List <E2ERelationship>()); } finally { // clean up created files created string fromPath = corpus.Storage.CorpusPathToAdapterPath($"local:/{tempFromFilePath}"); File.Delete(fromPath); } }
/// <summary> /// This method runs the tests with a set expected attributes & attribute context values and validated the actual result /// </summary> /// <param name="testName"></param> /// <param name="sourceEntityName"></param> /// <param name="expectedContext_*">expected attribute context object - for each resolution option combination</param> /// <param name="expected_*">expected attribute object - for each resolution option combination</param> /// <returns></returns> protected static async Task RunTestWithValues( string testName, string sourceEntityName, AttributeContextExpectedValue expectedContext_default, AttributeContextExpectedValue expectedContext_normalized, AttributeContextExpectedValue expectedContext_referenceOnly, AttributeContextExpectedValue expectedContext_structured, AttributeContextExpectedValue expectedContext_normalized_structured, AttributeContextExpectedValue expectedContext_referenceOnly_normalized, AttributeContextExpectedValue expectedContext_referenceOnly_structured, AttributeContextExpectedValue expectedContext_referenceOnly_normalized_structured, List <AttributeExpectedValue> expected_default, List <AttributeExpectedValue> expected_normalized, List <AttributeExpectedValue> expected_referenceOnly, List <AttributeExpectedValue> expected_structured, List <AttributeExpectedValue> expected_normalized_structured, List <AttributeExpectedValue> expected_referenceOnly_normalized, List <AttributeExpectedValue> expected_referenceOnly_structured, List <AttributeExpectedValue> expected_referenceOnly_normalized_structured, bool updateExpectedOutput = false ) { try { string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName); string testActualPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName); string testExpectedPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName); string corpusPath = testInputPath.Substring(0, testInputPath.Length - "/Input".Length); testActualPath = Path.GetFullPath(testActualPath); CdmCorpusDefinition corpus = new CdmCorpusDefinition(); corpus.SetEventCallback(new EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); corpus.Storage.Mount("local", new LocalAdapter(corpusPath)); corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath)); corpus.Storage.DefaultNamespace = "local"; string outFolderPath = corpus.Storage.AdapterPathToCorpusPath(testActualPath) + "/"; // interesting 'bug' CdmFolderDefinition outFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>(outFolderPath); CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/Input/{sourceEntityName}.cdm.json/{sourceEntityName}"); Assert.IsTrue(srcEntityDef != null); var resOpt = new ResolveOptions { WrtDoc = srcEntityDef.InDocument, Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { }) }; CdmEntityDefinition resolvedEntityDef = null; string outputEntityName = string.Empty; string outputEntityFileName = string.Empty; string entityFileName = string.Empty; if (expectedContext_default != null && expected_default != null) { entityFileName = "d"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_normalized != null && expected_normalized != null) { entityFileName = "n"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_referenceOnly != null && expected_referenceOnly != null) { entityFileName = "ro"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_structured != null && expected_structured != null) { entityFileName = "s"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "structured" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_normalized_structured != null && expected_normalized_structured != null) { entityFileName = "n_s"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "normalized", "structured" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_referenceOnly_normalized != null && expected_referenceOnly_normalized != null) { entityFileName = "ro_n"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly", "normalized" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_referenceOnly_structured != null && expected_referenceOnly_structured != null) { entityFileName = "ro_s"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly", "structured" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } if (expectedContext_referenceOnly_normalized_structured != null && expected_referenceOnly_normalized_structured != null) { entityFileName = "ro_n_s"; resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> { "referenceOnly", "normalized", "structured" }); outputEntityName = $"{sourceEntityName}_R_{entityFileName}"; outputEntityFileName = $"{outputEntityName}.cdm.json"; resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder); await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput); } } catch (Exception e) { Assert.Fail(e.Message); } }
public async Task TestWithoutNesting() { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList"); corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId); // Test fetching an object from invalid namespace results in at least one error message in the recorder _ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar"); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNamespaceNotRegistered, true); // Test fetching a good object, this should leave event recorder empty _ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json"); TestNoLogsState(corpus); // Test saving a manifest to invalid namespace results in at least one error message in the recorder var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy"); await manifest.SaveAsAsync("foo:/bar", true); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrValdnMissingDoc, true); // Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder await manifest.CreateResolvedManifestAsync("new dummy", null); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrResolveManifestFailed, true); // Test resolving an entity without WRT doc, this should yield at least one error message in the recorder var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2"); await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved"); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrDocWrtDocNotfound, true); // Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder await manifest.FileStatusCheckAsync(); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace, true); // Repeat the same test but with status level 'None', no events should be recorded corpus.Ctx.ReportAtLevel = CdmStatusLevel.None; await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved"); TestNoLogsState(corpus); // Test checking file status on a data partition // We're at log level 'Progress', so we get the EnterScope/LeaveScope messages too corpus.Ctx.ReportAtLevel = CdmStatusLevel.Progress; var part = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "part"); await part.FileStatusCheckAsync(); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath, true); // Test checking file status on a data partition pattern var refDoc = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "RefEntDoc"); var partPattern = corpus.MakeObject <CdmDataPartitionPatternDefinition>(CdmObjectType.DataPartitionPatternDef, "partPattern"); partPattern.InDocument = refDoc; await partPattern.FileStatusCheckAsync(); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullCorpusPath, true); // Test calculating relationships - no errors/warnings await corpus.CalculateEntityGraphAsync(manifest); TestBasicLogsState(corpus); // Test populating relationships in manifest - no errors/warnings await manifest.PopulateManifestRelationshipsAsync(); TestBasicLogsState(corpus); // Test filtering code logic corpus.Ctx.SuppressedLogCodes.Add(CdmLogCode.ErrPathNullObjectPath); var part2 = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "part2"); await part2.FileStatusCheckAsync(); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath, false); }
public async Task TestEntityAttribute() { string testName = "TestEntityAttribute"; string entityName = "ThreeMusketeers"; CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName); foreach (List <string> resOpt in resOptsCombinations) { await ProjectionTestUtils.LoadEntityForResolutionOptionAndSave(corpus, testName, testsSubpath, entityName, resOpt); } CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}"); CdmEntityDefinition nonStructuredResolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> { }); // Original set of attributes: ["name", "age", "address"] // in non-structured form // addArtifactAttribute : { "key" , "insertAtTop": true } // Expand 1...3; // renameAttributes = { {a}_{o}_key, apply to "key" } // renameAttributes = { {a}_{m}_{o}_value, apply to "name", "age", "address" } // alterTraits = { indicates.expansionInfo.mapKey(expansionName: "{a}", ordinal: "{o}") , apply to "key" , "argumentsContainWildcards" : true } // alterTraits = { has.expansionInfo.mapValue(expansionName: "{a}", ordinal: "{o}", memberAttribute: "{mo}") , apply to "name", "age", "address" , "argumentsContainWildcards" : true } // addArtifactAttribute : "personCount" // alterTraits = { indicates.expansionInfo.count(expansionName: "{a}") , apply to "personCount" , "argumentsContainWildcards" : true } Assert.AreEqual(13, nonStructuredResolvedEntity.Attributes.Count); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[0] as CdmTypeAttributeDefinition, "key_1_key", 1, "ThreePeople", isKey: true); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[1] as CdmTypeAttributeDefinition, "ThreePeople_name_1_value", 1, "ThreePeople", "name"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[2] as CdmTypeAttributeDefinition, "ThreePeople_age_1_value", 1, "ThreePeople", "age"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[3] as CdmTypeAttributeDefinition, "ThreePeople_address_1_value", 1, "ThreePeople", "address"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[4] as CdmTypeAttributeDefinition, "key_2_key", 2, "ThreePeople", isKey: true); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[5] as CdmTypeAttributeDefinition, "ThreePeople_name_2_value", 2, "ThreePeople", "name"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[6] as CdmTypeAttributeDefinition, "ThreePeople_age_2_value", 2, "ThreePeople", "age"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[7] as CdmTypeAttributeDefinition, "ThreePeople_address_2_value", 2, "ThreePeople", "address"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[8] as CdmTypeAttributeDefinition, "key_3_key", 3, "ThreePeople", isKey: true); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[9] as CdmTypeAttributeDefinition, "ThreePeople_name_3_value", 3, "ThreePeople", "name"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[10] as CdmTypeAttributeDefinition, "ThreePeople_age_3_value", 3, "ThreePeople", "age"); ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[11] as CdmTypeAttributeDefinition, "ThreePeople_address_3_value", 3, "ThreePeople", "address"); Assert.AreEqual("personCount", (nonStructuredResolvedEntity.Attributes[12] as CdmTypeAttributeDefinition).Name); Assert.IsNotNull(nonStructuredResolvedEntity.Attributes[12].AppliedTraits.Item("indicates.expansionInfo.count")); Assert.AreEqual("ThreePeople", (nonStructuredResolvedEntity.Attributes[12].AppliedTraits.Item("indicates.expansionInfo.count") as CdmTraitReference).Arguments[0].Value); // Original set of attributes: ["name", "age", "address"] // in structured form // addAttributeGroup: favorite people // alterTraits = { is.dataFormat.mapValue } // addArtifactAttribute : { "favorite People Key" (with trait "is.dataFormat.mapKey") , "insertAtTop": true } // addAttributeGroup: favorite People Group // alterTraits = { is.dataFormat.map } CdmEntityDefinition structuredResolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> { "structured" }); Assert.AreEqual(1, structuredResolvedEntity.Attributes.Count); CdmAttributeGroupDefinition attGroupDefinition = ProjectionTestUtils.ValidateAttributeGroup(structuredResolvedEntity.Attributes, "favorite People Group"); Assert.IsNotNull(attGroupDefinition.ExhibitsTraits.Item("is.dataFormat.map")); Assert.AreEqual("favorite People Key", (attGroupDefinition.Members[0] as CdmTypeAttributeDefinition).Name); Assert.IsNotNull(attGroupDefinition.Members[0].AppliedTraits.Item("is.dataFormat.mapKey")); Assert.AreEqual(CdmObjectType.AttributeGroupRef, attGroupDefinition.Members[1].ObjectType); CdmAttributeGroupReference innerAttGroupRef = attGroupDefinition.Members[1] as CdmAttributeGroupReference; Assert.IsNotNull(innerAttGroupRef.ExplicitReference); CdmAttributeGroupDefinition innerAttGroupDefinition = innerAttGroupRef.ExplicitReference as CdmAttributeGroupDefinition; Assert.AreEqual("favorite people", innerAttGroupDefinition.AttributeGroupName); Assert.IsNotNull(innerAttGroupDefinition.ExhibitsTraits.Item("is.dataFormat.mapValue")); }
static async Task Main() { // Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths var cdmCorpus = new CdmCorpusDefinition(); Console.WriteLine("configure storage adapters"); // Configure storage adapters to point at the target local manifest location and at the fake public standards var pathFromExeToExampleRoot = "../../../../../../"; cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "6-create-net-new-entities")); cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local // Fake cdm, normaly use the github adapter // Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); // Example how to mount to the ADLS. // cdmCorpus.Storage.Mount("adls", // new ADLSAdapter( // "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. // "/<FILESYSTEM-NAME>", // Root. // "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. // "<CLIENT-ID>", // Client ID. // "<CLIENT-SECRET>" // Client secret. // )); // This sample is going to simulate the steps a tool would follow in order to create a new manifest document // in some user storage folder with two types of entities - new net entity and entity extended from some public standards, // Note: If we want to create a relationship between a new custom entity to an existing entity which is loaded from some public standards, // we need to create an entity extending from the existing enbtity and add a relationship to the attribute of the extended entity, // since we can't modify attributes from an 'abstract' schemaDefintion in the public standards // This sample also creates a relationship between a net new entity to an existing entity, and a relationship between two net new entities // // the steps are: // 1. create a temporary 'manifest' object at the root of the corpus // 2. create two net new entities without extending any exsiting entity, create a relationship from one to the other, and add them to the manifest // 3. create one entity which extends from the public standards, create a relationship from it to an net new entity, and add the entity to the manifest // 4. make a 'resolved' version of each entity doc in our local folder. (please check out the second example 2-create-manifest for more details) // CreateResolvedManifestAsync on our starting manifest. this will resolve everything and find all of the relationships between entities for us // 5. save the new document(s) Console.WriteLine("Make placeholder manifest"); // Make the temp manifest and add it to the root of the local documents in the corpus CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract"); // Add the temp manifest to the root of the local documents in the corpus var localRoot = cdmCorpus.Storage.FetchRootFolder("local"); localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json"); // Create two entities from scratch, and add some attributes, traits, properties, and relationships in between Console.WriteLine("Create net new entities"); // Create the simplest entity - CustomPerson // Create the entity definition instance var personEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false); // Add type attributes to the entity instance var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId"); personEntity.Attributes.Add(personAttributeId); var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "string"); personEntity.Attributes.Add(personAttributeName); // Add properties to the entity instance personEntity.DisplayName = CustomPersonEntityName; personEntity.Version = "0.0.1"; personEntity.Description = "This is a custom entity created for the sample."; // Create the document which contains the entity var personEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely personEntityDoc.Imports.Add(FoundationJsonPath); personEntityDoc.Definitions.Add(personEntity); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name); // Add the entity to the manifest manifestAbstract.Entities.Add(personEntity); // Create an entity - CustomAccount which has a relationship with the entity CustomPerson // Create the entity definition instance var accountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false); // Add type attributes to the entity instance var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId"); accountEntity.Attributes.Add(accountAttributeId); var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "string"); accountEntity.Attributes.Add(accountAttributeName); // Add properties to the entity instance accountEntity.DisplayName = CustomAccountEntityName; accountEntity.Version = "0.0.1"; accountEntity.Description = "This is a custom entity created for the sample."; // In this sample, every account has one person who owns the account // the relationship is actually an entity attribute var attributeExplanation = "The owner of the account, which is a person."; // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show // more details of how to use resolution guidance to customize your data var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation); accountEntity.Attributes.Add(accountOwnerAttribute); // Create the document which contains the entity var accountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely accountEntityDoc.Imports.Add(FoundationJsonPath); // the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits, // the import to the entity reference CustomPerson's doc is required accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json"); accountEntityDoc.Definitions.Add(accountEntity); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name); // Add the entity to the manifest manifestAbstract.Entities.Add(accountEntity); // Create an entity which extends "Account" from the standard, it contains everything that "Account" has var extendedStandardAccountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false); // This function with 'true' will make a simple reference to the base extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "Account", true); var attrExplanation = "This is a simple custom account for this sample."; // Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show // more details of how to use resolution guidance to customize your data var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation); extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute); var extendedStandardAccountntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false); // Add an import to the foundations doc so the traits about partitons will resolve nicely extendedStandardAccountntityDoc.Imports.Add(FoundationJsonPath); // The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required // it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required extendedStandardAccountntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json"); extendedStandardAccountntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json"); // Add the document to the root of the local documents in the corpus localRoot.Documents.Add(extendedStandardAccountntityDoc, extendedStandardAccountntityDoc.Name); extendedStandardAccountntityDoc.Definitions.Add(extendedStandardAccountEntity); // Add the entity to the manifest manifestAbstract.Entities.Add(extendedStandardAccountEntity); // Create the resolved version of everything in the root folder too Console.WriteLine("Resolve the placeholder"); var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null); Console.WriteLine("Save the docs"); foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities) { // Get the entity being pointed at var localEDef = eDef; var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved); // Make a fake partition, just to demo that var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description"); localEDef.DataPartitions.Add(part); part.Explanation = "not real data, just for demo"; // We have existing partition files for the custom entities, so we need to make the partition point to the file location part.Location = $"local:/{entDef.EntityName}/partition-data.csv"; // Add trait to partition for csv params var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false); csvTrait.Arguments.Add("columnHeaders", "true"); csvTrait.Arguments.Add("delimiter", ","); } // We can save the docs as manifest.cdm.json format or model.json // Save as manifest.cdm.json await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true); // Save as a model.json // await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.model.json", true); }
public async Task TestProjectionUsingObjectModel() { CdmCorpusDefinition corpus = ProjectionTestUtils.GetLocalCorpus(testsSubpath, "TestProjectionUsingObjectModel"); corpus.Storage.Mount("local", new LocalAdapter(TestHelper.GetActualOutputFolderPath(testsSubpath, "TestProjectionUsingObjectModel"))); CdmFolderDefinition localRoot = corpus.Storage.FetchRootFolder("local"); CdmManifestDefinition manifestDefault = CreateDefaultManifest(corpus, localRoot); CdmEntityDefinition entityTestSource = CreateEntityTestSource(corpus, manifestDefault, localRoot); CdmEntityDefinition entityTestEntityProjection = CreateEntityTestEntityProjection(corpus, manifestDefault, localRoot); CdmEntityDefinition entityTestEntityNestedProjection = CreateEntityTestEntityNestedProjection(corpus, manifestDefault, localRoot); CdmEntityDefinition entityTestEntityAttributeProjection = CreateEntityTestEntityAttributeProjection(corpus, manifestDefault, localRoot); CdmEntityDefinition entityTestOperationCollection = CreateEntityTestOperationCollection(corpus, manifestDefault, localRoot); // Save manifest and entities await manifestDefault.SaveAsAsync($"{manifestDefault.ManifestName}.manifest.cdm.json", saveReferenced : true); string expected = "TestSource"; CdmObjectType expectedType = CdmObjectType.ProjectionDef; string actual = null; CdmObjectType actualType = CdmObjectType.Error; // Try to read back the newly persisted manifest and projection based entities CdmManifestDefinition manifestReadBack = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{manifestDefault.ManifestName}.manifest.cdm.json"); Assert.AreEqual(5, manifestReadBack.Entities.Count); Assert.AreEqual(entityTestSource.EntityName, manifestReadBack.Entities[0].EntityName); Assert.AreEqual(entityTestEntityProjection.EntityName, manifestReadBack.Entities[1].EntityName); Assert.AreEqual(entityTestEntityNestedProjection.EntityName, manifestReadBack.Entities[2].EntityName); Assert.AreEqual(entityTestEntityAttributeProjection.EntityName, manifestReadBack.Entities[3].EntityName); // Read back the newly persisted manifest and projection based entity TestEntityProjection and validate CdmEntityDefinition entityTestEntityProjectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestEntityProjection.EntityName}.cdm.json/{entityTestEntityProjection.EntityName}", manifestReadBack); Assert.IsNotNull(entityTestEntityProjectionReadBack); actual = ((CdmEntityReference)((CdmProjection)entityTestEntityProjectionReadBack.ExtendsEntity.ExplicitReference).Source).NamedReference; actualType = ((CdmProjection)entityTestEntityProjectionReadBack.ExtendsEntity.ExplicitReference).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(expectedType, actualType); // Read back the newly persisted manifest and projection based entity TestEntityNestedProjection and validate CdmEntityDefinition entityTestEntityNestedProjectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestEntityNestedProjection.EntityName}.cdm.json/{entityTestEntityNestedProjection.EntityName}", manifestReadBack); Assert.IsNotNull(entityTestEntityNestedProjectionReadBack); actual = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entityTestEntityNestedProjectionReadBack.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).Source.NamedReference; actualType = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entityTestEntityNestedProjectionReadBack.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(expectedType, actualType); // Read back the newly persisted manifest and projection based entity TestEntityAttributeProjection and validate CdmEntityDefinition entityTestEntityAttributeProjectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestEntityAttributeProjection.EntityName}.cdm.json/{entityTestEntityAttributeProjection.EntityName}", manifestReadBack); Assert.IsNotNull(entityTestEntityAttributeProjectionReadBack); actual = ((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entityTestEntityAttributeProjectionReadBack.Attributes[0]).Entity).ExplicitReference).Source).NamedReference; actualType = ((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entityTestEntityAttributeProjectionReadBack.Attributes[0]).Entity).ExplicitReference).ObjectType; Assert.AreEqual(expected, actual); Assert.AreEqual(expectedType, actualType); // Read back operations collections and validate CdmEntityDefinition entityTestOperationCollectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestOperationCollection.EntityName}.cdm.json/{entityTestOperationCollection.EntityName}", manifestReadBack); Assert.IsNotNull(entityTestOperationCollectionReadBack); int actualOperationCount = ((CdmProjection)entityTestOperationCollectionReadBack.ExtendsEntity.ExplicitReference).Operations.Count; Assert.AreEqual(9, actualOperationCount); CdmOperationCollection operations = ((CdmProjection)entityTestOperationCollectionReadBack.ExtendsEntity.ExplicitReference).Operations; Assert.AreEqual(CdmOperationType.AddCountAttribute, operations[0].Type); Assert.AreEqual(CdmOperationType.AddSupportingAttribute, operations[1].Type); Assert.AreEqual(CdmOperationType.AddTypeAttribute, operations[2].Type); Assert.AreEqual(CdmOperationType.ExcludeAttributes, operations[3].Type); Assert.AreEqual(CdmOperationType.ArrayExpansion, operations[4].Type); Assert.AreEqual(CdmOperationType.CombineAttributes, operations[5].Type); Assert.AreEqual(CdmOperationType.RenameAttributes, operations[6].Type); Assert.AreEqual(CdmOperationType.ReplaceAsForeignKey, operations[7].Type); Assert.AreEqual(CdmOperationType.IncludeAttributes, operations[8].Type); }