/// <summary> /// Common test code for these test cases /// </summary> /// <param name="testName"></param> /// <param name="entityName"></param> private void TestRun(string testName, string entityName) { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName); string inputFolder = TestHelper.GetInputFolderPath(testsSubpath, testName); string expectedOutputFolder = TestHelper.GetExpectedOutputFolderPath(testsSubpath, testName); string actualOutputFolder = TestHelper.GetActualOutputFolderPath(testsSubpath, testName); if (!Directory.Exists(actualOutputFolder)) { Directory.CreateDirectory(actualOutputFolder); } CdmManifestDefinition manifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json").GetAwaiter().GetResult(); Assert.IsNotNull(manifest); CdmEntityDefinition entity = corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}", manifest).GetAwaiter().GetResult(); Assert.IsNotNull(entity); CdmEntityDefinition resolvedEntity = TestUtils.GetResolvedEntity(corpus, entity, new List <string> { "referenceOnly" }).GetAwaiter().GetResult(); string actualAttrCtx = GetAttributeContextString(resolvedEntity, entityName, actualOutputFolder); string expectedStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, $"AttrCtx_{entityName}.txt")); string expectedAttrCtx = File.ReadAllText(expectedStringFilePath); Assert.AreEqual(expectedAttrCtx, actualAttrCtx); corpus.CalculateEntityGraphAsync(manifest).GetAwaiter().GetResult(); manifest.PopulateManifestRelationshipsAsync().GetAwaiter().GetResult(); string actualRelationshipsString = ListRelationships(corpus, entity, actualOutputFolder, entityName); string expectedRelationshipsStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, $"REL_{entityName}.txt")); string expectedRelationshipsString = File.ReadAllText(expectedRelationshipsStringFilePath); Assert.AreEqual(expectedRelationshipsString, actualRelationshipsString); CdmFolderDefinition outputFolder = corpus.Storage.FetchRootFolder("output"); outputFolder.Documents.Add(manifest); string manifestFileName = $"saved.manifest.cdm.json"; manifest.SaveAsAsync(manifestFileName, saveReferenced: true).GetAwaiter().GetResult(); string actualManifestPath = Path.Combine(actualOutputFolder, manifestFileName); if (!File.Exists(actualManifestPath)) { Assert.Fail("Unable to save manifest with relationship"); } else { CdmManifestDefinition savedManifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"output:/{manifestFileName}").GetAwaiter().GetResult(); string actualSavedManifestRel = GetRelationshipStrings(savedManifest.Relationships); string expectedSavedManifestRel = File.ReadAllText(Path.Combine(expectedOutputFolder, $"MANIFEST_REL_{entityName}.txt")); Assert.AreEqual(expectedSavedManifestRel, actualSavedManifestRel); } }
public async Task TestSelfReference() { CdmCorpusDefinition cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, "TestSelfReference"); CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/SelfReference.manifest.cdm.json"); await cdmCorpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); Assert.AreEqual(1, manifest.Relationships.Count); CdmE2ERelationship rel = manifest.Relationships[0]; Assert.AreEqual("CustTable.cdm.json/CustTable", rel.FromEntity); Assert.AreEqual("CustTable.cdm.json/CustTable", rel.ToEntity); Assert.AreEqual("FactoringAccountRelationship", rel.FromEntityAttribute); Assert.AreEqual("PaymTermId", rel.ToEntityAttribute); }
static async Task ExploreManifest(CdmCorpusDefinition cdmCorpus, string manifestPath) { Console.WriteLine($"\nLoading manifest {manifestPath} ..."); CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestPath); if (manifest == null) { Console.WriteLine($"Unable to load manifest {manifestPath}. Please inspect error log for additional details."); return; } // ------------------------------------------------------------------------------------------------------------ // List all the entities found in the manifest and allow the user to choose which entity to explore. while (true) { int index = 1; if (manifest.Entities.Count > 0) { Console.WriteLine("List of all entities:"); foreach (var entDec in manifest.Entities) { // Print entity declarations. // Assume there are only local entities in this manifest for simplicity. Console.Write(" " + index.ToString().PadRight(3)); Console.Write(" " + entDec.EntityName.PadRight(35)); Console.WriteLine(" " + entDec.EntityPath); index++; } } if (manifest.SubManifests.Count > 0) { Console.WriteLine("List of all sub-manifests:"); foreach (var manifestDecl in manifest.SubManifests) { // Print sub-manifest declarations. Console.Write(" " + index.ToString().PadRight(3)); Console.Write(" " + manifestDecl.ManifestName.PadRight(35)); Console.WriteLine(" " + manifestDecl.Definition); index++; } } if (index == 1) { Console.Write("No Entities or Sub-manifest found. Press [enter] to exit."); Console.ReadLine(); break; } Console.Write("Enter a number to show details for that Entity or Sub-manifest (press [enter] to exit): "); // Get the user's choice. string input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { break; } // Make sure the user's input is a number. int num = 0; if (!int.TryParse(input, out num)) { Console.WriteLine("\nEnter a number."); Console.WriteLine(); continue; } // User can select an entry that is a sub-manifest if (num > manifest.Entities.Count) { int subNum = num - manifest.Entities.Count - 1; // Re-enter this method supplying the absolute path of the submanifest definition (relative to the current manifest) await ExploreManifest(cdmCorpus, cdmCorpus.Storage.CreateAbsoluteCorpusPath(manifest.SubManifests[subNum].Definition, manifest)); continue; } index = 1; foreach (var entityDec in manifest.Entities) { if (index == num) { Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ..."); // From the path to the entity, get the actual schema description. // Take the local relative path in this doc and make sure it works. var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc while (true) { // List all the metadata properties of this entity that can be explored. Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:"); Console.WriteLine(" 1: Attributes"); Console.WriteLine(" 2: Traits"); Console.WriteLine(" 3: Properties"); Console.WriteLine(" 4: Data partition locations"); Console.WriteLine(" 5: Relationships"); Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): "); // Get the user's choice. input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { Console.WriteLine(); break; } // Make sure the user's input is a number. int choice = 0; if (int.TryParse(input, out choice)) { switch (choice) { // List the entity's attributes. case 1: ListAttributes(entSelected); break; // List the entity's traits. case 2: ListTraits(entSelected); break; // List the entity's properties. case 3: ListProperties(entSelected, entityDec); break; // List the entity's data partition locations. case 4: ListDataPartitionLocations(cdmCorpus, entityDec); break; // List the entity's relationships. case 5: if (manifest.Relationships != null && manifest.Relationships.Count > 0) { // The manifest file contains pre-calculated entity relationships, so we can read them directly. ListRelationshipsFromManifest(manifest, entSelected); } else { // The manifest file doesn't contain relationships, so we have to compute the relationships first. await cdmCorpus.CalculateEntityGraphAsync(manifest); ListRelationships(cdmCorpus, entSelected); } break; default: Console.WriteLine("\nEnter a number between 1-5."); break; } } else { Console.WriteLine("\nEnter a number."); } } } index++; } } }
public async Task TestWithoutNesting() { CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList"); corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId); // Test fetching an object from invalid namespace results in at least one error message in the recorder _ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar"); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNamespaceNotRegistered); // Test fetching a good object, this should leave event recorder empty _ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json"); TestNoLogsState(corpus); // Test saving a manifest to invalid namespace results in at least one error message in the recorder var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy"); await manifest.SaveAsAsync("foo:/bar", true); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrValdnMissingDoc); // Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder await manifest.CreateResolvedManifestAsync("new dummy", null); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrResolveManifestFailed); // Test resolving an entity without WRT doc, this should yield at least one error message in the recorder var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2"); await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved"); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrDocWrtDocNotfound); // Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder await manifest.FileStatusCheckAsync(); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace); // Repeat the same test but with status level 'None', no events should be recorded corpus.Ctx.ReportAtLevel = CdmStatusLevel.None; await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved"); TestNoLogsState(corpus); // Test checking file status on a data partition // We're at log level 'Progress', so we get the EnterScope/LeaveScope messages too corpus.Ctx.ReportAtLevel = CdmStatusLevel.Progress; var part = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "part"); await part.FileStatusCheckAsync(); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath); // Test checking file status on a data partition pattern var refDoc = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "RefEntDoc"); var partPattern = corpus.MakeObject <CdmDataPartitionPatternDefinition>(CdmObjectType.DataPartitionPatternDef, "partPattern"); partPattern.InDocument = refDoc; await partPattern.FileStatusCheckAsync(); TestBasicLogsState(corpus); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace); TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrDocAdapterNotFound); // Test calculating relationships - no errors/warnings await corpus.CalculateEntityGraphAsync(manifest); TestBasicLogsState(corpus); // Test populating relationships in manifest - no errors/warnings await manifest.PopulateManifestRelationshipsAsync(); TestBasicLogsState(corpus); }
public async Task TestUpdateRelationships() { var expectedRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestUpdateRelationships", "expectedRels.json")).ToObject <List <E2ERelationship> >(); string tempFromFilePath = "fromEntTemp.cdm.json"; string tempFromEntityPath = "local:/fromEntTemp.cdm.json/fromEnt"; string tempToEntityPath = "local:/toEnt.cdm.json/toEnt"; // Initialize corpus and entity files CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestUpdateRelationships"); CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/main.manifest.cdm.json"); CdmManifestDefinition manifestNoToEnt = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/mainNoToEnt.manifest.cdm.json"); CdmEntityDefinition fromEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/fromEnt.cdm.json/fromEnt"); await fromEnt.InDocument.SaveAsAsync(tempFromFilePath, options : new CopyOptions() { IsTopLevelDocument = false }); async Task reloadFromEntity() { await fromEnt.InDocument.SaveAsAsync(tempFromFilePath, options : new CopyOptions() { IsTopLevelDocument = false }); // fetch again to reset the cache await corpus.FetchObjectAsync <CdmEntityDefinition>(tempFromEntityPath, null, false, true); } try { // 1. test when entity attribute is removed await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); // check that the relationship has been created correctly VerifyRelationships(manifest, expectedRels); // now remove the entity attribute, which removes the relationship CdmAttributeItem removedAttribute = fromEnt.Attributes[0]; fromEnt.Attributes.RemoveAt(0); await reloadFromEntity(); await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); // check that the relationship has been removed VerifyRelationships(manifest, new List <E2ERelationship>()); // 2. test when the to entity is removed // restore the entity to the original state fromEnt.Attributes.Add(removedAttribute); await reloadFromEntity(); await corpus.CalculateEntityGraphAsync(manifest); await manifest.PopulateManifestRelationshipsAsync(); // check that the relationship has been created correctly VerifyRelationships(manifest, expectedRels); // remove the to entity fromEnt.Attributes.RemoveAt(0); await reloadFromEntity(); // fetch again to reset the cache await corpus.FetchObjectAsync <CdmEntityDefinition>(tempToEntityPath, null, false, true); await corpus.CalculateEntityGraphAsync(manifestNoToEnt); await manifestNoToEnt.PopulateManifestRelationshipsAsync(); // check that the relationship has been removed VerifyRelationships(manifestNoToEnt, new List <E2ERelationship>()); } finally { // clean up created files created string fromPath = corpus.Storage.CorpusPathToAdapterPath($"local:/{tempFromFilePath}"); File.Delete(fromPath); } }
static async Task Main(string[] args) { // ------------------------------------------------------------------------------------------------------------ // Instantiate a corpus. The corpus is the collection of all documents and folders created or discovered // while navigating objects and paths. var cdmCorpus = new CdmCorpusDefinition(); // ------------------------------------------------------------------------------------------------------------ // Configure storage adapters and mount them to the corpus. // We want our storage adapters to point at the local manifest location and at the example public standards. string pathFromExeToExampleRoot = "../../../../../../"; // Storage adapter pointing to the target local manifest location. cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "1-read-manifest")); // 'local' is our default namespace. // Any paths that start navigating without a device tag (ex. 'cdm') will just default to the 'local' namepace. cdmCorpus.Storage.DefaultNamespace = "local"; // Storage adapter pointing to the example public standards. // This is a fake 'cdm'; normally the Github adapter would be used to point at the real public standards. // Mount it as the 'cdm' device, not the default, so that we must use "cdm:<folder-path>" to get there. cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards")); // Example how to mount to the ADLS: // cdmCorpus.Storage.Mount("adls", // new ADLSAdapter( // "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname. // "/<FILESYSTEM-NAME>", // Root. // "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID. // "<CLIENT-ID>", // Client ID. // "<CLIENT-SECRET>" // Client secret. // ) // ); // ------------------------------------------------------------------------------------------------------------ // Open the default manifest file at the root. var manifestFile = "default.manifest.cdm.json"; // This method turns relative corpus paths into absolute paths in case we are in some sub-folders // and don't know it. var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestFile); // ------------------------------------------------------------------------------------------------------------ // List all the entities found in the manifest and allow the user to choose which entity to explore. while (true) { Console.WriteLine($"List of all entities found in {manifestFile}:"); // Loop over any entity declarations. int iEnt = 1; foreach (var entDec in manifest.Entities) { // Print it out. // Assume there are only local entities in this manifest for simplicity. Console.Write(" " + iEnt.ToString().PadRight(3)); Console.Write(" " + entDec.EntityName.PadRight(35)); Console.WriteLine(" " + entDec.EntityPath); iEnt++; } Console.Write("Enter a number to show details for that Entity (press [enter] to exit): "); // Get the user's choice. string input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { break; } // Make sure the user's input is a number. int num = 0; if (int.TryParse(input, out num)) { iEnt = 1; foreach (var entityDec in manifest.Entities) { if (iEnt == num) { Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ..."); // From the path to the entity, get the actual schema description. // Take the local relative path in this doc and make sure it works. var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc while (true) { // List all the metadata properties of this entity that can be explored. Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:"); Console.WriteLine(" 1: Attributes"); Console.WriteLine(" 2: Traits"); Console.WriteLine(" 3: Properties"); Console.WriteLine(" 4: Data partition locations"); Console.WriteLine(" 5: Relationships"); Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): "); // Get the user's choice. input = Console.ReadLine(); if (string.IsNullOrEmpty(input)) { Console.WriteLine(); break; } // Make sure the user's input is a number. int choice = 0; if (int.TryParse(input, out choice)) { switch (choice) { // List the entity's attributes. case 1: ListAttributes(entSelected); break; // List the entity's traits. case 2: ListTraits(entSelected); break; // List the entity's properties. case 3: ListProperties(entSelected, entityDec); break; // List the entity's data partition locations. case 4: ListDataPartitionLocations(entityDec); break; // List the entity's relationships. case 5: if (manifest.Relationships != null && manifest.Relationships.Count > 0) { // The manifest file contains pre-calculated entity relationships, so we can read them directly. ListRelationshipsFromManifest(manifest, entSelected); } else { // The manifest file doesn't contain relationships, so we have to compute the relationships first. await cdmCorpus.CalculateEntityGraphAsync(manifest); ListRelationships(cdmCorpus, entSelected); } break; default: Console.WriteLine("\nEnter a number between 1-5."); break; } } else { Console.WriteLine("\nEnter a number."); } } } iEnt++; } } else { Console.WriteLine("\nEnter a number."); Console.WriteLine(); } } }
public async Task TestCalculateRelationshipsAndPopulateManifests() { var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests"); var expectedAllManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedAllManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedAllSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedAllSubManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedExclusiveManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedExclusiveManifestRels.json")).ToObject <List <E2ERelationship> >(); var expectedExclusiveSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedExclusiveSubManifestRels.json")).ToObject <List <E2ERelationship> >(); CdmCorpusDefinition corpus = new CdmCorpusDefinition(); corpus.SetEventCallback(new Utilities.EventCallback { Invoke = CommonDataModelLoader.ConsoleStatusReport }, CdmStatusLevel.Warning); corpus.Storage.Mount("local", new LocalAdapter(testInputPath)); corpus.Storage.DefaultNamespace = "local"; CdmManifestDefinition rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json"); string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(rootManifest.SubManifests[0].Definition); CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition; await corpus.CalculateEntityGraphAsync(rootManifest); await rootManifest.PopulateManifestRelationshipsAsync(); Assert.AreEqual(rootManifest.Relationships.Count, 5); Assert.AreEqual(subManifest.Relationships.Count, 7); // check that each relationship has been created correctly foreach (E2ERelationship expectedRel in expectedAllManifestRels) { List <CdmE2ERelationship> found = rootManifest.Relationships.Where(x => x.FromEntity == expectedRel.FromEntity && x.FromEntityAttribute == expectedRel.FromEntityAttribute && x.ToEntity == expectedRel.ToEntity && x.ToEntityAttribute == expectedRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } foreach (E2ERelationship expectedSubRel in expectedAllSubManifestRels) { List <CdmE2ERelationship> found = subManifest.Relationships.Where(x => x.FromEntity == expectedSubRel.FromEntity && x.FromEntityAttribute == expectedSubRel.FromEntityAttribute && x.ToEntity == expectedSubRel.ToEntity && x.ToEntityAttribute == expectedSubRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } // make sure only relationships where to and from entities are in the manifest are found with the "exclusive" option is passed in await rootManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.Exclusive); Assert.AreEqual(rootManifest.Relationships.Count, 3); Assert.AreEqual(subManifest.Relationships.Count, 3); // check that each relationship has been created correctly foreach (E2ERelationship expectedRel in expectedExclusiveManifestRels) { List <CdmE2ERelationship> found = rootManifest.Relationships.Where(x => x.FromEntity == expectedRel.FromEntity && x.FromEntityAttribute == expectedRel.FromEntityAttribute && x.ToEntity == expectedRel.ToEntity && x.ToEntityAttribute == expectedRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } foreach (E2ERelationship expectedSubRel in expectedExclusiveSubManifestRels) { List <CdmE2ERelationship> found = subManifest.Relationships.Where(x => x.FromEntity == expectedSubRel.FromEntity && x.FromEntityAttribute == expectedSubRel.FromEntityAttribute && x.ToEntity == expectedSubRel.ToEntity && x.ToEntityAttribute == expectedSubRel.ToEntityAttribute ).ToList(); Assert.AreEqual(found.Count, 1); } // make sure no relationships are added when "none" relationship option is passed in await rootManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.None); Assert.AreEqual(rootManifest.Relationships.Count, 0); Assert.AreEqual(subManifest.Relationships.Count, 0); }