static async Task ExploreManifest(CdmCorpusDefinition cdmCorpus, string manifestPath)
{
Console.WriteLine($"\nLoading manifest {manifestPath} ...");
CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestPath);
if (manifest == null)
{
Console.WriteLine($"Unable to load manifest {manifestPath}. Please inspect error log for additional details.");
return;
}
// ------------------------------------------------------------------------------------------------------------
// List all the entities found in the manifest and allow the user to choose which entity to explore.
while (true)
{
int index = 1;
if (manifest.Entities.Count > 0)
{
Console.WriteLine("List of all entities:");
foreach (var entDec in manifest.Entities)
{
// Print entity declarations.
// Assume there are only local entities in this manifest for simplicity.
Console.Write(" " + index.ToString().PadRight(3));
Console.Write(" " + entDec.EntityName.PadRight(35));
Console.WriteLine(" " + entDec.EntityPath);
index++;
}
}
if (manifest.SubManifests.Count > 0)
{
Console.WriteLine("List of all sub-manifests:");
foreach (var manifestDecl in manifest.SubManifests)
{
// Print sub-manifest declarations.
Console.Write(" " + index.ToString().PadRight(3));
Console.Write(" " + manifestDecl.ManifestName.PadRight(35));
Console.WriteLine(" " + manifestDecl.Definition);
index++;
}
}
if (index == 1)
{
Console.Write("No Entities or Sub-manifest found. Press [enter] to exit.");
Console.ReadLine();
break;
}
Console.Write("Enter a number to show details for that Entity or Sub-manifest (press [enter] to exit): ");
// Get the user's choice.
string input = Console.ReadLine();
if (string.IsNullOrEmpty(input))
{
break;
}
// Make sure the user's input is a number.
int num = 0;
if (!int.TryParse(input, out num))
{
Console.WriteLine("\nEnter a number.");
Console.WriteLine();
continue;
}
// User can select an entry that is a sub-manifest
if (num > manifest.Entities.Count)
{
int subNum = num - manifest.Entities.Count - 1;
// Re-enter this method supplying the absolute path of the submanifest definition (relative to the current manifest)
await ExploreManifest(cdmCorpus, cdmCorpus.Storage.CreateAbsoluteCorpusPath(manifest.SubManifests[subNum].Definition, manifest));
continue;
}
index = 1;
foreach (var entityDec in manifest.Entities)
{
if (index == num)
{
Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ...");
// From the path to the entity, get the actual schema description.
// Take the local relative path in this doc and make sure it works.
var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc
while (true)
{
// List all the metadata properties of this entity that can be explored.
Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:");
Console.WriteLine(" 1: Attributes");
Console.WriteLine(" 2: Traits");
Console.WriteLine(" 3: Properties");
Console.WriteLine(" 4: Data partition locations");
Console.WriteLine(" 5: Relationships");
Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): ");
// Get the user's choice.
input = Console.ReadLine();
if (string.IsNullOrEmpty(input))
{
Console.WriteLine();
break;
}
// Make sure the user's input is a number.
int choice = 0;
if (int.TryParse(input, out choice))
{
switch (choice)
{
// List the entity's attributes.
case 1:
ListAttributes(entSelected);
break;
// List the entity's traits.
case 2:
ListTraits(entSelected);
break;
// List the entity's properties.
case 3:
ListProperties(entSelected, entityDec);
break;
// List the entity's data partition locations.
case 4:
ListDataPartitionLocations(cdmCorpus, entityDec);
break;
// List the entity's relationships.
case 5:
if (manifest.Relationships != null && manifest.Relationships.Count > 0)
{
// The manifest file contains pre-calculated entity relationships, so we can read them directly.
ListRelationshipsFromManifest(manifest, entSelected);
}
else
{
// The manifest file doesn't contain relationships, so we have to compute the relationships first.
await cdmCorpus.CalculateEntityGraphAsync(manifest);
ListRelationships(cdmCorpus, entSelected);
}
break;
default:
Console.WriteLine("\nEnter a number between 1-5.");
break;
}
}
else
{
Console.WriteLine("\nEnter a number.");
}
}
}
index++;
}
}
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var corpus = new CdmCorpusDefinition();
// set callback to receive error and warning logs.
corpus.SetEventCallback(new EventCallback
{
Invoke = (level, message) =>
{
Console.WriteLine(message);
}
}, CdmStatusLevel.Warning);
Console.WriteLine("Configure storage adapters.");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../";
corpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "8-logical-manipulation-using-projections/sample-data"));
corpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the CDM Standards adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
corpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
Console.WriteLine("Create logical entity definition.");
var logicalFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("local:/");
var logicalDoc = logicalFolder.Documents.Add("Person.cdm.json");
logicalDoc.Imports.Add("Address.cdm.json");
var entity = logicalDoc.Definitions.Add("Person");
// Add "name" data typed attribute.
var nameAttr = entity.Attributes.Add("name") as CdmTypeAttributeDefinition;
nameAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true);
// Add "age" data typed attribute.
var ageAttr = entity.Attributes.Add("age") as CdmTypeAttributeDefinition;
ageAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true);
// Add "address" entity typed attribute.
var entityAttr = new CdmEntityAttributeDefinition(corpus.Ctx, "address")
{
Entity = new CdmEntityReference(corpus.Ctx, "Address", true)
};
ApplyArrayExpansion(entityAttr, 1, 3, "{m}{A}{o}", "countAttribute");
ApplyDefaultBehavior(entityAttr, "addressFK", "address");
entity.Attributes.Add(entityAttr);
// Add "email" data typed attribute.
var emailAttr = entity.Attributes.Add("email") as CdmTypeAttributeDefinition;
emailAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true);
// Save the logical definition of Person.
await entity.InDocument.SaveAsAsync("Person.cdm.json");
Console.WriteLine("Get \"resolved\" folder where the resolved entities will be saved.");
var resolvedFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("local:/resolved/");
var resOpt = new ResolveOptions(entity);
// To get more information about directives and their meaning refer to
// https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
// We will start by resolving this entity with the "normalized" direcitve.
// This directive will be used on this and the next two examples so we can analize the resolved entity
// without the array expansion.
Console.WriteLine("Resolving logical entity with normalized directive.");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
var resNormalizedEntity = await entity.CreateResolvedEntityAsync($"normalized_{entity.EntityName}", resOpt, resolvedFolder);
await resNormalizedEntity.InDocument.SaveAsAsync($"{resNormalizedEntity.EntityName}.cdm.json");
// Another common scenario is to resolve an entity using the "referenceOnly" directive.
// This directives is used to replace the relationships with a foreign key.
Console.WriteLine("Resolving logical entity with referenceOnly directive.");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var resReferenceOnlyEntity = await entity.CreateResolvedEntityAsync($"referenceOnly_{entity.EntityName}", resOpt, resolvedFolder);
await resReferenceOnlyEntity.InDocument.SaveAsAsync($"{resReferenceOnlyEntity.EntityName}.cdm.json");
// When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
// a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
// entity attribute and we use the "structured" directive to resolve the entity attribute as an attribute group.
Console.WriteLine("Resolving logical entity with structured directive.");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
var resStructuredEntity = await entity.CreateResolvedEntityAsync($"structured_{entity.EntityName}", resOpt, resolvedFolder);
await resStructuredEntity.InDocument.SaveAsAsync($"{resStructuredEntity.EntityName}.cdm.json");
// Now let us remove the "normalized" directive so the array expansion operation can run.
Console.WriteLine("Resolving logical entity without directives (array expansion).");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
var resArrayEntity = await entity.CreateResolvedEntityAsync($"array_expansion_{entity.EntityName}", resOpt, resolvedFolder);
await resArrayEntity.InDocument.SaveAsAsync($"{resArrayEntity.EntityName}.cdm.json");
}
public async Task TestCalculateRelationshipsOnResolvedEntities()
{
var expectedResolvedManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedResolvedSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedSubManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedResolvedExcManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedExcManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedResolvedExcSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedExcSubManifestRels.json")).ToObject <List <E2ERelationship> >();
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities");
var rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json");
var resolvedManifest = await LoadAndResolveManifest(corpus, rootManifest, "-resolved");
string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(resolvedManifest.SubManifests[0].Definition);
CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition;
// using createResolvedManifest will only populate exclusive relationships
VerifyRelationships(resolvedManifest, expectedResolvedExcManifestRels);
VerifyRelationships(subManifest, expectedResolvedExcSubManifestRels);
// check that each relationship has been created correctly with the all flag
await resolvedManifest.PopulateManifestRelationshipsAsync();
await subManifest.PopulateManifestRelationshipsAsync();
VerifyRelationships(resolvedManifest, expectedResolvedManifestRels);
VerifyRelationships(subManifest, expectedResolvedSubManifestRels);
// it is not enough to check if the relationships are correct.
// We need to check if the incoming and outgoing relationships are
// correct as well. One being correct can cover up the other being wrong
// A
var aEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[0].EntityPath, resolvedManifest);
var aInRels = corpus.FetchIncomingRelationships(aEnt);
var aOutRels = corpus.FetchOutgoingRelationships(aEnt);
Assert.AreEqual(0, aInRels.Count);
Assert.AreEqual(1, aOutRels.Count);
Assert.AreEqual("local:/A-resolved.cdm.json/A", aOutRels[0].FromEntity);
Assert.AreEqual("local:/B-resolved.cdm.json/B", aOutRels[0].ToEntity);
// B
var bEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[1].EntityPath, resolvedManifest);
var bInRels = corpus.FetchIncomingRelationships(bEnt);
var bOutRels = corpus.FetchOutgoingRelationships(bEnt);
Assert.AreEqual(2, bInRels.Count);
Assert.AreEqual("local:/A-resolved.cdm.json/A", bInRels[0].FromEntity);
Assert.AreEqual("local:/B-resolved.cdm.json/B", bInRels[0].ToEntity);
Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", bInRels[1].FromEntity);
Assert.AreEqual("local:/B-resolved.cdm.json/B", bInRels[1].ToEntity);
Assert.AreEqual(0, bOutRels.Count);
// C
var cEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[0].EntityPath, subManifest);
var cInRels = corpus.FetchIncomingRelationships(cEnt);
var cOutRels = corpus.FetchOutgoingRelationships(cEnt);
Assert.AreEqual(0, cInRels.Count);
Assert.AreEqual(2, cOutRels.Count);
Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", cOutRels[0].FromEntity);
Assert.AreEqual("local:/B-resolved.cdm.json/B", cOutRels[0].ToEntity);
Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", cOutRels[1].FromEntity);
Assert.AreEqual("local:/sub/D-resolved.cdm.json/D", cOutRels[1].ToEntity);
// D
var dEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[1].EntityPath, subManifest);
var dInRels = corpus.FetchIncomingRelationships(dEnt);
var dOutRels = corpus.FetchOutgoingRelationships(dEnt);
Assert.AreEqual(1, dInRels.Count);
Assert.AreEqual("local:/sub/C-resolved.cdm.json/C", dInRels[0].FromEntity);
Assert.AreEqual("local:/sub/D-resolved.cdm.json/D", dInRels[0].ToEntity);
Assert.AreEqual(0, dOutRels.Count);
// E
var eEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[2].EntityPath, resolvedManifest);
var eInRels = corpus.FetchIncomingRelationships(eEnt);
var eOutRels = corpus.FetchOutgoingRelationships(eEnt);
Assert.AreEqual(1, eInRels.Count);
Assert.AreEqual(0, eOutRels.Count);
Assert.AreEqual("local:/sub/F-resolved.cdm.json/F", eInRels[0].FromEntity);
Assert.AreEqual("local:/E-resolved.cdm.json/E", eInRels[0].ToEntity);
// F
var fEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[2].EntityPath, subManifest);
var fInRels = corpus.FetchIncomingRelationships(fEnt);
var fOutRels = corpus.FetchOutgoingRelationships(fEnt);
Assert.AreEqual(0, fInRels.Count);
Assert.AreEqual(1, fOutRels.Count);
Assert.AreEqual("local:/sub/F-resolved.cdm.json/F", fOutRels[0].FromEntity);
Assert.AreEqual("local:/E-resolved.cdm.json/E", fOutRels[0].ToEntity);
}
public async Task TestRefreshesDataPartitionPatterns()
{
var inputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestRefreshDataPartitionPatterns");
var actualLastModTime = new FileInfo(Path.GetFullPath(inputPath)).LastWriteTimeUtc;
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(inputPath));
cdmCorpus.Storage.DefaultNamespace = "local";
var cdmManifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/patternManifest.manifest.cdm.json");
var partitionEntity = cdmManifest.Entities.AllItems[0];
Assert.AreEqual(partitionEntity.DataPartitions.Count, 1);
var timeBeforeLoad = DateTime.Now;
await cdmManifest.FileStatusCheckAsync();
// file status check should check patterns and add two more partitions that match the pattern
// should not re-add already existing partitions
// Mac and Windows behave differently when listing file content, so we don't want to be strict about partition file order
int totalExpectedPartitionsFound = 0;
foreach (CdmDataPartitionDefinition partition in partitionEntity.DataPartitions.AllItems)
{
switch (partition.Location)
{
case "partitions/existingPartition.csv":
totalExpectedPartitionsFound++;
break;
case "partitions/someSubFolder/someSubPartition.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.SpecializedSchema, "test special schema");
Assert.IsTrue(partition.LastFileStatusCheckTime > timeBeforeLoad);
// inherits the exhibited traits from pattern
Assert.AreEqual(partition.ExhibitsTraits.Count, 1);
Assert.AreEqual(partition.ExhibitsTraits.AllItems[0].NamedReference, "is");
Assert.AreEqual(partition.Arguments.Count, 1);
Assert.IsTrue(partition.Arguments.ContainsKey("testParam1"));
List <string> argArray = partition.Arguments["testParam1"];
Assert.AreEqual(argArray.Count, 1);
Assert.AreEqual(argArray[0], "/someSubFolder/someSub");
break;
case "partitions/newPartition.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 1);
break;
case "partitions/2018/folderCapture.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 1);
Assert.AreEqual(partition.Arguments.ContainsKey("year"), true);
Assert.AreEqual(partition.Arguments["year"][0], "2018");
break;
case "partitions/2018/8/15/folderCapture.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 3);
Assert.AreEqual(partition.Arguments.ContainsKey("year"), true);
Assert.AreEqual(partition.Arguments["year"][0], "2018");
Assert.AreEqual(partition.Arguments.ContainsKey("month"), true);
Assert.AreEqual(partition.Arguments["month"][0], "8");
Assert.AreEqual(partition.Arguments.ContainsKey("day"), true);
Assert.AreEqual(partition.Arguments["day"][0], "15");
break;
case "partitions/2018/8/15/folderCaptureRepeatedGroup.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 1);
Assert.AreEqual(partition.Arguments.ContainsKey("day"), true);
Assert.AreEqual(partition.Arguments["day"][0], "15");
break;
case "partitions/testTooFew.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 0);
break;
case "partitions/testTooMany.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 0);
break;
}
}
Assert.AreEqual(totalExpectedPartitionsFound, 8);
}
private async Task CreateManifest(CdmCorpusDefinition cdmCorpus)
{
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add each declaration, this example is about medical appointments and care plans
manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address");
manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan");
manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept");
manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact");
manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device");
manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment");
manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter");
manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare");
manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", "");
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add("cdm:/foundations.cdm.json");
Console.WriteLine("Save the documents");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// Define the location of the partition, relative to the manifest
var location = $"local:/{entDef.EntityName}/partition-data.csv";
part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved);
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
// Get the actual location of the partition file from the corpus
string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location);
// Make a fake file with nothing but header for columns
string header = "";
foreach (CdmTypeAttributeDefinition att in entDef.Attributes)
{
if (header != "")
{
header += ",";
}
header += att.Name;
}
Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{entDef.EntityName}"));
File.WriteAllText(partPath, header);
}
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
}
public async Task TestPartitionPatternWithGlob()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestPartitionPatternWithGlob");
int patternWithGlobAndRegex = 0;
corpus.SetEventCallback(new EventCallback {
Invoke = (CdmStatusLevel statusLevel, string message) => {
if (message.Equals("CdmDataPartitionPatternDefinition | The Data Partition Pattern contains both a glob pattern (/testfile.csv) and a regular expression (/subFolder/testSubFile.csv) set, the glob pattern will be used. | FileStatusCheckAsync"))
{
patternWithGlobAndRegex++;
}
}
}, CdmStatusLevel.Warning);
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("pattern.manifest.cdm.json");
await manifest.FileStatusCheckAsync();
// one pattern object contains both glob and regex
Assert.AreEqual(1, patternWithGlobAndRegex);
// make sure '.' in glob is not converted to '.' in regex
CdmLocalEntityDeclarationDefinition dotIsEscaped = (CdmLocalEntityDeclarationDefinition)manifest.Entities[0];
Assert.AreEqual(dotIsEscaped.DataPartitionPatterns[0].GlobPattern, "test.ile.csv");
Assert.AreEqual(dotIsEscaped.DataPartitions.Count, 0);
// star pattern should not match anything
CdmLocalEntityDeclarationDefinition onlyStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[1];
Assert.AreEqual(onlyStar.DataPartitionPatterns[0].GlobPattern, "*");
Assert.AreEqual(onlyStar.DataPartitions.Count, 0);
// star can match nothing
CdmLocalEntityDeclarationDefinition starNoMatch = (CdmLocalEntityDeclarationDefinition)manifest.Entities[2];
Assert.AreEqual(starNoMatch.DataPartitionPatterns[0].GlobPattern, "/testfile*.csv");
Assert.AreEqual(starNoMatch.DataPartitions.Count, 1);
Assert.AreEqual(starNoMatch.DataPartitions[0].Location, "/partitions/testfile.csv");
// star at root level
// this should match any files at root level, none in subfolders
CdmLocalEntityDeclarationDefinition starAtRoot = (CdmLocalEntityDeclarationDefinition)manifest.Entities[3];
Assert.AreEqual(starAtRoot.DataPartitionPatterns[0].GlobPattern, "/*.csv");
Assert.AreEqual(starAtRoot.DataPartitions.Count, 1);
Assert.AreEqual(starAtRoot.DataPartitions[0].Location, "/partitions/testfile.csv");
// star at deeper level
CdmLocalEntityDeclarationDefinition starAtDeeperLevel = (CdmLocalEntityDeclarationDefinition)manifest.Entities[4];
Assert.AreEqual(starAtDeeperLevel.DataPartitionPatterns[0].GlobPattern, "/*/*.csv");
Assert.AreEqual(starAtDeeperLevel.DataPartitions.Count, 1);
Assert.AreEqual(starAtDeeperLevel.DataPartitions[0].Location, "/partitions/subFolder/testSubFile.csv");
// pattern that ends with star
CdmLocalEntityDeclarationDefinition endsWithStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[5];
Assert.AreEqual(endsWithStar.DataPartitionPatterns[0].GlobPattern, "/testfile*");
Assert.AreEqual(endsWithStar.DataPartitions.Count, 1);
Assert.AreEqual(endsWithStar.DataPartitions[0].Location, "/partitions/testfile.csv");
// globstar (**) on its own matches
CdmLocalEntityDeclarationDefinition globStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[6];
Assert.AreEqual(globStar.DataPartitionPatterns[0].GlobPattern, "**");
Assert.AreEqual(2, globStar.DataPartitions.Count);
Assert.AreEqual(1, globStar.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, globStar.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
// globstar at the beginning of the pattern
CdmLocalEntityDeclarationDefinition beginsWithGlobstar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[7];
Assert.AreEqual(beginsWithGlobstar.DataPartitionPatterns[0].GlobPattern, "/**.csv");
Assert.AreEqual(1, beginsWithGlobstar.DataPartitions.Count);
Assert.AreEqual(beginsWithGlobstar.DataPartitions[0].Location, "/partitions/testfile.csv");
// globstar at the end of the pattern
CdmLocalEntityDeclarationDefinition endsWithGlobstar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[8];
Assert.AreEqual(endsWithGlobstar.DataPartitionPatterns[0].GlobPattern, "/**");
Assert.AreEqual(endsWithGlobstar.DataPartitions.Count, 2);
Assert.AreEqual(1, endsWithGlobstar.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, endsWithGlobstar.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
// globstar matches zero or more folders
CdmLocalEntityDeclarationDefinition zeroOrMoreFolders = (CdmLocalEntityDeclarationDefinition)manifest.Entities[9];
Assert.AreEqual(zeroOrMoreFolders.DataPartitionPatterns[0].GlobPattern, "/**/*.csv");
Assert.AreEqual(2, zeroOrMoreFolders.DataPartitions.Count);
Assert.AreEqual(1, zeroOrMoreFolders.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, zeroOrMoreFolders.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
// globstar matches zero or more folders without starting slash
CdmLocalEntityDeclarationDefinition zeroOrMoreNoStartingSlash = (CdmLocalEntityDeclarationDefinition)manifest.Entities[10];
Assert.AreEqual(zeroOrMoreNoStartingSlash.DataPartitionPatterns[0].GlobPattern, "/**/*.csv");
Assert.AreEqual(2, zeroOrMoreNoStartingSlash.DataPartitions.Count);
Assert.AreEqual(1, zeroOrMoreNoStartingSlash.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, zeroOrMoreNoStartingSlash.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
// question mark in the middle of a pattern
CdmLocalEntityDeclarationDefinition questionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[11];
Assert.AreEqual(questionMark.DataPartitionPatterns[0].GlobPattern, "/test?ile.csv");
Assert.AreEqual(1, questionMark.DataPartitions.Count);
Assert.AreEqual(questionMark.DataPartitions[0].Location, "/partitions/testfile.csv");
// question mark at the beginning of a pattern
CdmLocalEntityDeclarationDefinition beginsWithQuestionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[12];
Assert.AreEqual(beginsWithQuestionMark.DataPartitionPatterns[0].GlobPattern, "/?estfile.csv");
Assert.AreEqual(1, beginsWithQuestionMark.DataPartitions.Count);
Assert.AreEqual(beginsWithQuestionMark.DataPartitions[0].Location, "/partitions/testfile.csv");
// question mark at the end of a pattern
CdmLocalEntityDeclarationDefinition endsWithQuestionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[13];
Assert.AreEqual(endsWithQuestionMark.DataPartitionPatterns[0].GlobPattern, "/testfile.cs?");
Assert.AreEqual(1, endsWithQuestionMark.DataPartitions.Count);
Assert.AreEqual(endsWithQuestionMark.DataPartitions[0].Location, "/partitions/testfile.csv");
// backslash in glob can match slash
CdmLocalEntityDeclarationDefinition backslashInPattern = (CdmLocalEntityDeclarationDefinition)manifest.Entities[14];
Assert.AreEqual(backslashInPattern.DataPartitionPatterns[0].GlobPattern, "\\testfile.csv");
Assert.AreEqual(1, backslashInPattern.DataPartitions.Count);
Assert.AreEqual(backslashInPattern.DataPartitions[0].Location, "/partitions/testfile.csv");
// pattern object includes glob pattern and regular expression
CdmLocalEntityDeclarationDefinition globAndRegex = (CdmLocalEntityDeclarationDefinition)manifest.Entities[15];
Assert.AreEqual(globAndRegex.DataPartitionPatterns[0].GlobPattern, "/testfile.csv");
Assert.AreEqual(globAndRegex.DataPartitionPatterns[0].RegularExpression, "/subFolder/testSubFile.csv");
Assert.AreEqual(1, globAndRegex.DataPartitions.Count);
// matching this file means the glob pattern was (correctly) used
Assert.AreEqual(globAndRegex.DataPartitions[0].Location, "/partitions/testfile.csv");
}
/// <summary>
/// Get the text version of all the resolved entities.
/// </summary>
/// <param name="cdmCorpus"> The CDM corpus. </param>
/// <param name="directives"> The directives to use while getting the resolved entities. </param>
/// <param name="manifest"> The manifest to be resolved. </param>
/// <param name="spew"> The object used to store the text to be returned. </param>
/// <returns> The text version of the resolved entities. (it's in a form that facilitates debugging) </returns>
internal static async Task <string> ListAllResolved(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CdmManifestDefinition manifest, StringSpewCatcher spew = null)
{
// make sure the corpus has a set of default artifact attributes
await cdmCorpus.PrepareArtifactAttributesAsync();
var seen = new HashSet <string>();
Func <CdmManifestDefinition, Task> seekEntities = null;
seekEntities = async(CdmManifestDefinition f) =>
{
if (f.Entities != null)
{
if (spew != null)
{
spew.SpewLine(f.FolderPath);
}
foreach (CdmEntityDeclarationDefinition entity in f.Entities)
{
string corpusPath;
CdmEntityDeclarationDefinition ent = entity;
CdmObject currentFile = f;
while (ent is CdmReferencedEntityDeclarationDefinition)
{
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(ent.EntityPath, currentFile);
ent = await cdmCorpus.FetchObjectAsync <CdmReferencedEntityDeclarationDefinition>(corpusPath);
currentFile = ent;
}
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(((CdmLocalEntityDeclarationDefinition)ent).EntityPath, currentFile);
ResolveOptions resOpt = new ResolveOptions()
{
ImportsLoadStrategy = ImportsLoadStrategy.Load
};
CdmEntityDefinition newEnt = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(corpusPath, null, resOpt);
resOpt.WrtDoc = newEnt.InDocument;
resOpt.Directives = directives;
ResolvedEntity resEnt = new ResolvedEntity(resOpt, newEnt);
if (spew != null)
{
resEnt.Spew(resOpt, spew, " ", true);
}
}
}
if (f.SubManifests != null)
{
// folder.SubManifests.ForEach(async f =>
foreach (CdmManifestDeclarationDefinition subManifest in f.SubManifests)
{
string corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(subManifest.Definition, f);
await seekEntities(await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(corpusPath));
}
}
};
await seekEntities(manifest);
if (spew != null)
{
return(spew.GetContent());
}
return("");
}
private static async Task RunTest(string testName, string sourceEntityName)
{
try
{
string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName);
string testExpectedOutputPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName);
string testActualOutputPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName);
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("localInput", new LocalAdapter(testInputPath));
corpus.Storage.Mount("localExpectedOutput", new LocalAdapter(testExpectedOutputPath));
corpus.Storage.Mount("localActualOutput", new LocalAdapter(testActualOutputPath));
corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath));
corpus.Storage.DefaultNamespace = "localInput";
CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"localInput:/{sourceEntityName}.cdm.json/{sourceEntityName}") as CdmEntityDefinition;
Assert.IsTrue(srcEntityDef != null);
var resOpt = new ResolveOptions
{
WrtDoc = srcEntityDef.InDocument,
Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
})
};
CdmFolderDefinition actualOutputFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("localActualOutput:/");
CdmEntityDefinition resolvedEntityDef = null;
string outputEntityFileName = string.Empty;
string entityFileName = string.Empty;
entityFileName = "default";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "normalized";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly_normalized";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "normalized_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly_normalized_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized", "structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public async Task TestCalculateRelationshipsAndPopulateManifests()
{
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests");
var expectedAllManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedAllManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedAllSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedAllSubManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedExclusiveManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedExclusiveManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedExclusiveSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsAndPopulateManifests", "expectedExclusiveSubManifestRels.json")).ToObject <List <E2ERelationship> >();
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new Utilities.EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(testInputPath));
corpus.Storage.DefaultNamespace = "local";
CdmManifestDefinition rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json");
string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(rootManifest.SubManifests[0].Definition);
CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition;
await corpus.CalculateEntityGraphAsync(rootManifest);
await rootManifest.PopulateManifestRelationshipsAsync();
Assert.AreEqual(rootManifest.Relationships.Count, 5);
Assert.AreEqual(subManifest.Relationships.Count, 7);
// check that each relationship has been created correctly
foreach (E2ERelationship expectedRel in expectedAllManifestRels)
{
List <CdmE2ERelationship> found = rootManifest.Relationships.Where(x =>
x.FromEntity == expectedRel.FromEntity &&
x.FromEntityAttribute == expectedRel.FromEntityAttribute &&
x.ToEntity == expectedRel.ToEntity &&
x.ToEntityAttribute == expectedRel.ToEntityAttribute
).ToList();
Assert.AreEqual(found.Count, 1);
}
foreach (E2ERelationship expectedSubRel in expectedAllSubManifestRels)
{
List <CdmE2ERelationship> found = subManifest.Relationships.Where(x =>
x.FromEntity == expectedSubRel.FromEntity &&
x.FromEntityAttribute == expectedSubRel.FromEntityAttribute &&
x.ToEntity == expectedSubRel.ToEntity &&
x.ToEntityAttribute == expectedSubRel.ToEntityAttribute
).ToList();
Assert.AreEqual(found.Count, 1);
}
// make sure only relationships where to and from entities are in the manifest are found with the "exclusive" option is passed in
await rootManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.Exclusive);
Assert.AreEqual(rootManifest.Relationships.Count, 3);
Assert.AreEqual(subManifest.Relationships.Count, 3);
// check that each relationship has been created correctly
foreach (E2ERelationship expectedRel in expectedExclusiveManifestRels)
{
List <CdmE2ERelationship> found = rootManifest.Relationships.Where(x =>
x.FromEntity == expectedRel.FromEntity &&
x.FromEntityAttribute == expectedRel.FromEntityAttribute &&
x.ToEntity == expectedRel.ToEntity &&
x.ToEntityAttribute == expectedRel.ToEntityAttribute
).ToList();
Assert.AreEqual(found.Count, 1);
}
foreach (E2ERelationship expectedSubRel in expectedExclusiveSubManifestRels)
{
List <CdmE2ERelationship> found = subManifest.Relationships.Where(x =>
x.FromEntity == expectedSubRel.FromEntity &&
x.FromEntityAttribute == expectedSubRel.FromEntityAttribute &&
x.ToEntity == expectedSubRel.ToEntity &&
x.ToEntityAttribute == expectedSubRel.ToEntityAttribute
).ToList();
Assert.AreEqual(found.Count, 1);
}
// make sure no relationships are added when "none" relationship option is passed in
await rootManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.None);
Assert.AreEqual(rootManifest.Relationships.Count, 0);
Assert.AreEqual(subManifest.Relationships.Count, 0);
}
static async Task Main(string[] args)
{
// ------------------------------------------------------------------------------------------------------------
// Instantiate a corpus. The corpus is the collection of all documents and folders created or discovered
// while navigating objects and paths.
var cdmCorpus = new CdmCorpusDefinition();
// ------------------------------------------------------------------------------------------------------------
// Configure storage adapters and mount them to the corpus.
// We want our storage adapters to point at the local manifest location and at the example public standards.
string pathFromExeToExampleRoot = "../../../../../../";
// Storage adapter pointing to the target local manifest location.
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "1-read-manifest"));
// 'local' is our default namespace.
// Any paths that start navigating without a device tag (ex. 'cdm') will just default to the 'local' namepace.
cdmCorpus.Storage.DefaultNamespace = "local";
// Storage adapter pointing to the example public standards.
// This is a fake 'cdm'; normally the Github adapter would be used to point at the real public standards.
// Mount it as the 'cdm' device, not the default, so that we must use "cdm:<folder-path>" to get there.
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS:
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// )
// );
// ------------------------------------------------------------------------------------------------------------
// Open the default manifest file at the root.
var manifestFile = "default.manifest.cdm.json";
// This method turns relative corpus paths into absolute paths in case we are in some sub-folders
// and don't know it.
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestFile);
// ------------------------------------------------------------------------------------------------------------
// List all the entities found in the manifest and allow the user to choose which entity to explore.
while (true)
{
Console.WriteLine($"List of all entities found in {manifestFile}:");
// Loop over any entity declarations.
int iEnt = 1;
foreach (var entDec in manifest.Entities)
{
// Print it out.
// Assume there are only local entities in this manifest for simplicity.
Console.Write(" " + iEnt.ToString().PadRight(3));
Console.Write(" " + entDec.EntityName.PadRight(35));
Console.WriteLine(" " + entDec.EntityPath);
iEnt++;
}
Console.Write("Enter a number to show details for that Entity (press [enter] to exit): ");
// Get the user's choice.
string input = Console.ReadLine();
if (string.IsNullOrEmpty(input))
{
break;
}
// Make sure the user's input is a number.
int num = 0;
if (int.TryParse(input, out num))
{
iEnt = 1;
foreach (var entityDec in manifest.Entities)
{
if (iEnt == num)
{
Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ...");
// From the path to the entity, get the actual schema description.
// Take the local relative path in this doc and make sure it works.
var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc
while (true)
{
// List all the metadata properties of this entity that can be explored.
Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:");
Console.WriteLine(" 1: Attributes");
Console.WriteLine(" 2: Traits");
Console.WriteLine(" 3: Properties");
Console.WriteLine(" 4: Data partition locations");
Console.WriteLine(" 5: Relationships");
Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): ");
// Get the user's choice.
input = Console.ReadLine();
if (string.IsNullOrEmpty(input))
{
Console.WriteLine();
break;
}
// Make sure the user's input is a number.
int choice = 0;
if (int.TryParse(input, out choice))
{
switch (choice)
{
// List the entity's attributes.
case 1:
ListAttributes(entSelected);
break;
// List the entity's traits.
case 2:
ListTraits(entSelected);
break;
// List the entity's properties.
case 3:
ListProperties(entSelected, entityDec);
break;
// List the entity's data partition locations.
case 4:
ListDataPartitionLocations(entityDec);
break;
// List the entity's relationships.
case 5:
if (manifest.Relationships != null && manifest.Relationships.Count > 0)
{
// The manifest file contains pre-calculated entity relationships, so we can read them directly.
ListRelationshipsFromManifest(manifest, entSelected);
}
else
{
// The manifest file doesn't contain relationships, so we have to compute the relationships first.
await cdmCorpus.CalculateEntityGraphAsync(manifest);
ListRelationships(cdmCorpus, entSelected);
}
break;
default:
Console.WriteLine("\nEnter a number between 1-5.");
break;
}
}
else
{
Console.WriteLine("\nEnter a number.");
}
}
}
iEnt++;
}
}
else
{
Console.WriteLine("\nEnter a number.");
Console.WriteLine();
}
}
}
public async Task TestCalculateRelationshipsOnResolvedEntities()
{
var expectedResolvedManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedManifestRels.json")).ToObject <List <E2ERelationship> >();
var expectedResolvedSubManifestRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities", "expectedResolvedSubManifestRels.json")).ToObject <List <E2ERelationship> >();
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestCalculateRelationshipsOnResolvedEntities");
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new Utilities.EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(testInputPath));
corpus.Storage.DefaultNamespace = "local";
var rootManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json");
var resolvedManifest = await LoadAndResolveManifest(corpus, rootManifest, "-resolved");
string subManifestPath = corpus.Storage.CreateAbsoluteCorpusPath(resolvedManifest.SubManifests[0].Definition);
CdmManifestDefinition subManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>(subManifestPath) as CdmManifestDefinition;
// using createResolvedManifest will only populate exclusive relationships
Assert.AreEqual(resolvedManifest.Relationships.Count, expectedResolvedManifestRels.Count);
Assert.AreEqual(subManifest.Relationships.Count, expectedResolvedSubManifestRels.Count);
// check that each relationship has been created correctly
foreach (E2ERelationship expectedRel in expectedResolvedManifestRels)
{
List <CdmE2ERelationship> found = resolvedManifest.Relationships.Where(x =>
x.FromEntity == expectedRel.FromEntity &&
x.FromEntityAttribute == expectedRel.FromEntityAttribute &&
x.ToEntity == expectedRel.ToEntity &&
x.ToEntityAttribute == expectedRel.ToEntityAttribute
).ToList();
Assert.AreEqual(found.Count, 1);
}
foreach (E2ERelationship expectedSubRel in expectedResolvedSubManifestRels)
{
List <CdmE2ERelationship> found = subManifest.Relationships.Where(x =>
x.FromEntity == expectedSubRel.FromEntity &&
x.FromEntityAttribute == expectedSubRel.FromEntityAttribute &&
x.ToEntity == expectedSubRel.ToEntity &&
x.ToEntityAttribute == expectedSubRel.ToEntityAttribute
).ToList();
Assert.AreEqual(found.Count, 1);
}
// it is not enough to check if the relationships are correct.
// We need to check if the incoming and outgoing relationships are
// correct as well. One being correct can cover up the other being wrong
// A
var aEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[0].EntityPath, resolvedManifest);
var aInRels = corpus.FetchIncomingRelationships(aEnt);
var aOutRels = corpus.FetchOutgoingRelationships(aEnt);
Assert.AreEqual(aInRels.Count, 0);
Assert.AreEqual(aOutRels.Count, 1);
Assert.AreEqual(aOutRels[0].FromEntity, "local:/A-resolved.cdm.json/A");
Assert.AreEqual(aOutRels[0].ToEntity, "local:/B-resolved.cdm.json/B");
// B
var bEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(resolvedManifest.Entities[1].EntityPath, resolvedManifest);
var bInRels = corpus.FetchIncomingRelationships(bEnt);
var bOutRels = corpus.FetchOutgoingRelationships(bEnt);
Assert.AreEqual(bInRels.Count, 1);
Assert.AreEqual(bInRels[0].FromEntity, "local:/A-resolved.cdm.json/A");
Assert.AreEqual(bInRels[0].ToEntity, "local:/B-resolved.cdm.json/B");
Assert.AreEqual(bOutRels.Count, 0);
// C
var cEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[0].EntityPath, subManifest);
var cInRels = corpus.FetchIncomingRelationships(cEnt);
var cOutRels = corpus.FetchOutgoingRelationships(cEnt);
Assert.AreEqual(cInRels.Count, 0);
Assert.AreEqual(cOutRels.Count, 2);
Assert.AreEqual(cOutRels[0].FromEntity, "local:/sub/C-resolved.cdm.json/C");
// TODO: this should point to the resolved entity, currently an open bug
Assert.AreEqual(cOutRels[0].ToEntity, "local:/B.cdm.json/B");
Assert.AreEqual(cOutRels[1].FromEntity, "local:/sub/C-resolved.cdm.json/C");
Assert.AreEqual(cOutRels[1].ToEntity, "local:/sub/D-resolved.cdm.json/D");
// D
var dEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>(subManifest.Entities[1].EntityPath, subManifest);
var dInRels = corpus.FetchIncomingRelationships(dEnt);
var dOutRels = corpus.FetchOutgoingRelationships(dEnt);
Assert.AreEqual(dInRels.Count, 1);
Assert.AreEqual(dInRels[0].FromEntity, "local:/sub/C-resolved.cdm.json/C");
Assert.AreEqual(dInRels[0].ToEntity, "local:/sub/D-resolved.cdm.json/D");
Assert.AreEqual(dOutRels.Count, 0);
}
static async Task Main(string[] args)
{
var pathFromExeToExampleRoot = "../../../../../../";
// ------------------------------------------------------------------------------------------------------------
// Instantiate corpus and set up the default namespace to be local
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
// ------------------------------------------------------------------------------------------------------------
// Set up adapters for managing access to different files-system locations
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
cdmCorpus.Storage.Mount("local", new LocalAdapter(modelJsonRoot));
// Example how to mount to the ADLS - make sure the hostname and root entered here are also changed
// in the example.model.json file we load in the next section
cdmCorpus.Storage.Mount("adls",
new ADLSAdapter(
"<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
"/<FILESYSTEM-NAME>", // Root.
"72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
"<CLIENT-ID>", // Client ID.
"<CLIENT-SECRET>" // Client secret.
));
// ------------------------------------------------------------------------------------------------------------
// Load a model.json file from local FS
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/example.model.json");
// ------------------------------------------------------------------------------------------------------------
// Explore entities and partitions defined in the model
Console.WriteLine("Listing entity declarations:");
foreach (CdmEntityDeclarationDefinition decl in manifest.Entities)
{
Console.WriteLine(" " + decl.EntityName);
// TODO: This can be rewritten in a different way since data partition gives null for referenced entities, suggestions are welcome.
if (decl.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
foreach (CdmDataPartitionDefinition dataPart in decl.DataPartitions)
{
Console.WriteLine(" " + dataPart.Location);
}
}
}
// ------------------------------------------------------------------------------------------------------------
// Make changes to the model
// Create a new document where the new entity's definition will be stored
var newEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "NewEntity.cdm.json", false);
newEntityDoc.Imports.Add("cdm:/foundations.cdm.json");
cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(newEntityDoc);
var newEntity = newEntityDoc.Definitions.Add(CdmObjectType.EntityDef, "NewEntity") as CdmEntityDefinition;
// Define new string attribute and add it to the entity definition
var newAttribute = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "NewAttribute", false);
newAttribute.DataFormat = CdmDataFormat.String;
newEntity.Attributes.Add(newAttribute);
// Call will create EntityDeclarationDefinition based on entity definition and add it to manifest.Entities
var newEntityDecl = manifest.Entities.Add(newEntity);
// Define a partition and add it to the local declaration
var newPartition = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "NewPartition", false);
newPartition.Location = "adls:/NewPartition.csv";
newEntityDecl.DataPartitions.Add(newPartition);
// ------------------------------------------------------------------------------------------------------------
// Save the file to ADLSg2 - we achieve that by adding the manifest to the root folder of
// the ADLS file-system and performing a save on the manifest
CdmFolderDefinition adlsFolder = cdmCorpus.Storage.FetchRootFolder("adls");
adlsFolder.Documents.Add(manifest);
await manifest.SaveAsAsync("example.model.json", true);
}
public async Task TestLoadProjection()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestLoadProjection");
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json");
string expected = "TestSource";
string actual = null;
CdmObjectType actualType = CdmObjectType.Error;
#region TestEntityStringReference.cdm.json
CdmEntityDefinition entTestEntityStringReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityStringReference.cdm.json/TestEntityStringReference", manifest);
Assert.IsNotNull(entTestEntityStringReference);
actual = ((CdmEntityReference)entTestEntityStringReference.ExtendsEntity).NamedReference;
actualType = ((CdmEntityReference)entTestEntityStringReference.ExtendsEntity).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.EntityRef, actualType);
#endregion // TestEntityStringReference.cdm.json
#region TestEntityEntityReference.cdm.json
CdmEntityDefinition entTestEntityEntityReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityEntityReference.cdm.json/TestEntityEntityReference", manifest);
Assert.IsNotNull(entTestEntityEntityReference);
actual = ((CdmEntityReference)entTestEntityEntityReference.ExtendsEntity).NamedReference;
actualType = ((CdmEntityReference)entTestEntityEntityReference.ExtendsEntity).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.EntityRef, actualType);
#endregion // TestEntityEntityReference.cdm.json
#region TestEntityProjection.cdm.json
CdmEntityDefinition entTestEntityProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityProjection.cdm.json/TestEntityProjection", manifest);
Assert.IsNotNull(entTestEntityProjection);
actual = ((CdmEntityReference)((CdmProjection)entTestEntityProjection.ExtendsEntity.ExplicitReference).Source).NamedReference;
actualType = ((CdmProjection)entTestEntityProjection.ExtendsEntity.ExplicitReference).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.ProjectionDef, actualType);
#endregion // TestEntityProjection.cdm.json
#region TestEntityNestedProjection.cdm.json
CdmEntityDefinition entTestEntityNestedProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityNestedProjection.cdm.json/TestEntityNestedProjection", manifest);
Assert.IsNotNull(entTestEntityNestedProjection);
actual = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entTestEntityNestedProjection.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).Source.NamedReference;
actualType = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entTestEntityNestedProjection.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.ProjectionDef, actualType);
#endregion // TestEntityNestedProjection.cdm.json
#region TestEntityAttributeStringReference.cdm.json
CdmEntityDefinition entTestEntityAttributeStringReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeStringReference.cdm.json/TestEntityAttributeStringReference", manifest);
Assert.IsNotNull(entTestEntityAttributeStringReference);
actual = ((CdmEntityAttributeDefinition)entTestEntityAttributeStringReference.Attributes[0]).Entity.NamedReference;
actualType = ((CdmEntityAttributeDefinition)entTestEntityAttributeStringReference.Attributes[0]).Entity.ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.EntityRef, actualType);
#endregion // TestEntityAttributeStringReference.cdm.json
#region TestEntityAttributeEntityReference.cdm.json
CdmEntityDefinition entTestEntityAttributeEntityReference = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeEntityReference.cdm.json/TestEntityAttributeEntityReference", manifest);
Assert.IsNotNull(entTestEntityAttributeEntityReference);
actual = ((CdmEntityAttributeDefinition)entTestEntityAttributeEntityReference.Attributes[0]).Entity.NamedReference;
actualType = ((CdmEntityAttributeDefinition)entTestEntityAttributeEntityReference.Attributes[0]).Entity.ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.EntityRef, actualType);
#endregion // TestEntityAttributeEntityReference.cdm.json
#region TestEntityAttributeProjection.cdm.json
CdmEntityDefinition entTestEntityAttributeProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeProjection.cdm.json/TestEntityAttributeProjection", manifest);
Assert.IsNotNull(entTestEntityAttributeProjection);
actual = ((CdmProjection)((CdmEntityAttributeDefinition)entTestEntityAttributeProjection.Attributes[0]).Entity.ExplicitReference).Source.NamedReference;
actualType = ((CdmEntityAttributeDefinition)entTestEntityAttributeProjection.Attributes[0]).Entity.ExplicitReference.ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.ProjectionDef, actualType);
#endregion // TestEntityAttributeProjection.cdm.json
#region TestEntityAttributeNestedProjection.cdm.json
CdmEntityDefinition entTestEntityAttributeNestedProjection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityAttributeNestedProjection.cdm.json/TestEntityAttributeNestedProjection", manifest);
Assert.IsNotNull(entTestEntityAttributeNestedProjection);
actual = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entTestEntityAttributeNestedProjection.Attributes[0]).Entity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).Source.NamedReference;
actualType = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entTestEntityAttributeNestedProjection.Attributes[0]).Entity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(CdmObjectType.ProjectionDef, actualType);
#endregion // TestEntityAttributeNestedProjection.cdm.json
#region TestOperationCollection.cdm.json
CdmEntityDefinition entTestOperationCollection = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestOperationCollection.cdm.json/TestOperationCollection", manifest);
Assert.IsNotNull(entTestOperationCollection);
int actualOperationCount = ((CdmProjection)entTestOperationCollection.ExtendsEntity.ExplicitReference).Operations.Count;
Assert.AreEqual(9, actualOperationCount);
CdmOperationCollection operations = ((CdmProjection)entTestOperationCollection.ExtendsEntity.ExplicitReference).Operations;
Assert.AreEqual(CdmOperationType.AddCountAttribute, operations[0].Type);
Assert.AreEqual(CdmOperationType.AddSupportingAttribute, operations[1].Type);
Assert.AreEqual(CdmOperationType.AddTypeAttribute, operations[2].Type);
Assert.AreEqual(CdmOperationType.ExcludeAttributes, operations[3].Type);
Assert.AreEqual(CdmOperationType.ArrayExpansion, operations[4].Type);
Assert.AreEqual(CdmOperationType.CombineAttributes, operations[5].Type);
Assert.AreEqual(CdmOperationType.RenameAttributes, operations[6].Type);
Assert.AreEqual(CdmOperationType.ReplaceAsForeignKey, operations[7].Type);
Assert.AreEqual(CdmOperationType.IncludeAttributes, operations[8].Type);
#endregion // TestOperationCollection.cdm.json
#region TestEntityTrait.cdm.json
CdmEntityDefinition entTestEntityTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityTrait.cdm.json/TestEntityTrait", manifest);
Assert.IsNotNull(entTestEntityTrait);
Assert.AreEqual("TestAttribute", ((CdmTypeAttributeDefinition)entTestEntityTrait.Attributes[0]).Name);
Assert.AreEqual("testDataType", ((CdmTypeAttributeDefinition)entTestEntityTrait.Attributes[0]).DataType.NamedReference);
#endregion // TestEntityTrait.cdm.json
#region TestEntityExtendsTrait.cdm.json
CdmEntityDefinition entTestEntityExtendsTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestEntityExtendsTrait.cdm.json/TestEntityExtendsTrait", manifest);
Assert.IsNotNull(entTestEntityExtendsTrait);
Assert.AreEqual("TestExtendsTraitAttribute", ((CdmTypeAttributeDefinition)entTestEntityExtendsTrait.Attributes[0]).Name);
Assert.AreEqual("testDerivedDataType", ((CdmTypeAttributeDefinition)entTestEntityExtendsTrait.Attributes[0]).DataType.NamedReference);
#endregion // TestEntityExtendsTrait.cdm.json
#region TestProjectionTrait.cdm.json
CdmEntityDefinition entTestProjectionTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestProjectionTrait.cdm.json/TestProjectionTrait", manifest);
Assert.IsNotNull(entTestProjectionTrait);
Assert.AreEqual("TestProjectionAttribute", ((CdmTypeAttributeDefinition)entTestProjectionTrait.Attributes[0]).Name);
Assert.AreEqual("testDataType", ((CdmTypeAttributeDefinition)entTestProjectionTrait.Attributes[0]).DataType.NamedReference);
#endregion // TestProjectionTrait.cdm.json
#region TestProjectionExtendsTrait.cdm.json
CdmEntityDefinition entTestProjectionExtendsTrait = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/TestProjectionExtendsTrait.cdm.json/TestProjectionExtendsTrait", manifest);
Assert.IsNotNull(entTestProjectionExtendsTrait);
Assert.AreEqual("TestProjectionAttributeB", ((CdmTypeAttributeDefinition)entTestProjectionExtendsTrait.Attributes[0]).Name);
Assert.AreEqual("testExtendsDataTypeB", ((CdmTypeAttributeDefinition)entTestProjectionExtendsTrait.Attributes[0]).DataType.NamedReference);
#endregion // TestProjectionExtendsTrait.cdm.json
}
static async Task Main()
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("Configuring storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
var pathFromExeToExampleRoot = "../";
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "output"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "base-files"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Creating placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local documents in the corpus
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json");
//READ CSV - OBSERVE SCHEMA: "entity, atribute, datatype" every row, no headers
Console.WriteLine("Loading CDM definition file");
List <string> entitylist = new List <string>();
List <string> attributelist = new List <string>();
List <string> datatypelist = new List <string>();
using (var reader = new System.IO.StreamReader(@"../cdmdefinition.csv"))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(',');
entitylist.Add(values[0]);
attributelist.Add(values[1]);
datatypelist.Add(values[2]);
}
}
// Create entities and add attributes
string entityname;
CdmEntityDefinition entity;
bool isnew = true;
for (int i = 0; i < entitylist.Count; i++)
{
isnew = true;
entityname = entitylist[i];
Console.WriteLine("Creating " + entityname + " entity");
entity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, entityname, false);
while (i == 0 || entitylist[i] == entitylist[i - 1] || isnew)
{
isnew = false;
var attribute = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, attributelist[i], "hasA", datatypelist[i]);
entity.Attributes.Add(attribute);
i++;
if (i == entitylist.Count)
{
break;
}
}
if (i != entitylist.Count)
{
i--;
}
// Create the document which contains the entity
var entitydoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityname}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
entitydoc.Imports.Add(FoundationJsonPath);
entitydoc.Definitions.Add(entity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(entitydoc, entitydoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(entity);
}
//THIAGO - MANUAL BREAKDOWN BEFORE AUTOMATING
/*var entityname = "FixedAssetMaster";
* var entity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, entityname, false);
* var attribute1 = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, "Attribute1", "hasA", "string");
* entity.Attributes.Add(attribute1);
* var attribute2 = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, "Attribute2", "hasA", "string");
* entity.Attributes.Add(attribute2);
* // Create the document which contains the entity
* var entitydoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityname}.cdm.json", false);
* // Add an import to the foundations doc so the traits about partitons will resolve nicely
* entitydoc.Imports.Add(FoundationJsonPath);
* entitydoc.Definitions.Add(entity);
* // Add the document to the root of the local documents in the corpus
* localRoot.Documents.Add(entitydoc, entitydoc.Name);
* // Add the entity to the manifest
* manifestAbstract.Entities.Add(entity);*/
/*
* // Create the simplest entity - CustomPerson
* // Create the entity definition instance
* var personEntity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false);
* // Add type attributes to the entity instance
* var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId");
* personEntity.Attributes.Add(personAttributeId);
* var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "string");
* personEntity.Attributes.Add(personAttributeName);
* // Add properties to the entity instance
* personEntity.DisplayName = CustomPersonEntityName;
* personEntity.Version = "0.0.1";
* personEntity.Description = "This is a custom entity created for the sample.";
* // Create the document which contains the entity
* var personEntityDoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false);
* // Add an import to the foundations doc so the traits about partitons will resolve nicely
* personEntityDoc.Imports.Add(FoundationJsonPath);
* personEntityDoc.Definitions.Add(personEntity);
* // Add the document to the root of the local documents in the corpus
* localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name);
* // Add the entity to the manifest
* manifestAbstract.Entities.Add(personEntity);
*
*
* // Create an entity - CustomAccount which has a relationship with the entity CustomPerson
* // Create the entity definition instance
* var accountEntity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false);
* // Add type attributes to the entity instance
* var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId");
* accountEntity.Attributes.Add(accountAttributeId);
* var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "string");
* accountEntity.Attributes.Add(accountAttributeName);
* // Add properties to the entity instance
* accountEntity.DisplayName = CustomAccountEntityName;
* accountEntity.Version = "0.0.1";
* accountEntity.Description = "This is a custom entity created for the sample.";
* // In this sample, every account has one person who owns the account
* // the relationship is actually an entity attribute
* var attributeExplanation = "The owner of the account, which is a person.";
* // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
* // more details of how to use resolution guidance to customize your data
* var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation);
* accountEntity.Attributes.Add(accountOwnerAttribute);
* // Create the document which contains the entity
* var accountEntityDoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false);
* // Add an import to the foundations doc so the traits about partitons will resolve nicely
* accountEntityDoc.Imports.Add(FoundationJsonPath);
* // the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits,
* // the import to the entity reference CustomPerson's doc is required
* accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json");
* accountEntityDoc.Definitions.Add(accountEntity);
* // Add the document to the root of the local documents in the corpus
* localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name);
* // Add the entity to the manifest
* manifestAbstract.Entities.Add(accountEntity);
*
*
* // Create an entity which extends "Account" from the standard, it contains everything that "Account" has
* var extendedStandardAccountEntity = cdmCorpus.MakeObject<CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false);
* // This function with 'true' will make a simple reference to the base
* extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject<CdmEntityReference>(CdmObjectType.EntityRef, "Account", true);
* var attrExplanation = "This is a simple custom account for this sample.";
* // Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount
* // You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
* // more details of how to use resolution guidance to customize your data
* var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation);
* extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute);
* var extendedStandardAccountEntityDoc = cdmCorpus.MakeObject<CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false);
* // Add an import to the foundations doc so the traits about partitons will resolve nicely
* extendedStandardAccountEntityDoc.Imports.Add(FoundationJsonPath);
* // The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required
* // it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required
* extendedStandardAccountEntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json");
* extendedStandardAccountEntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json");
* // Add the document to the root of the local documents in the corpus
* localRoot.Documents.Add(extendedStandardAccountEntityDoc, extendedStandardAccountEntityDoc.Name);
* extendedStandardAccountEntityDoc.Definitions.Add(extendedStandardAccountEntity);
* // Add the entity to the manifest
* manifestAbstract.Entities.Add(extendedStandardAccountEntity);
*/
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolving the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add(FoundationJsonPath);
Console.WriteLine("Saving the documents");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// We have existing partition files for the custom entities, so we need to make the partition point to the file location
part.Location = $"local:/{entDef.EntityName}/partition-data.csv";
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
}
// We can save the documents as manifest.cdm.json format or model.json
// Save as manifest.cdm.json
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// Save as a model.json
// await manifestResolved.SaveAsAsync("model.json", true);
Console.WriteLine("Finished");
Console.ReadKey();
}
private async Task LogicalManipulationUsingProjections(CdmCorpusDefinition corpus)
{
Console.WriteLine("Create logical entity definition.");
var logicalFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("output:/");
var logicalDoc = logicalFolder.Documents.Add("Person.cdm.json");
logicalDoc.Imports.Add("local:/Address.cdm.json");
var entity = logicalDoc.Definitions.Add("Person");
// Add "name" data typed attribute.
var nameAttr = entity.Attributes.Add("name") as CdmTypeAttributeDefinition;
nameAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true);
// Add "age" data typed attribute.
var ageAttr = entity.Attributes.Add("age") as CdmTypeAttributeDefinition;
ageAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true);
// Add "address" entity typed attribute.
var entityAttr = new CdmEntityAttributeDefinition(corpus.Ctx, "address")
{
Entity = new CdmEntityReference(corpus.Ctx, "Address", true)
};
ApplyArrayExpansion(entityAttr, 1, 3, "{m}{A}{o}", "countAttribute");
ApplyDefaultBehavior(entityAttr, "addressFK", "address");
entity.Attributes.Add(entityAttr);
// Add "email" data typed attribute.
var emailAttr = entity.Attributes.Add("email") as CdmTypeAttributeDefinition;
emailAttr.DataType = new CdmDataTypeReference(corpus.Ctx, "string", true);
// Save the logical definition of Person.
await entity.InDocument.SaveAsAsync("Person.cdm.json");
Console.WriteLine("Get \"resolved\" folder where the resolved entities will be saved.");
var resolvedFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("output:/");
var resOpt = new ResolveOptions(entity);
// To get more information about directives and their meaning refer to
// https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
// We will start by resolving this entity with the "normalized" direcitve.
// This directive will be used on this and the next two examples so we can analize the resolved entity
// without the array expansion.
Console.WriteLine("Resolving logical entity with normalized directive.");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
var resNormalizedEntity = await entity.CreateResolvedEntityAsync($"normalized_{entity.EntityName}", resOpt, resolvedFolder);
await resNormalizedEntity.InDocument.SaveAsAsync($"{resNormalizedEntity.EntityName}.cdm.json");
// Another common scenario is to resolve an entity using the "referenceOnly" directive.
// This directives is used to replace the relationships with a foreign key.
Console.WriteLine("Resolving logical entity with referenceOnly directive.");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var resReferenceOnlyEntity = await entity.CreateResolvedEntityAsync($"referenceOnly_{entity.EntityName}", resOpt, resolvedFolder);
await resReferenceOnlyEntity.InDocument.SaveAsAsync($"{resReferenceOnlyEntity.EntityName}.cdm.json");
// When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
// a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
// entity attribute and we use the "structured" directive to resolve the entity attribute as an attribute group.
Console.WriteLine("Resolving logical entity with structured directive.");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
var resStructuredEntity = await entity.CreateResolvedEntityAsync($"structured_{entity.EntityName}", resOpt, resolvedFolder);
await resStructuredEntity.InDocument.SaveAsAsync($"{resStructuredEntity.EntityName}.cdm.json");
// Now let us remove the "normalized" directive so the array expansion operation can run.
Console.WriteLine("Resolving logical entity without directives (array expansion).");
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
var resArrayEntity = await entity.CreateResolvedEntityAsync($"array_expansion_{entity.EntityName}", resOpt, resolvedFolder);
await resArrayEntity.InDocument.SaveAsAsync($"{resArrayEntity.EntityName}.cdm.json");
}
/// <summary>
/// Common test code for these test cases
/// </summary>
/// <param name="testName"></param>
/// <param name="entityName"></param>
private async Task TestRun(string testName, string entityName, bool isEntitySet)
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
string inputFolder = TestHelper.GetInputFolderPath(testsSubpath, testName);
string expectedOutputFolder = TestHelper.GetExpectedOutputFolderPath(testsSubpath, testName);
string actualOutputFolder = TestHelper.GetActualOutputFolderPath(testsSubpath, testName);
if (!Directory.Exists(actualOutputFolder))
{
Directory.CreateDirectory(actualOutputFolder);
}
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json");
Assert.IsNotNull(manifest);
CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}", manifest);
Assert.IsNotNull(entity);
CdmEntityDefinition resolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> {
"referenceOnly"
});
AssertEntityShapeInResolvedEntity(resolvedEntity, isEntitySet);
await AttributeContextUtil.ValidateAttributeContext(corpus, expectedOutputFolder, entityName, resolvedEntity);
await corpus.CalculateEntityGraphAsync(manifest);
await manifest.PopulateManifestRelationshipsAsync();
string actualRelationshipsString = ListRelationships(corpus, entity, actualOutputFolder, entityName);
string relationshipsFilename = $"REL_{entityName}.txt";
File.WriteAllText(Path.Combine(actualOutputFolder, relationshipsFilename), actualRelationshipsString);
string expectedRelationshipsStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, relationshipsFilename));
string expectedRelationshipsString = File.ReadAllText(expectedRelationshipsStringFilePath);
Assert.AreEqual(expectedRelationshipsString, actualRelationshipsString);
CdmFolderDefinition outputFolder = corpus.Storage.FetchRootFolder("output");
outputFolder.Documents.Add(manifest);
string manifestFileName = $"saved.manifest.cdm.json";
await manifest.SaveAsAsync(manifestFileName, saveReferenced : true);
string actualManifestPath = Path.Combine(actualOutputFolder, manifestFileName);
if (!File.Exists(actualManifestPath))
{
Assert.Fail("Unable to save manifest with relationship");
}
else
{
CdmManifestDefinition savedManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"output:/{manifestFileName}");
string actualSavedManifestRel = GetRelationshipStrings(savedManifest.Relationships);
string manifestRelationshipsFilename = $"MANIFEST_REL_{entityName}.txt";
File.WriteAllText(Path.Combine(actualOutputFolder, manifestRelationshipsFilename), actualSavedManifestRel);
string expectedSavedManifestRel = File.ReadAllText(Path.Combine(expectedOutputFolder, manifestRelationshipsFilename));
Assert.AreEqual(expectedSavedManifestRel, actualSavedManifestRel);
}
}
static async Task Main(string[] args)
{
Console.WriteLine("Hello World!");
string pathToDocRoot;
string docGroup;
CdmManifestDefinition manifest;
string testEnt;
bool testCorpus = false;
bool resolveEnt = false;
bool spewAll = true;
bool rePersist = false;
CdmCorpusDefinition cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
if (testCorpus)
{
pathToDocRoot = "../../../../../../../../CDM.Tools.Internal/TestCorpus";
//pathToDocRoot = "../perfTestCorpus";
//docGroup = "E2EResolution";
//docGroup = "POVResolution";
//docGroup = "MiniDyn";
//docGroup = "composites";
//docGroup = "KnowledgeGraph";
//docGroup = "overrides";
docGroup = "webClicks";
//testEnt = "/E2EResolution/E2EArrayOne.cdm.json/E2EArrayOne";
//testEnt = "/MiniDyn/sub/Lead.cdm.json/Lead";
// testEnt = "/POVResolution/sub1/Main.cdm.json/Main";
testEnt = "local:/MiniDyn/Account.cdm.json/Account";
}
else
{
pathToDocRoot = "../../../../../../../../CDM.SchemaDocuments";
testEnt = "local:/core/applicationCommon/foundationCommon/crmCommon/Account.cdm.json/Account";
docGroup = "standards";
}
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathToDocRoot));
manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{docGroup}.manifest.cdm.json");
string version = "";
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Progress);
Console.WriteLine("reading source files");
if (resolveEnt)
{
// AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet<string> { "normalized", "xstructured", "referenceOnly" });
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var ent = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(testEnt);
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = ent.InDocument, Directives = directives
};
var x = await ent.CreateResolvedEntityAsync("RESOLVED_KILL", resOpt);
resOpt.WrtDoc = x.InDocument;
CommonDataModelLoader.PersistDocument(cdmCorpus.RootPath, resOpt, new CopyOptions {
StringRefs = false, RemoveSingleRowLocalizedTableTraits = true
});
}
if (spewAll)
{
Console.WriteLine("list all resolved");
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "xstructured", "referenceOnly"
});
await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher());
}
if (rePersist)
{
Console.WriteLine("persist corpus");
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "xstructured", "referenceOnly"
});
}
//ListAllTraits(cdmCorpus);
Console.WriteLine("done");
Console.ReadLine();
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../";
// Mount is as a local device.
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "3-customize-entities"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
// Open the default manifest at the root, used later when done
// This method turns relative corpus paths into absolute ones in case we are in some sub-folders and don't know it
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("default.manifest.cdm.json");
Console.WriteLine("Define new extension");
// First we will make a new document right in the same folder as the manifest
var docAbs = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "MobileCareTeam.cdm.json");
// Import the cdm description of the original so the symbols will resolve
docAbs.Imports.Add("cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CareTeam.cdm.json", null);
// We will make a new trait to identify things that are known to be temporary, used later
// In theory this would be defined somewhere central so it can be shared
var traitTemp = docAbs.Definitions.Add(CdmObjectType.TraitDef, "means.temporary") as CdmTraitDefinition;
// Extends the standard 'means' base trait
traitTemp.ExtendsTrait = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means", true);
// Add a parameter for the expected duration in days
var param = cdmCorpus.MakeObject <CdmParameterDefinition>(CdmObjectType.ParameterDef, "estimatedDays");
// By not using "true" on the last arg, this becomes an real reference object in the json. go look at the difference from "means" when this is done
param.DataTypeRef = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "integer");
param.DefaultValue = "30";
traitTemp.Parameters.Add(param);
// Make an entity definition and add it to the list of definitions in the document.
CdmEntityDefinition entAbs = docAbs.Definitions.Add(CdmObjectType.EntityDef, "MobileCareTeam") as CdmEntityDefinition;
// This entity extends the standard
// This function with 'true' will make a simple reference to the base
entAbs.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "CareTeam", true);
// and we will add an attribute
CdmTypeAttributeDefinition attNew = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "currentCity");
// The attribute is a type is 'City" this is one of the predefined semantic types in meanings.cdm.json
attNew.DataType = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "city", true);
attNew.Description = "The current city where the mobile care team is working";
// also apply our fancy new 'temporary' trait. they stay in a city for 90 days on average
CdmTraitReference tr = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means.temporary");
tr.Arguments.Add("estimatedDays", "90");
attNew.AppliedTraits.Add(tr);
// Add attribute to the entity
entAbs.Attributes.Add(attNew);
// The entity abstract definition is done, add the document to the corpus in the root folder and then save that doc
cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(docAbs);
// next step is to remove all of the guesswork out of decoding the entity shape by 'resolving' it to a relational by reference shape
Console.WriteLine("Make a local 'resolved' copy");
// Now resolve it
// Made the entity and document have a different name to avoid conflicts in this folder
var entFlat = await entAbs.CreateResolvedEntityAsync("LocalMobileCareTeam");
// Now just add the pointer into our manifest.
Console.WriteLine("Add to manifest");
manifest.Entities.Add(entFlat);
// This function will update all of the fileStatus times in the manifest
// await manifest.RefreshFileStatus();
// Save the manifest along with linked definition files
await manifest.SaveAsAsync("default-resolved.manifest.cdm.json", true);
}
private async Task CustomizeEntities(CdmCorpusDefinition cdmCorpus)
{
// Open the default manifest at the root, used later when done
// This method turns relative corpus paths into absolute ones in case we are in some sub-folders and don't know it
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("default.manifest.cdm.json");
Console.WriteLine("Define new extension");
// First we will make a new document right in the same folder as the manifest
var docAbs = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "MobileCareTeam.cdm.json");
// Import the cdm description of the original so the symbols will resolve
docAbs.Imports.Add("cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CareTeam.cdm.json", null);
// We will make a new trait to identify things that are known to be temporary, used later
// In theory this would be defined somewhere central so it can be shared
var traitTemp = docAbs.Definitions.Add(CdmObjectType.TraitDef, "means.temporary") as CdmTraitDefinition;
// Extends the standard 'means' base trait
traitTemp.ExtendsTrait = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means", true);
// Add a parameter for the expected duration in days
var param = cdmCorpus.MakeObject <CdmParameterDefinition>(CdmObjectType.ParameterDef, "estimatedDays");
// By not using "true" on the last arg, this becomes an real reference object in the json. go look at the difference from "means" when this is done
param.DataTypeRef = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "integer");
param.DefaultValue = "30";
traitTemp.Parameters.Add(param);
// Make an entity definition and add it to the list of definitions in the document.
CdmEntityDefinition entAbs = docAbs.Definitions.Add(CdmObjectType.EntityDef, "MobileCareTeam") as CdmEntityDefinition;
// This entity extends the standard
// This function with 'true' will make a simple reference to the base
entAbs.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "CareTeam", true);
// and we will add an attribute
CdmTypeAttributeDefinition attNew = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "currentCity");
// The attribute is a type is 'City" this is one of the predefined semantic types in meanings.cdm.json
attNew.DataType = cdmCorpus.MakeObject <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "city", true);
attNew.Description = "The current city where the mobile care team is working.";
// also apply our fancy new 'temporary' trait. they stay in a city for 90 days on average
CdmTraitReference tr = cdmCorpus.MakeObject <CdmTraitReference>(CdmObjectType.TraitRef, "means.temporary");
tr.Arguments.Add("estimatedDays", "90");
attNew.AppliedTraits.Add(tr);
// Add attribute to the entity
entAbs.Attributes.Add(attNew);
// The entity abstract definition is done, add the document to the corpus in the root folder and then save that doc
cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(docAbs);
// next step is to remove all of the guesswork out of decoding the entity shape by 'resolving' it to a relational by reference shape
Console.WriteLine("Make a local 'resolved' copy");
// Now resolve it
// Made the entity and document have a different name to avoid conflicts in this folder
var entFlat = await entAbs.CreateResolvedEntityAsync("LocalMobileCareTeam");
// Now just add the pointer into our manifest.
Console.WriteLine("Add to manifest");
manifest.Entities.Add(entFlat);
// This function will update all of the fileStatus times in the manifest
// await manifest.RefreshAsync(null);
// Save the manifest along with linked definition files.
await manifest.SaveAsAsync("default-resolved.manifest.cdm.json", true);
}
private async Task CreateNetNewEntities(CdmCorpusDefinition cdmCorpus)
{
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local adapter in the corpus
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json");
// Create two entities from scratch, and add some attributes, traits, properties, and relationships in between
Console.WriteLine("Create net new entities");
// Create the simplest entity - CustomPerson
// Create the entity definition instance
var personEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false);
// Add type attributes to the entity instance
var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId");
personEntity.Attributes.Add(personAttributeId);
var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "name");
personEntity.Attributes.Add(personAttributeName);
// Add properties to the entity instance
personEntity.DisplayName = CustomPersonEntityName;
personEntity.Version = "0.0.1";
personEntity.Description = "This is a custom entity created for the sample.";
// Create the document which contains the entity
var personEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
personEntityDoc.Imports.Add(FoundationJsonPath);
personEntityDoc.Definitions.Add(personEntity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(personEntity);
// Create an entity - CustomAccount which has a relationship with the entity CustomPerson
// Create the entity definition instance
var accountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false);
// Add type attributes to the entity instance
var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId");
accountEntity.Attributes.Add(accountAttributeId);
var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "name");
accountEntity.Attributes.Add(accountAttributeName);
// Add properties to the entity instance
accountEntity.DisplayName = CustomAccountEntityName;
accountEntity.Version = "0.0.1";
accountEntity.Description = "This is a custom entity created for the sample.";
// In this sample, every account has one person who owns the account
// the relationship is actually an entity attribute
var attributeExplanation = "The owner of the account, which is a person.";
// You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
// more details of how to use resolution guidance to customize your data
var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation);
accountEntity.Attributes.Add(accountOwnerAttribute);
// Create the document which contains the entity
var accountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
accountEntityDoc.Imports.Add(FoundationJsonPath);
// the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits,
// the import to the entity reference CustomPerson's doc is required
accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json");
accountEntityDoc.Definitions.Add(accountEntity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(accountEntity);
// Create an entity which extends "Account" from the standard, it contains everything that "Account" has
var extendedStandardAccountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false);
// This function with 'true' will make a simple reference to the base
extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "Account", true);
var attrExplanation = "This is a simple custom account for this sample.";
// Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount
// You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
// more details of how to use resolution guidance to customize your data
var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation);
extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute);
var extendedStandardAccountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
extendedStandardAccountEntityDoc.Imports.Add(FoundationJsonPath);
// The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required
// it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required
extendedStandardAccountEntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json");
extendedStandardAccountEntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json");
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(extendedStandardAccountEntityDoc, extendedStandardAccountEntityDoc.Name);
extendedStandardAccountEntityDoc.Definitions.Add(extendedStandardAccountEntity);
// Add the entity to the manifest
manifestAbstract.Entities.Add(extendedStandardAccountEntity);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add(FoundationJsonPath);
Console.WriteLine("Save the documents");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// We have existing partition files for the custom entities, so we need to make the partition point to the file location
part.Location = $"local:/{entDef.EntityName}/partition-data.csv";
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
}
// We can save the documents as manifest.cdm.json format or model.json
// Save as manifest.cdm.json
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// Save as a model.json
// await manifestResolved.SaveAsAsync("model.json", true);
}
public async Task TestResolvedAttributeLimit()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestResolvedAttributeLimit");
CdmEntityDefinition mainEntity = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/mainEntity.cdm.json/mainEntity");
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = mainEntity.InDocument, Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
})
};
// if attribute limit is reached, entity should be null
resOpt.ResolvedAttributeLimit = 4;
var resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_zeroAtts", resOpt);
Assert.IsNull(resEnt);
// when the attribute limit is set to null, there should not be a limit on the possible number of attributes
resOpt.ResolvedAttributeLimit = null;
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var ras = mainEntity.FetchResolvedAttributes(resOpt);
resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_normalized_referenceOnly", resOpt);
// there are 5 total attributes
Assert.AreEqual(ras.ResolvedAttributeCount, 5);
Assert.AreEqual(ras.Set.Count, 5);
Assert.AreEqual(mainEntity.Attributes.Count, 3);
// there are 2 attributes grouped in an entity attribute
// and 2 attributes grouped in an attribute group
Assert.AreEqual(((mainEntity.Attributes[2] as CdmAttributeGroupReference).ExplicitReference as CdmAttributeGroupDefinition).Members.Count, 2);
// using the default limit number
resOpt = new ResolveOptions {
WrtDoc = mainEntity.InDocument, Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
})
};
ras = mainEntity.FetchResolvedAttributes(resOpt);
resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_normalized_referenceOnly", resOpt);
// there are 5 total attributes
Assert.AreEqual(ras.ResolvedAttributeCount, 5);
Assert.AreEqual(ras.Set.Count, 5);
Assert.AreEqual(mainEntity.Attributes.Count, 3);
// there are 2 attributes grouped in an entity attribute
// and 2 attributes grouped in an attribute group
Assert.AreEqual(((mainEntity.Attributes[2] as CdmAttributeGroupReference).ExplicitReference as CdmAttributeGroupDefinition).Members.Count, 2);
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
ras = mainEntity.FetchResolvedAttributes(resOpt);
resEnt = await mainEntity.CreateResolvedEntityAsync($"{mainEntity.EntityName}_normalized_structured", resOpt);
// there are 5 total attributes
Assert.AreEqual(ras.ResolvedAttributeCount, 5);
// the attribute count is different because one attribute is a group that contains two different attributes
Assert.AreEqual(ras.Set.Count, 4);
Assert.AreEqual(mainEntity.Attributes.Count, 3);
// again there are 2 attributes grouped in an entity attribute
// and 2 attributes grouped in an attribute group
Assert.AreEqual(((mainEntity.Attributes[2] as CdmAttributeGroupReference).ExplicitReference as CdmAttributeGroupDefinition).Members.Count, 2);
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../";
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "2-create-manifest"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add each declaration, this example is about medical appointments and care plans
manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address");
manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan");
manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept");
manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact");
manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device");
manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment");
manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter");
manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare");
manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", "");
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add("cdm:/foundations.cdm.json");
Console.WriteLine("Save the docs");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// Define the location of the partition, relative to the manifest
var location = $"local:/{entDef.EntityName}/partition-data.csv";
part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved);
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
// Get the actual location of the partition file from the corpus
string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location);
// Make a fake file with nothing but header for columns
string header = "";
foreach (CdmTypeAttributeDefinition att in entDef.Attributes)
{
if (header != "")
{
header += ",";
}
header += att.Name;
}
Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{ entDef.EntityName}"));
File.WriteAllText(partPath, header);
}
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
}
public async Task TestWithoutNesting()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId);
// Test fetching an object from invalid namespace results in at least one error message in the recorder
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar");
Assert.IsNotNull(corpus.Ctx.Events, "Ctx.Events should not be null");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded when fetching object with incorrect path");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test fetching a good object, this should leave event recorder empty
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count == 0, "There should have been no events recorded when fetching object with correct path");
// Test saving a manifest to invalid namespace results in at least one error message in the recorder
var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy");
await manifest.SaveAsAsync("foo:/bar", true);
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder
await manifest.CreateResolvedManifestAsync("new dummy", null);
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test resolving an entity without WRT doc, this should yield at least one error message in the recorder
var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2");
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder
await manifest.FileStatusCheckAsync();
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Repeat the same test but with status level 'None', no events should be recorded
corpus.Ctx.ReportAtLevel = CdmStatusLevel.None;
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count == 0, "There should have been no events recorded when fetching object with correct path");
}
public async Task TestUpdateRelationships()
{
var expectedRels = JToken.Parse(TestHelper.GetExpectedOutputFileContent(testsSubpath, "TestUpdateRelationships", "expectedRels.json")).ToObject <List <E2ERelationship> >();
string tempFromFilePath = "fromEntTemp.cdm.json";
string tempFromEntityPath = "local:/fromEntTemp.cdm.json/fromEnt";
string tempToEntityPath = "local:/toEnt.cdm.json/toEnt";
// Initialize corpus and entity files
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestUpdateRelationships");
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/main.manifest.cdm.json");
CdmManifestDefinition manifestNoToEnt = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/mainNoToEnt.manifest.cdm.json");
CdmEntityDefinition fromEnt = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/fromEnt.cdm.json/fromEnt");
await fromEnt.InDocument.SaveAsAsync(tempFromFilePath, options : new CopyOptions()
{
IsTopLevelDocument = false
});
async Task reloadFromEntity()
{
await fromEnt.InDocument.SaveAsAsync(tempFromFilePath, options : new CopyOptions()
{
IsTopLevelDocument = false
});
// fetch again to reset the cache
await corpus.FetchObjectAsync <CdmEntityDefinition>(tempFromEntityPath, null, false, true);
}
try
{
// 1. test when entity attribute is removed
await corpus.CalculateEntityGraphAsync(manifest);
await manifest.PopulateManifestRelationshipsAsync();
// check that the relationship has been created correctly
VerifyRelationships(manifest, expectedRels);
// now remove the entity attribute, which removes the relationship
CdmAttributeItem removedAttribute = fromEnt.Attributes[0];
fromEnt.Attributes.RemoveAt(0);
await reloadFromEntity();
await corpus.CalculateEntityGraphAsync(manifest);
await manifest.PopulateManifestRelationshipsAsync();
// check that the relationship has been removed
VerifyRelationships(manifest, new List <E2ERelationship>());
// 2. test when the to entity is removed
// restore the entity to the original state
fromEnt.Attributes.Add(removedAttribute);
await reloadFromEntity();
await corpus.CalculateEntityGraphAsync(manifest);
await manifest.PopulateManifestRelationshipsAsync();
// check that the relationship has been created correctly
VerifyRelationships(manifest, expectedRels);
// remove the to entity
fromEnt.Attributes.RemoveAt(0);
await reloadFromEntity();
// fetch again to reset the cache
await corpus.FetchObjectAsync <CdmEntityDefinition>(tempToEntityPath, null, false, true);
await corpus.CalculateEntityGraphAsync(manifestNoToEnt);
await manifestNoToEnt.PopulateManifestRelationshipsAsync();
// check that the relationship has been removed
VerifyRelationships(manifestNoToEnt, new List <E2ERelationship>());
}
finally
{
// clean up created files created
string fromPath = corpus.Storage.CorpusPathToAdapterPath($"local:/{tempFromFilePath}");
File.Delete(fromPath);
}
}
/// <summary>
/// This method runs the tests with a set expected attributes & attribute context values and validated the actual result
/// </summary>
/// <param name="testName"></param>
/// <param name="sourceEntityName"></param>
/// <param name="expectedContext_*">expected attribute context object - for each resolution option combination</param>
/// <param name="expected_*">expected attribute object - for each resolution option combination</param>
/// <returns></returns>
protected static async Task RunTestWithValues(
string testName,
string sourceEntityName,
AttributeContextExpectedValue expectedContext_default,
AttributeContextExpectedValue expectedContext_normalized,
AttributeContextExpectedValue expectedContext_referenceOnly,
AttributeContextExpectedValue expectedContext_structured,
AttributeContextExpectedValue expectedContext_normalized_structured,
AttributeContextExpectedValue expectedContext_referenceOnly_normalized,
AttributeContextExpectedValue expectedContext_referenceOnly_structured,
AttributeContextExpectedValue expectedContext_referenceOnly_normalized_structured,
List <AttributeExpectedValue> expected_default,
List <AttributeExpectedValue> expected_normalized,
List <AttributeExpectedValue> expected_referenceOnly,
List <AttributeExpectedValue> expected_structured,
List <AttributeExpectedValue> expected_normalized_structured,
List <AttributeExpectedValue> expected_referenceOnly_normalized,
List <AttributeExpectedValue> expected_referenceOnly_structured,
List <AttributeExpectedValue> expected_referenceOnly_normalized_structured,
bool updateExpectedOutput = false
)
{
try
{
string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName);
string testActualPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName);
string testExpectedPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName);
string corpusPath = testInputPath.Substring(0, testInputPath.Length - "/Input".Length);
testActualPath = Path.GetFullPath(testActualPath);
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(corpusPath));
corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath));
corpus.Storage.DefaultNamespace = "local";
string outFolderPath = corpus.Storage.AdapterPathToCorpusPath(testActualPath) + "/"; // interesting 'bug'
CdmFolderDefinition outFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>(outFolderPath);
CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/Input/{sourceEntityName}.cdm.json/{sourceEntityName}");
Assert.IsTrue(srcEntityDef != null);
var resOpt = new ResolveOptions
{
WrtDoc = srcEntityDef.InDocument,
Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
})
};
CdmEntityDefinition resolvedEntityDef = null;
string outputEntityName = string.Empty;
string outputEntityFileName = string.Empty;
string entityFileName = string.Empty;
if (expectedContext_default != null && expected_default != null)
{
entityFileName = "d";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_normalized != null && expected_normalized != null)
{
entityFileName = "n";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_referenceOnly != null && expected_referenceOnly != null)
{
entityFileName = "ro";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_structured != null && expected_structured != null)
{
entityFileName = "s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_normalized_structured != null && expected_normalized_structured != null)
{
entityFileName = "n_s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_referenceOnly_normalized != null && expected_referenceOnly_normalized != null)
{
entityFileName = "ro_n";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_referenceOnly_structured != null && expected_referenceOnly_structured != null)
{
entityFileName = "ro_s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
if (expectedContext_referenceOnly_normalized_structured != null && expected_referenceOnly_normalized_structured != null)
{
entityFileName = "ro_n_s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized", "structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef, updateExpectedOutput);
}
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public async Task TestWithoutNesting()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId);
// Test fetching an object from invalid namespace results in at least one error message in the recorder
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNamespaceNotRegistered, true);
// Test fetching a good object, this should leave event recorder empty
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json");
TestNoLogsState(corpus);
// Test saving a manifest to invalid namespace results in at least one error message in the recorder
var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy");
await manifest.SaveAsAsync("foo:/bar", true);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrValdnMissingDoc, true);
// Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder
await manifest.CreateResolvedManifestAsync("new dummy", null);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrResolveManifestFailed, true);
// Test resolving an entity without WRT doc, this should yield at least one error message in the recorder
var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2");
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrDocWrtDocNotfound, true);
// Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder
await manifest.FileStatusCheckAsync();
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace, true);
// Repeat the same test but with status level 'None', no events should be recorded
corpus.Ctx.ReportAtLevel = CdmStatusLevel.None;
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
TestNoLogsState(corpus);
// Test checking file status on a data partition
// We're at log level 'Progress', so we get the EnterScope/LeaveScope messages too
corpus.Ctx.ReportAtLevel = CdmStatusLevel.Progress;
var part = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "part");
await part.FileStatusCheckAsync();
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath, true);
// Test checking file status on a data partition pattern
var refDoc = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "RefEntDoc");
var partPattern = corpus.MakeObject <CdmDataPartitionPatternDefinition>(CdmObjectType.DataPartitionPatternDef, "partPattern");
partPattern.InDocument = refDoc;
await partPattern.FileStatusCheckAsync();
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullCorpusPath, true);
// Test calculating relationships - no errors/warnings
await corpus.CalculateEntityGraphAsync(manifest);
TestBasicLogsState(corpus);
// Test populating relationships in manifest - no errors/warnings
await manifest.PopulateManifestRelationshipsAsync();
TestBasicLogsState(corpus);
// Test filtering code logic
corpus.Ctx.SuppressedLogCodes.Add(CdmLogCode.ErrPathNullObjectPath);
var part2 = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "part2");
await part2.FileStatusCheckAsync();
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath, false);
}
public async Task TestEntityAttribute()
{
string testName = "TestEntityAttribute";
string entityName = "ThreeMusketeers";
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
foreach (List <string> resOpt in resOptsCombinations)
{
await ProjectionTestUtils.LoadEntityForResolutionOptionAndSave(corpus, testName, testsSubpath, entityName, resOpt);
}
CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}");
CdmEntityDefinition nonStructuredResolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> {
});
// Original set of attributes: ["name", "age", "address"]
// in non-structured form
// addArtifactAttribute : { "key" , "insertAtTop": true }
// Expand 1...3;
// renameAttributes = { {a}_{o}_key, apply to "key" }
// renameAttributes = { {a}_{m}_{o}_value, apply to "name", "age", "address" }
// alterTraits = { indicates.expansionInfo.mapKey(expansionName: "{a}", ordinal: "{o}") , apply to "key" , "argumentsContainWildcards" : true }
// alterTraits = { has.expansionInfo.mapValue(expansionName: "{a}", ordinal: "{o}", memberAttribute: "{mo}") , apply to "name", "age", "address" , "argumentsContainWildcards" : true }
// addArtifactAttribute : "personCount"
// alterTraits = { indicates.expansionInfo.count(expansionName: "{a}") , apply to "personCount" , "argumentsContainWildcards" : true }
Assert.AreEqual(13, nonStructuredResolvedEntity.Attributes.Count);
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[0] as CdmTypeAttributeDefinition, "key_1_key", 1, "ThreePeople", isKey: true);
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[1] as CdmTypeAttributeDefinition, "ThreePeople_name_1_value", 1, "ThreePeople", "name");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[2] as CdmTypeAttributeDefinition, "ThreePeople_age_1_value", 1, "ThreePeople", "age");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[3] as CdmTypeAttributeDefinition, "ThreePeople_address_1_value", 1, "ThreePeople", "address");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[4] as CdmTypeAttributeDefinition, "key_2_key", 2, "ThreePeople", isKey: true);
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[5] as CdmTypeAttributeDefinition, "ThreePeople_name_2_value", 2, "ThreePeople", "name");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[6] as CdmTypeAttributeDefinition, "ThreePeople_age_2_value", 2, "ThreePeople", "age");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[7] as CdmTypeAttributeDefinition, "ThreePeople_address_2_value", 2, "ThreePeople", "address");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[8] as CdmTypeAttributeDefinition, "key_3_key", 3, "ThreePeople", isKey: true);
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[9] as CdmTypeAttributeDefinition, "ThreePeople_name_3_value", 3, "ThreePeople", "name");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[10] as CdmTypeAttributeDefinition, "ThreePeople_age_3_value", 3, "ThreePeople", "age");
ValidateAttributeTrait(nonStructuredResolvedEntity.Attributes[11] as CdmTypeAttributeDefinition, "ThreePeople_address_3_value", 3, "ThreePeople", "address");
Assert.AreEqual("personCount", (nonStructuredResolvedEntity.Attributes[12] as CdmTypeAttributeDefinition).Name);
Assert.IsNotNull(nonStructuredResolvedEntity.Attributes[12].AppliedTraits.Item("indicates.expansionInfo.count"));
Assert.AreEqual("ThreePeople", (nonStructuredResolvedEntity.Attributes[12].AppliedTraits.Item("indicates.expansionInfo.count") as CdmTraitReference).Arguments[0].Value);
// Original set of attributes: ["name", "age", "address"]
// in structured form
// addAttributeGroup: favorite people
// alterTraits = { is.dataFormat.mapValue }
// addArtifactAttribute : { "favorite People Key" (with trait "is.dataFormat.mapKey") , "insertAtTop": true }
// addAttributeGroup: favorite People Group
// alterTraits = { is.dataFormat.map }
CdmEntityDefinition structuredResolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> {
"structured"
});
Assert.AreEqual(1, structuredResolvedEntity.Attributes.Count);
CdmAttributeGroupDefinition attGroupDefinition = ProjectionTestUtils.ValidateAttributeGroup(structuredResolvedEntity.Attributes, "favorite People Group");
Assert.IsNotNull(attGroupDefinition.ExhibitsTraits.Item("is.dataFormat.map"));
Assert.AreEqual("favorite People Key", (attGroupDefinition.Members[0] as CdmTypeAttributeDefinition).Name);
Assert.IsNotNull(attGroupDefinition.Members[0].AppliedTraits.Item("is.dataFormat.mapKey"));
Assert.AreEqual(CdmObjectType.AttributeGroupRef, attGroupDefinition.Members[1].ObjectType);
CdmAttributeGroupReference innerAttGroupRef = attGroupDefinition.Members[1] as CdmAttributeGroupReference;
Assert.IsNotNull(innerAttGroupRef.ExplicitReference);
CdmAttributeGroupDefinition innerAttGroupDefinition = innerAttGroupRef.ExplicitReference as CdmAttributeGroupDefinition;
Assert.AreEqual("favorite people", innerAttGroupDefinition.AttributeGroupName);
Assert.IsNotNull(innerAttGroupDefinition.ExhibitsTraits.Item("is.dataFormat.mapValue"));
}
static async Task Main()
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
var pathFromExeToExampleRoot = "../../../../../../";
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "6-create-net-new-entities"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
// This sample is going to simulate the steps a tool would follow in order to create a new manifest document
// in some user storage folder with two types of entities - new net entity and entity extended from some public standards,
// Note: If we want to create a relationship between a new custom entity to an existing entity which is loaded from some public standards,
// we need to create an entity extending from the existing enbtity and add a relationship to the attribute of the extended entity,
// since we can't modify attributes from an 'abstract' schemaDefintion in the public standards
// This sample also creates a relationship between a net new entity to an existing entity, and a relationship between two net new entities
//
// the steps are:
// 1. create a temporary 'manifest' object at the root of the corpus
// 2. create two net new entities without extending any exsiting entity, create a relationship from one to the other, and add them to the manifest
// 3. create one entity which extends from the public standards, create a relationship from it to an net new entity, and add the entity to the manifest
// 4. make a 'resolved' version of each entity doc in our local folder. (please check out the second example 2-create-manifest for more details)
// CreateResolvedManifestAsync on our starting manifest. this will resolve everything and find all of the relationships between entities for us
// 5. save the new document(s)
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local documents in the corpus
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json");
// Create two entities from scratch, and add some attributes, traits, properties, and relationships in between
Console.WriteLine("Create net new entities");
// Create the simplest entity - CustomPerson
// Create the entity definition instance
var personEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false);
// Add type attributes to the entity instance
var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId");
personEntity.Attributes.Add(personAttributeId);
var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "string");
personEntity.Attributes.Add(personAttributeName);
// Add properties to the entity instance
personEntity.DisplayName = CustomPersonEntityName;
personEntity.Version = "0.0.1";
personEntity.Description = "This is a custom entity created for the sample.";
// Create the document which contains the entity
var personEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
personEntityDoc.Imports.Add(FoundationJsonPath);
personEntityDoc.Definitions.Add(personEntity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(personEntity);
// Create an entity - CustomAccount which has a relationship with the entity CustomPerson
// Create the entity definition instance
var accountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false);
// Add type attributes to the entity instance
var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId");
accountEntity.Attributes.Add(accountAttributeId);
var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "string");
accountEntity.Attributes.Add(accountAttributeName);
// Add properties to the entity instance
accountEntity.DisplayName = CustomAccountEntityName;
accountEntity.Version = "0.0.1";
accountEntity.Description = "This is a custom entity created for the sample.";
// In this sample, every account has one person who owns the account
// the relationship is actually an entity attribute
var attributeExplanation = "The owner of the account, which is a person.";
// You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
// more details of how to use resolution guidance to customize your data
var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation);
accountEntity.Attributes.Add(accountOwnerAttribute);
// Create the document which contains the entity
var accountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
accountEntityDoc.Imports.Add(FoundationJsonPath);
// the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits,
// the import to the entity reference CustomPerson's doc is required
accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json");
accountEntityDoc.Definitions.Add(accountEntity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(accountEntity);
// Create an entity which extends "Account" from the standard, it contains everything that "Account" has
var extendedStandardAccountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false);
// This function with 'true' will make a simple reference to the base
extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "Account", true);
var attrExplanation = "This is a simple custom account for this sample.";
// Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount
// You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
// more details of how to use resolution guidance to customize your data
var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation);
extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute);
var extendedStandardAccountntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
extendedStandardAccountntityDoc.Imports.Add(FoundationJsonPath);
// The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required
// it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required
extendedStandardAccountntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json");
extendedStandardAccountntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json");
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(extendedStandardAccountntityDoc, extendedStandardAccountntityDoc.Name);
extendedStandardAccountntityDoc.Definitions.Add(extendedStandardAccountEntity);
// Add the entity to the manifest
manifestAbstract.Entities.Add(extendedStandardAccountEntity);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
Console.WriteLine("Save the docs");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// We have existing partition files for the custom entities, so we need to make the partition point to the file location
part.Location = $"local:/{entDef.EntityName}/partition-data.csv";
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
}
// We can save the docs as manifest.cdm.json format or model.json
// Save as manifest.cdm.json
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// Save as a model.json
// await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.model.json", true);
}
public async Task TestProjectionUsingObjectModel()
{
CdmCorpusDefinition corpus = ProjectionTestUtils.GetLocalCorpus(testsSubpath, "TestProjectionUsingObjectModel");
corpus.Storage.Mount("local", new LocalAdapter(TestHelper.GetActualOutputFolderPath(testsSubpath, "TestProjectionUsingObjectModel")));
CdmFolderDefinition localRoot = corpus.Storage.FetchRootFolder("local");
CdmManifestDefinition manifestDefault = CreateDefaultManifest(corpus, localRoot);
CdmEntityDefinition entityTestSource = CreateEntityTestSource(corpus, manifestDefault, localRoot);
CdmEntityDefinition entityTestEntityProjection = CreateEntityTestEntityProjection(corpus, manifestDefault, localRoot);
CdmEntityDefinition entityTestEntityNestedProjection = CreateEntityTestEntityNestedProjection(corpus, manifestDefault, localRoot);
CdmEntityDefinition entityTestEntityAttributeProjection = CreateEntityTestEntityAttributeProjection(corpus, manifestDefault, localRoot);
CdmEntityDefinition entityTestOperationCollection = CreateEntityTestOperationCollection(corpus, manifestDefault, localRoot);
// Save manifest and entities
await manifestDefault.SaveAsAsync($"{manifestDefault.ManifestName}.manifest.cdm.json", saveReferenced : true);
string expected = "TestSource";
CdmObjectType expectedType = CdmObjectType.ProjectionDef;
string actual = null;
CdmObjectType actualType = CdmObjectType.Error;
// Try to read back the newly persisted manifest and projection based entities
CdmManifestDefinition manifestReadBack = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{manifestDefault.ManifestName}.manifest.cdm.json");
Assert.AreEqual(5, manifestReadBack.Entities.Count);
Assert.AreEqual(entityTestSource.EntityName, manifestReadBack.Entities[0].EntityName);
Assert.AreEqual(entityTestEntityProjection.EntityName, manifestReadBack.Entities[1].EntityName);
Assert.AreEqual(entityTestEntityNestedProjection.EntityName, manifestReadBack.Entities[2].EntityName);
Assert.AreEqual(entityTestEntityAttributeProjection.EntityName, manifestReadBack.Entities[3].EntityName);
// Read back the newly persisted manifest and projection based entity TestEntityProjection and validate
CdmEntityDefinition entityTestEntityProjectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestEntityProjection.EntityName}.cdm.json/{entityTestEntityProjection.EntityName}", manifestReadBack);
Assert.IsNotNull(entityTestEntityProjectionReadBack);
actual = ((CdmEntityReference)((CdmProjection)entityTestEntityProjectionReadBack.ExtendsEntity.ExplicitReference).Source).NamedReference;
actualType = ((CdmProjection)entityTestEntityProjectionReadBack.ExtendsEntity.ExplicitReference).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(expectedType, actualType);
// Read back the newly persisted manifest and projection based entity TestEntityNestedProjection and validate
CdmEntityDefinition entityTestEntityNestedProjectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestEntityNestedProjection.EntityName}.cdm.json/{entityTestEntityNestedProjection.EntityName}", manifestReadBack);
Assert.IsNotNull(entityTestEntityNestedProjectionReadBack);
actual = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entityTestEntityNestedProjectionReadBack.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).Source.NamedReference;
actualType = ((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmProjection)((CdmEntityReference)entityTestEntityNestedProjectionReadBack.ExtendsEntity).ExplicitReference).Source).ExplicitReference).Source).ExplicitReference).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(expectedType, actualType);
// Read back the newly persisted manifest and projection based entity TestEntityAttributeProjection and validate
CdmEntityDefinition entityTestEntityAttributeProjectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestEntityAttributeProjection.EntityName}.cdm.json/{entityTestEntityAttributeProjection.EntityName}", manifestReadBack);
Assert.IsNotNull(entityTestEntityAttributeProjectionReadBack);
actual = ((CdmEntityReference)((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entityTestEntityAttributeProjectionReadBack.Attributes[0]).Entity).ExplicitReference).Source).NamedReference;
actualType = ((CdmProjection)((CdmEntityReference)((CdmEntityAttributeDefinition)entityTestEntityAttributeProjectionReadBack.Attributes[0]).Entity).ExplicitReference).ObjectType;
Assert.AreEqual(expected, actual);
Assert.AreEqual(expectedType, actualType);
// Read back operations collections and validate
CdmEntityDefinition entityTestOperationCollectionReadBack = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityTestOperationCollection.EntityName}.cdm.json/{entityTestOperationCollection.EntityName}", manifestReadBack);
Assert.IsNotNull(entityTestOperationCollectionReadBack);
int actualOperationCount = ((CdmProjection)entityTestOperationCollectionReadBack.ExtendsEntity.ExplicitReference).Operations.Count;
Assert.AreEqual(9, actualOperationCount);
CdmOperationCollection operations = ((CdmProjection)entityTestOperationCollectionReadBack.ExtendsEntity.ExplicitReference).Operations;
Assert.AreEqual(CdmOperationType.AddCountAttribute, operations[0].Type);
Assert.AreEqual(CdmOperationType.AddSupportingAttribute, operations[1].Type);
Assert.AreEqual(CdmOperationType.AddTypeAttribute, operations[2].Type);
Assert.AreEqual(CdmOperationType.ExcludeAttributes, operations[3].Type);
Assert.AreEqual(CdmOperationType.ArrayExpansion, operations[4].Type);
Assert.AreEqual(CdmOperationType.CombineAttributes, operations[5].Type);
Assert.AreEqual(CdmOperationType.RenameAttributes, operations[6].Type);
Assert.AreEqual(CdmOperationType.ReplaceAsForeignKey, operations[7].Type);
Assert.AreEqual(CdmOperationType.IncludeAttributes, operations[8].Type);
}