private async Task ReadLocalSaveAdls(CdmCorpusDefinition cdmCorpus)
{
// ------------------------------------------------------------------------------------------------------------
// Load a model.json file from local FS
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/model.json");
// ------------------------------------------------------------------------------------------------------------
// Explore entities and partitions defined in the model
Console.WriteLine("Listing entity declarations:");
foreach (CdmEntityDeclarationDefinition decl in manifest.Entities)
{
Console.WriteLine(" " + decl.EntityName);
// TODO: This can be rewritten in a different way since data partition gives null for referenced entities, suggestions are welcome.
if (decl.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
foreach (CdmDataPartitionDefinition dataPart in decl.DataPartitions)
{
Console.WriteLine(" " + dataPart.Location);
}
}
}
// ------------------------------------------------------------------------------------------------------------
// Make changes to the model
// Create a new document where the new entity's definition will be stored
var newEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "NewEntity.cdm.json", false);
newEntityDoc.Imports.Add("cdm:/foundations.cdm.json");
cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(newEntityDoc);
var newEntity = newEntityDoc.Definitions.Add(CdmObjectType.EntityDef, "NewEntity") as CdmEntityDefinition;
// Define new string attribute and add it to the entity definition
var newAttribute = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "NewAttribute", false);
newAttribute.DataFormat = CdmDataFormat.String;
newEntity.Attributes.Add(newAttribute);
// Call will create EntityDeclarationDefinition based on entity definition and add it to manifest.Entities
var newEntityDecl = manifest.Entities.Add(newEntity);
// Define a partition and add it to the local declaration
var newPartition = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "NewPartition", false);
newPartition.Location = "adls:/NewPartition.csv";
newEntityDecl.DataPartitions.Add(newPartition);
// ------------------------------------------------------------------------------------------------------------
// Save the file to ADLSg2 - we achieve that by adding the manifest to the root folder of
// the ADLS file-system and performing a save on the manifest
CdmFolderDefinition adlsFolder = cdmCorpus.Storage.FetchRootFolder("adls");
adlsFolder.Documents.Add(manifest);
await manifest.SaveAsAsync("model.json", true);
}
public async Task TestSaveProjection()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestSaveProjection");
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json");
CdmEntityDefinition entitySales = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/Sales.cdm.json/Sales", manifest);
Assert.IsNotNull(entitySales);
CdmFolderDefinition actualRoot = corpus.Storage.FetchRootFolder("output");
Assert.IsNotNull(actualRoot);
actualRoot.Documents.Add(entitySales.InDocument, "Persisted_Sales.cdm.json");
await actualRoot.Documents[0].SaveAsAsync("output:/Persisted_Sales.cdm.json");
CdmEntityDefinition entityActual = await corpus.FetchObjectAsync <CdmEntityDefinition>($"output:/Persisted_Sales.cdm.json/Sales", manifest);
Assert.IsNotNull(entityActual);
Entity entityContentActual = PersistenceLayer.ToData <CdmEntityDefinition, Entity>(entityActual, null, null, "CdmFolder");
Assert.IsNotNull(entityContentActual);
Assert.IsNotNull(entityContentActual.HasAttributes);
Assert.IsTrue(entityContentActual.HasAttributes.Count == 1);
Assert.IsFalse(entityContentActual.HasAttributes[0].ToString().Contains(@"""entityReference"""));
}
public void TestDocumentCollectionAddRange()
{
var manifest = CdmCollectionHelperFunctions.GenerateManifest();
var folder = new CdmFolderDefinition(manifest.Ctx, "Folder");
folder.Corpus = manifest.Ctx.Corpus;
folder.FolderPath = "FolderPath/";
folder.Namespace = "Namespace";
Assert.AreEqual(0, folder.Documents.Count);
var document = new CdmDocumentDefinition(manifest.Ctx, "DocumentName");
var document2 = new CdmDocumentDefinition(manifest.Ctx, "DocumentName2");
var documentList = new List <CdmDocumentDefinition> {
document, document2
};
folder.Documents.AddRange(documentList);
Assert.AreEqual(2, folder.Documents.Count);
Assert.AreEqual(document, folder.Documents[0]);
Assert.AreEqual(document2, folder.Documents[1]);
Assert.AreEqual("DocumentName", document.Name);
Assert.AreEqual("FolderPath/", document.FolderPath);
Assert.AreEqual(folder, document.Owner);
Assert.AreEqual("Namespace", document.Namespace);
Assert.IsTrue(document.NeedsIndexing);
Assert.AreEqual("DocumentName2", document2.Name);
Assert.AreEqual("FolderPath/", document2.FolderPath);
Assert.AreEqual(folder, document2.Owner);
Assert.AreEqual("Namespace", document2.Namespace);
Assert.IsTrue(document2.NeedsIndexing);
}
public void TestFolderCollectionAddRange()
{
var manifest = CdmCollectionHelperFunctions.GenerateManifest("C:\\Root\\Path");
var parentFolder = new CdmFolderDefinition(manifest.Ctx, "ParentFolder");
parentFolder.Namespace = "TheNamespace";
parentFolder.FolderPath = "ParentFolderPath/";
CdmFolderCollection childFolders = parentFolder.ChildFolders;
var childFolder = new CdmFolderDefinition(manifest.Ctx, "ChildFolder1");
var childFolder2 = new CdmFolderDefinition(manifest.Ctx, "ChildFolder2");
var childList = new List <CdmFolderDefinition> {
childFolder, childFolder2
};
Assert.AreEqual(0, childFolders.Count);
childFolders.AddRange(childList);
Assert.AreEqual(2, childFolders.Count);
Assert.AreEqual(childFolder, childFolders[0]);
Assert.AreEqual(manifest.Ctx, childFolder.Ctx);
Assert.AreEqual("ChildFolder1", childFolder.Name);
Assert.AreEqual(parentFolder, childFolder.Owner);
Assert.AreEqual("TheNamespace", childFolder.Namespace);
Assert.AreEqual(parentFolder.FolderPath + childFolder.Name + "/", childFolder.FolderPath);
Assert.AreEqual(childFolder2, childFolders[1]);
Assert.AreEqual("ChildFolder2", childFolder2.Name);
Assert.AreEqual(parentFolder, childFolder2.Owner);
Assert.AreEqual("TheNamespace", childFolder2.Namespace);
Assert.AreEqual(parentFolder.FolderPath + childFolder2.Name + "/", childFolder2.FolderPath);
}
public void TestCdmDefinitionCollectionAdd()
{
var document = CdmCollectionHelperFunctions.GenerateManifest("C:\\Nothing");
document.IsDirty = false;
var attribute = new CdmAttributeContext(document.Ctx, "the attribute");
var folder = new CdmFolderDefinition(document.Ctx, "The folder");
var trait = new CdmTraitDefinition(document.Ctx, "The trait");
var addedAttribute = document.Definitions.Add(attribute);
var addedFolder = document.Definitions.Add(folder);
var addedTrait = document.Definitions.Add(trait);
Assert.AreEqual(true, document.IsDirty);
Assert.AreEqual(3, document.Definitions.Count);
Assert.AreEqual(attribute, addedAttribute);
Assert.AreEqual(folder, addedFolder);
Assert.AreEqual(trait, addedTrait);
Assert.AreEqual(attribute, document.Definitions[0]);
Assert.AreEqual(folder, document.Definitions[1]);
Assert.AreEqual(trait, document.Definitions[2]);
Assert.AreEqual(document, attribute.DocCreatedIn);
Assert.AreEqual(document, folder.DocCreatedIn);
Assert.AreEqual(document, trait.DocCreatedIn);
Assert.AreEqual(document, attribute.Owner);
Assert.AreEqual(document, folder.Owner);
Assert.AreEqual(document, trait.Owner);
}
public async Task TestEntityProjUsingObjectModel()
{
CdmCorpusDefinition corpus = ProjectionTestUtils.GetLocalCorpus(testsSubpath, "TestEntityProjUsingObjectModel");
corpus.Storage.Mount("local", new LocalAdapter(TestHelper.GetActualOutputFolderPath(testsSubpath, "TestEntityProjUsingObjectModel")));
CdmFolderDefinition localRoot = corpus.Storage.FetchRootFolder("local");
// Create an entity
CdmEntityDefinition entity = ProjectionTestUtils.CreateEntity(corpus, localRoot);
// Create a projection
CdmProjection projection = ProjectionTestUtils.CreateProjection(corpus, localRoot);
// Create an ArrayExpansion operation
CdmOperationArrayExpansion arrayExpansionOp = corpus.MakeObject <CdmOperationArrayExpansion>(CdmObjectType.OperationArrayExpansionDef);
arrayExpansionOp.StartOrdinal = 1;
arrayExpansionOp.EndOrdinal = 2;
projection.Operations.Add(arrayExpansionOp);
// Create an entity reference to hold this projection
CdmEntityReference projectionEntityRef = corpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, null);
projectionEntityRef.ExplicitReference = projection;
// Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity
CdmProjection projection2 = corpus.MakeObject <CdmProjection>(CdmObjectType.ProjectionDef);
projection2.Source = projectionEntityRef;
// Create a RenameAttributes operation
CdmOperationRenameAttributes renameAttrsOp = corpus.MakeObject <CdmOperationRenameAttributes>(CdmObjectType.OperationRenameAttributesDef);
renameAttrsOp.RenameFormat = "{m}{o}";
projection2.Operations.Add(renameAttrsOp);
// Create an entity reference to hold this projection
CdmEntityReference projectionEntityRef2 = corpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, null);
projectionEntityRef2.ExplicitReference = projection2;
// Set the entity's ExtendEntity to be the projection
entity.ExtendsEntity = projectionEntityRef2;
// Resolve the entity
CdmEntityDefinition resolvedEntity = await entity.CreateResolvedEntityAsync($"Resolved_{entity.EntityName}.cdm.json", null, localRoot);
// Verify correctness of the resolved attributes after running the projections
// Original set of attributes: ["id", "name", "value", "date"]
// Expand 1...2, renameFormat = {m}{o}
Assert.AreEqual(8, resolvedEntity.Attributes.Count);
Assert.AreEqual("id1", (resolvedEntity.Attributes[0] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("name1", (resolvedEntity.Attributes[1] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("value1", (resolvedEntity.Attributes[2] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("date1", (resolvedEntity.Attributes[3] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("id2", (resolvedEntity.Attributes[4] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("name2", (resolvedEntity.Attributes[5] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("value2", (resolvedEntity.Attributes[6] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("date2", (resolvedEntity.Attributes[7] as CdmTypeAttributeDefinition).Name);
}
/// <summary>
/// Resolves an entity
/// </summary>
/// <param name="corpus">The corpus</param>
/// <param name="inputEntity">The entity to resolve</param>
/// <param name="resolutionOptions">The resolution options</param>
/// <param name="addResOptToName">Whether to add the resolution options as part of the resolved entity name</param>
public static async Task <CdmEntityDefinition> GetResolvedEntity(CdmCorpusDefinition corpus, CdmEntityDefinition inputEntity, List <string> resolutionOptions, bool addResOptToName = false)
{
HashSet <string> roHashSet = new HashSet <string>(resolutionOptions);
string resolvedEntityName = "";
if (addResOptToName)
{
string fileNameSuffix = GetResolutionOptionNameSuffix(resolutionOptions);
resolvedEntityName = $"Resolved_{inputEntity.EntityName}{fileNameSuffix}";
}
else
{
resolvedEntityName = $"Resolved_{inputEntity.EntityName}";
}
ResolveOptions ro = new ResolveOptions(inputEntity.InDocument)
{
Directives = new AttributeResolutionDirectiveSet(roHashSet)
};
CdmFolderDefinition resolvedFolder = corpus.Storage.FetchRootFolder("output");
CdmEntityDefinition resolvedEntity = await inputEntity.CreateResolvedEntityAsync(resolvedEntityName, ro, resolvedFolder);
await resolvedEntity.InDocument.SaveAsAsync($"{resolvedEntityName}.cdm.json", saveReferenced : false);
return(resolvedEntity);
}
public void TestDocumentCollectionAdd()
{
var manifest = CdmCollectionHelperFunctions.GenerateManifest();
var folder = new CdmFolderDefinition(manifest.Ctx, "Folder");
folder.Corpus = manifest.Ctx.Corpus;
folder.FolderPath = "FolderPath/";
folder.Namespace = "Namespace";
var document = new CdmDocumentDefinition(manifest.Ctx, "DocumentName");
Assert.AreEqual(0, folder.Documents.Count);
var addedDocument = folder.Documents.Add(document);
Assert.AreEqual(1, folder.Documents.Count);
Assert.AreEqual(document, folder.Documents[0]);
Assert.AreEqual(document, addedDocument);
Assert.AreEqual("FolderPath/", document.FolderPath);
Assert.AreEqual(folder, document.Owner);
Assert.AreEqual("Namespace", document.Namespace);
Assert.IsTrue(document.NeedsIndexing);
var doc = folder.Documents.Add(document);
Assert.IsNull(doc);
}
/// <summary>
/// Mounts a namespace to the specified adapter.
/// </summary>
public void Mount(string nameSpace, StorageAdapter adapter)
{
if (string.IsNullOrEmpty(nameSpace))
{
Logger.Error(nameof(StorageManager), this.Ctx, "The namespace cannot be null or empty.", nameof(Mount));
return;
}
if (adapter != null)
{
if (adapter is StorageAdapterBase adapterBase)
{
adapterBase.Ctx = this.Ctx;
}
this.NamespaceAdapters[nameSpace] = adapter;
CdmFolderDefinition fd = new CdmFolderDefinition(this.Ctx, "");
fd.Corpus = this.Corpus as CdmCorpusDefinition;
fd.Namespace = nameSpace;
fd.FolderPath = "/";
this.NamespaceFolders[nameSpace] = fd;
this.systemDefinedNamespaces.Remove(nameSpace);
}
else
{
Logger.Error(nameof(StorageManager), this.Ctx, "The adapter cannot be null.", nameof(Mount));
}
}
/// <summary>
/// Given the namespace of a registered storage adapter, returns the root folder containing the sub-folders and documents.
/// </summary>
public CdmFolderDefinition FetchRootFolder(string nameSpace)
{
if (string.IsNullOrEmpty(nameSpace))
{
Logger.Error(nameof(StorageManager), this.Ctx, "The namespace cannot be null or empty.", nameof(FetchRootFolder));
return(null);
}
CdmFolderDefinition folder = null;
if (this.NamespaceFolders.ContainsKey(nameSpace))
{
this.NamespaceFolders.TryGetValue(nameSpace, out folder);
}
else
{
this.NamespaceFolders.TryGetValue(this.DefaultNamespace, out folder);
}
if (folder == null)
{
Logger.Error(nameof(StorageManager), this.Ctx, $"Adapter not found for the namespace '{nameSpace}'", nameof(FetchRootFolder));
}
return(folder);
}
public void TestZeroMinimumCardinality()
{
string testName = "TestZeroMinimumCardinality";
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
Assert.Fail(message);
}
}, CdmStatusLevel.Warning);
// Create Local Root Folder
CdmFolderDefinition localRoot = corpus.Storage.FetchRootFolder("local");
// Create Manifest
CdmManifestDefinition manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "default");
localRoot.Documents.Add(manifest, "default.manifest.cdm.json");
string entityName = "TestEntity";
// Create Entity
CdmEntityDefinition entity = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, entityName);
entity.ExtendsEntity = corpus.MakeRef <CdmEntityReference>(CdmObjectType.EntityRef, "CdmEntity", true);
// Create Entity Document
CdmDocumentDefinition document = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityName}.cdm.json", false);
document.Definitions.Add(entity);
localRoot.Documents.Add(document, document.Name);
manifest.Entities.Add(entity);
string attributeName = "testAttribute";
string attributeDataType = "string";
string attributePurpose = "hasA";
// Create Type Attribute
CdmTypeAttributeDefinition attribute = corpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, nameOrRef: attributeName, simpleNameRef: false);
attribute.DataType = corpus.MakeRef <CdmDataTypeReference>(CdmObjectType.DataTypeRef, refObj: attributeDataType, simpleNameRef: true);
attribute.Purpose = corpus.MakeRef <CdmPurposeReference>(CdmObjectType.PurposeRef, refObj: attributePurpose, simpleNameRef: true);
attribute.DisplayName = attributeName;
if (entity != null)
{
entity.Attributes.Add(attribute);
}
attribute.Cardinality = new CardinalitySettings(attribute)
{
Minimum = "0",
Maximum = "*"
};
Assert.IsTrue(attribute.IsNullable == true);
}
public void TestCdmDefinitionCollectionAddRange()
{
var document = CdmCollectionHelperFunctions.GenerateManifest();
document.IsDirty = false;
var attribute = new CdmAttributeContext(document.Ctx, "the attribute");
var folder = new CdmFolderDefinition(document.Ctx, "The folder");
var trait = new CdmTraitDefinition(document.Ctx, "The trait");
var definitionsList = new List <CdmObjectDefinition>()
{
attribute,
folder,
trait
};
document.Definitions.AddRange(definitionsList);
Assert.IsTrue(document.IsDirty);
Assert.AreEqual(3, document.Definitions.Count);
Assert.AreEqual(attribute, document.Definitions[0]);
Assert.AreEqual(folder, document.Definitions[1]);
Assert.AreEqual(trait, document.Definitions[2]);
Assert.AreEqual(document, attribute.InDocument);
Assert.AreEqual(document, trait.InDocument);
Assert.AreEqual(document, attribute.Owner);
Assert.AreEqual(document, folder.Owner);
Assert.AreEqual(document, trait.Owner);
}
public async Task TestImposedDirectives()
{
const string testName = "TestImposedDirectives";
string testExpectedOutputPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName);
string testActualOutputPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName);
var corpus = TestHelper.GetLocalCorpus(TestsSubpath, testName, null);
corpus.Storage.Mount("localActualOutput", new LocalAdapter(testActualOutputPath));
CdmFolderDefinition actualOutputFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("localActualOutput:/");
// Test "structured" imposed directive
var entity = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/Person_Structured.cdm.json/Person");
var resolvedEntity = await entity.CreateResolvedEntityAsync("Person_Resolved", null, actualOutputFolder);
await resolvedEntity.InDocument.SaveAsAsync("Person_Structured_Resolved.cdm.json", true, new CopyOptions());
ValidateOutput("Person_Structured_Resolved.cdm.json", testExpectedOutputPath, testActualOutputPath);
// Test default imposed directive
entity = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/Person_Default.cdm.json/Person");
resolvedEntity = await entity.CreateResolvedEntityAsync("Person_Resolved", null, actualOutputFolder);
await resolvedEntity.InDocument.SaveAsAsync("Person_Default_Resolved.cdm.json", true, new CopyOptions());
ValidateOutput("Person_Default_Resolved.cdm.json", testExpectedOutputPath, testActualOutputPath);
}
/// <summary>
/// Mounts a namespace to the specified adapter.
/// </summary>
public void Mount(string nameSpace, StorageAdapter adapter)
{
using (Logger.EnterScope(nameof(StorageManager), Ctx, nameof(Mount)))
{
if (string.IsNullOrEmpty(nameSpace))
{
Logger.Error(this.Ctx, Tag, nameof(Mount), null, CdmLogCode.ErrStorageNullNamespace);
return;
}
if (adapter != null)
{
if (adapter is StorageAdapterBase adapterBase)
{
adapterBase.Ctx = this.Ctx;
}
this.NamespaceAdapters[nameSpace] = adapter;
CdmFolderDefinition fd = new CdmFolderDefinition(this.Ctx, "");
fd.Corpus = this.Corpus as CdmCorpusDefinition;
fd.Namespace = nameSpace;
fd.FolderPath = "/";
this.NamespaceFolders[nameSpace] = fd;
this.systemDefinedNamespaces.Remove(nameSpace);
}
else
{
Logger.Error(this.Ctx, Tag, nameof(Mount), null, CdmLogCode.ErrStorageNullAdapter);
}
}
}
/// <summary>
/// Given the namespace of a registered storage adapter, returns the root folder containing the sub-folders and documents.
/// </summary>
public CdmFolderDefinition FetchRootFolder(string nameSpace)
{
using (Logger.EnterScope(nameof(StorageManager), Ctx, nameof(FetchRootFolder)))
{
if (string.IsNullOrEmpty(nameSpace))
{
Logger.Error(this.Ctx, Tag, nameof(FetchRootFolder), null, CdmLogCode.ErrStorageNullNamespace);
return(null);
}
CdmFolderDefinition folder = null;
if (this.NamespaceFolders.ContainsKey(nameSpace))
{
this.NamespaceFolders.TryGetValue(nameSpace, out folder);
}
else
{
this.NamespaceFolders.TryGetValue(this.DefaultNamespace, out folder);
}
if (folder == null)
{
Logger.Error(this.Ctx, Tag, nameof(FetchAdapter), null, CdmLogCode.ErrStorageAdapterNotFound, nameSpace);
}
return(folder);
}
}
public void TestFolderCollectionInsert()
{
var manifest = CdmCollectionHelperFunctions.GenerateManifest("C:\\Root\\Path");
var parentFolder = new CdmFolderDefinition(manifest.Ctx, "ParentFolder");
parentFolder.InDocument = manifest;
parentFolder.Namespace = "TheNamespace";
parentFolder.FolderPath = "ParentFolderPath/";
var childFolders = parentFolder.ChildFolders;
var childFolder = new CdmFolderDefinition(manifest.Ctx, "ChildFolder1");
var child1 = childFolders.Add("child1");
var child2 = childFolders.Add("child2");
manifest.IsDirty = false;
childFolders.Insert(1, childFolder);
Assert.AreEqual(3, childFolders.Count);
Assert.IsTrue(manifest.IsDirty);
Assert.AreEqual(child1, childFolders[0]);
Assert.AreEqual(childFolder, childFolders[1]);
Assert.AreEqual(child2, childFolders[2]);
Assert.AreEqual(manifest.Ctx, childFolder.Ctx);
Assert.AreEqual("ChildFolder1", childFolder.Name);
Assert.AreEqual(parentFolder, childFolder.Owner);
Assert.AreEqual("TheNamespace", childFolder.Namespace);
Assert.AreEqual(parentFolder.FolderPath + childFolder.Name + "/", childFolder.FolderPath);
}
public static void PersistCorpusFolder(string rootPath, CdmFolderDefinition cdmFolder, AttributeResolutionDirectiveSet directiveSet, CopyOptions options = null)
{
if (cdmFolder != null)
{
string folderPath = rootPath + cdmFolder.FolderPath;
Directory.CreateDirectory(folderPath);
if (cdmFolder.Documents != null)
{
cdmFolder.Documents.AllItems.ForEach(doc =>
{
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = doc, Directives = directiveSet
};
PersistDocument(rootPath, resOpt, options);
});
}
if (cdmFolder.ChildFolders != null)
{
cdmFolder.ChildFolders.AllItems.ForEach(f =>
{
PersistCorpusFolder(rootPath, f, directiveSet, options);
});
}
}
}
public void TestDocumentCollectionInsert()
{
var manifest = CdmCollectionHelperFunctions.GenerateManifest();
var folder = new CdmFolderDefinition(manifest.Ctx, "Folder");
folder.InDocument = manifest;
folder.Corpus = manifest.Ctx.Corpus;
folder.FolderPath = "FolderPath/";
folder.Namespace = "Namespace";
var document = new CdmDocumentDefinition(manifest.Ctx, "DocumentName");
var doc1 = folder.Documents.Add("doc1");
var doc2 = folder.Documents.Add("doc2");
manifest.IsDirty = false;
folder.Documents.Insert(2, document);
Assert.IsTrue(manifest.IsDirty);
Assert.AreEqual(3, folder.Documents.Count);
Assert.AreEqual(doc1, folder.Documents[0]);
Assert.AreEqual(doc2, folder.Documents[1]);
Assert.AreEqual(document, folder.Documents[2]);
Assert.AreEqual("FolderPath/", document.FolderPath);
Assert.AreEqual(folder, document.Owner);
Assert.AreEqual("Namespace", document.Namespace);
Assert.IsTrue(document.NeedsIndexing);
Assert.AreEqual(folder, document.Owner);
Assert.IsTrue(folder.DocumentLookup.ContainsKey(document.Name));
Assert.IsTrue(manifest.Ctx.Corpus.documentLibrary.Contains(Tuple.Create(folder, document)));
// reinsert same name doc
folder.Documents.Insert(2, document);
Assert.AreEqual(3, folder.Documents.Count);
}
/// <summary>
/// Common test code for these test cases
/// </summary>
/// <param name="testName"></param>
/// <param name="entityName"></param>
private void TestRun(string testName, string entityName)
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
string inputFolder = TestHelper.GetInputFolderPath(testsSubpath, testName);
string expectedOutputFolder = TestHelper.GetExpectedOutputFolderPath(testsSubpath, testName);
string actualOutputFolder = TestHelper.GetActualOutputFolderPath(testsSubpath, testName);
if (!Directory.Exists(actualOutputFolder))
{
Directory.CreateDirectory(actualOutputFolder);
}
CdmManifestDefinition manifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json").GetAwaiter().GetResult();
Assert.IsNotNull(manifest);
CdmEntityDefinition entity = corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}", manifest).GetAwaiter().GetResult();
Assert.IsNotNull(entity);
CdmEntityDefinition resolvedEntity = TestUtils.GetResolvedEntity(corpus, entity, new List <string> {
"referenceOnly"
}).GetAwaiter().GetResult();
string actualAttrCtx = GetAttributeContextString(resolvedEntity, entityName, actualOutputFolder);
string expectedStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, $"AttrCtx_{entityName}.txt"));
string expectedAttrCtx = File.ReadAllText(expectedStringFilePath);
Assert.AreEqual(expectedAttrCtx, actualAttrCtx);
corpus.CalculateEntityGraphAsync(manifest).GetAwaiter().GetResult();
manifest.PopulateManifestRelationshipsAsync().GetAwaiter().GetResult();
string actualRelationshipsString = ListRelationships(corpus, entity, actualOutputFolder, entityName);
string expectedRelationshipsStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, $"REL_{entityName}.txt"));
string expectedRelationshipsString = File.ReadAllText(expectedRelationshipsStringFilePath);
Assert.AreEqual(expectedRelationshipsString, actualRelationshipsString);
CdmFolderDefinition outputFolder = corpus.Storage.FetchRootFolder("output");
outputFolder.Documents.Add(manifest);
string manifestFileName = $"saved.manifest.cdm.json";
manifest.SaveAsAsync(manifestFileName, saveReferenced: true).GetAwaiter().GetResult();
string actualManifestPath = Path.Combine(actualOutputFolder, manifestFileName);
if (!File.Exists(actualManifestPath))
{
Assert.Fail("Unable to save manifest with relationship");
}
else
{
CdmManifestDefinition savedManifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"output:/{manifestFileName}").GetAwaiter().GetResult();
string actualSavedManifestRel = GetRelationshipStrings(savedManifest.Relationships);
string expectedSavedManifestRel = File.ReadAllText(Path.Combine(expectedOutputFolder, $"MANIFEST_REL_{entityName}.txt"));
Assert.AreEqual(expectedSavedManifestRel, actualSavedManifestRel);
}
}
public async Task TestNestedProjUsingObjectModel()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestNestedProjUsingObjectModel");
corpus.Storage.Mount("local", new LocalAdapter(TestHelper.GetActualOutputFolderPath(testsSubpath, "TestNestedProjUsingObjectModel")));
CdmFolderDefinition localRoot = corpus.Storage.FetchRootFolder("local");
// Create an entity
CdmEntityDefinition entity = ProjectionTestUtils.CreateEntity(corpus, localRoot);
// Create a projection
CdmProjection projection = ProjectionTestUtils.CreateProjection(corpus, localRoot);
// Create an ExcludeAttributes operation
CdmOperationExcludeAttributes excludeAttrsOp = corpus.MakeObject <CdmOperationExcludeAttributes>(CdmObjectType.OperationExcludeAttributesDef);
excludeAttrsOp.ExcludeAttributes.Add("id");
excludeAttrsOp.ExcludeAttributes.Add("date");
projection.Operations.Add(excludeAttrsOp);
// Create an entity reference to hold this projection
CdmEntityReference projectionEntityRef = corpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, null);
projectionEntityRef.ExplicitReference = projection;
// Create another projection that uses the previous projection as its source
CdmProjection projection2 = corpus.MakeObject <CdmProjection>(CdmObjectType.ProjectionDef);
projection2.Source = projectionEntityRef;
// Create an ExcludeAttributes operation
CdmOperationExcludeAttributes excludeAttrsOp2 = corpus.MakeObject <CdmOperationExcludeAttributes>(CdmObjectType.OperationExcludeAttributesDef);
excludeAttrsOp2.ExcludeAttributes.Add("value");
projection2.Operations.Add(excludeAttrsOp2);
// Create an entity reference to hold this projection
CdmEntityReference projectionEntityRef2 = corpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, null);
projectionEntityRef2.ExplicitReference = projection2;
// Create an entity attribute that contains this projection and add this to the entity
CdmEntityAttributeDefinition entityAttribute = corpus.MakeObject <CdmEntityAttributeDefinition>(CdmObjectType.EntityAttributeDef, "TestEntityAttribute");
entityAttribute.Entity = projectionEntityRef2;
entity.Attributes.Add(entityAttribute);
// Resolve the entity
CdmEntityDefinition resolvedEntity = await entity.CreateResolvedEntityAsync($"Resolved_{entity.EntityName}.cdm.json", null, localRoot);
// Verify correctness of the resolved attributes after running the ExcludeAttributes operations
// Original set of attributes: ["id", "name", "value", "date"]
// Excluded attributes: ["id", "date"], ["value"]
Assert.AreEqual(1, resolvedEntity.Attributes.Count);
Assert.AreEqual("name", (resolvedEntity.Attributes[0] as CdmTypeAttributeDefinition).Name);
}
public async Task <CdmManifestDefinition> GenerateCdmAsync(RelationalModel model)
{
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
CdmReferenceResolver resolver = new CdmReferenceResolver(options.ManifestName, options.EntitiesVersion);
var outputManifestFile = Path.Combine(Path.GetFullPath(options.OutputFolder), resolver.GetManifestFileName());
if (File.Exists(outputManifestFile))
{
if (options.OverrideExistingManifest)
{
File.Delete(outputManifestFile);
}
else
{
throw new Exception($"Manifest {outputManifestFile} already exists. Please use the override option.");
}
}
CdmFolderDefinition folder = corpus.GetLocalRootFolder(options.OutputFolder);
CdmManifestDefinition manifest = corpus.CreateCdmManifest(resolver.GetManifestName());
folder.Documents.Add(manifest);
foreach (var table in model.Tables)
{
string documentName = resolver.GetDocumentFileName(table.Name);
CdmDocumentDefinition entityDocument = corpus.CreateCdmDocument(documentName);
folder.Documents.Add(entityDocument);
var entityGenerator = new CdmEntityGenerator(corpus, resolver, entityDocument, options.EntitiesVersion);
CdmEntityDefinition entity = entityGenerator.GenerateEntity(table);
manifest.Entities.Add(entity);
var relationshipGenerator = new CdmEntityRelationshipGenerator(corpus, resolver);
IEnumerable <CdmE2ERelationship> relationships = relationshipGenerator.GenerateRelationships(table);
manifest.Relationships.AddRange(relationships);
}
if (options.HasTimestamps)
{
await manifest.FileStatusCheckAsync();
}
if (options.HasVirtualPartition)
{
CreateVirtualPartitionOnEntities(corpus, manifest);
}
return(manifest);
}
public async Task TestMissingPersistenceFormat()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestMissingPersistenceFormat");
CdmFolderDefinition folder = corpus.Storage.FetchRootFolder(corpus.Storage.DefaultNamespace);
CdmManifestDefinition manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "someManifest");
folder.Documents.Add(manifest);
// trying to save to an unsupported format should return false and not fail
bool succeded = await manifest.SaveAsAsync("manifest.unSupportedExtension");
Assert.IsFalse(succeded);
}
/// <summary>
/// Creates an entity
/// </summary>
public static CdmEntityDefinition CreateEntity(CdmCorpusDefinition corpus, CdmFolderDefinition localRoot)
{
string entityName = "TestEntity";
CdmEntityDefinition entity = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, entityName);
CdmDocumentDefinition entityDoc = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityName}.cdm.json", false);
entityDoc.Imports.Add(foundationJsonPath);
entityDoc.Definitions.Add(entity);
localRoot.Documents.Add(entityDoc, entityDoc.Name);
return(entity);
}
/// <summary>
/// Creates a projection
/// </summary>
public static CdmProjection CreateProjection(CdmCorpusDefinition corpus, CdmFolderDefinition localRoot)
{
// Create an entity reference to use as the source of the projection
CdmEntityReference projectionSource = corpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, null);
projectionSource.ExplicitReference = CreateSourceEntity(corpus, localRoot);
// Create the projection
CdmProjection projection = corpus.MakeObject <CdmProjection>(CdmObjectType.ProjectionDef);
projection.Source = projectionSource;
return(projection);
}
/// <summary>
/// Resolves an entity
/// </summary>
/// <param name="corpus">The corpus</param>
/// <param name="inputEntity">The entity to resolve</param>
/// <param name="directives">A set of directives to be used during resolution</param>
public static async Task <CdmEntityDefinition> GetResolvedEntity(CdmCorpusDefinition corpus, CdmEntityDefinition inputEntity, List <string> directives)
{
string resolvedEntityName = $"Resolved_{inputEntity.EntityName}";
ResolveOptions resOpt = new ResolveOptions(inputEntity.InDocument)
{
Directives = new AttributeResolutionDirectiveSet(new HashSet <string>(directives))
};
CdmFolderDefinition resolvedFolder = corpus.Storage.FetchRootFolder("output");
CdmEntityDefinition resolvedEntity = await inputEntity.CreateResolvedEntityAsync(resolvedEntityName, resOpt, resolvedFolder);
return(resolvedEntity);
}
/// <summary>
/// Mounts a namespace to the specified adapter.
/// </summary>
public void Mount(string nameSpace, StorageAdapter adapter)
{
if (adapter != null)
{
this.NamespaceAdapters[nameSpace] = adapter;
CdmFolderDefinition fd = new CdmFolderDefinition(this.Ctx, "");
fd.Corpus = this.Corpus as CdmCorpusDefinition;
fd.Namespace = nameSpace;
this.NamespaceFolders[nameSpace] = fd;
}
else
{
Logger.Error(nameof(StorageManager), this.Ctx, "The adapter cannot be null.", "Mount");
}
}
public void TestMissingConditionInJson()
{
string testName = "TestMissingConditionInJson";
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
Assert.Fail(message);
}
}, CdmStatusLevel.Warning);
CdmManifestDefinition manifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"default.manifest.cdm.json").GetAwaiter().GetResult();
string entityName = "SalesNestedFK";
CdmEntityDefinition entity = corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}", manifest).GetAwaiter().GetResult();
Assert.IsNotNull(entity);
ResolveOptions resOpt = new ResolveOptions(entity.InDocument)
{
// where, restOptsCombinations[1] == "referenceOnly"
Directives = new AttributeResolutionDirectiveSet(restOptsCombinations[1])
};
CdmFolderDefinition resolvedFolder = corpus.Storage.FetchRootFolder("output");
bool wasInfoMessageReceived = false;
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
if (StringUtils.EqualsWithIgnoreCase("CdmProjection | Optional expression missing. Implicit expression will automatically apply. | ConstructProjectionContext", message))
{
wasInfoMessageReceived = true;
}
}
}, CdmStatusLevel.Info);
CdmEntityDefinition resolvedEntity = entity.CreateResolvedEntityAsync($"Resolved_{entityName}.cdm.json", resOpt, resolvedFolder).GetAwaiter().GetResult();
Assert.IsNotNull(resolvedEntity);
Assert.IsTrue(wasInfoMessageReceived);
}
static async Task LoadDocument(CdmCorpusDefinition corpus, CdmFolderDefinition folder, FileInfo fi, string endMatch)
{
string postfix = fi.Name.Substring(fi.Name.IndexOf("."));
if (postfix == endMatch)
{
using (var reader = File.OpenText(fi.FullName))
{
string content = await reader.ReadToEndAsync().ConfigureAwait(false);
CdmDocumentDefinition doc = DocumentPersistence.FromData(corpus.Ctx, fi.Name, folder.Namespace, folder.FolderPath, JsonConvert.DeserializeObject <DocumentContent>(content));
folder.Documents.Add(doc);
Console.WriteLine($"Loading {fi.FullName}");
}
}
}
public void TestFolderCollectionRemove()
{
var manifest = CdmCollectionHelperFunctions.GenerateManifest("C:\\Root\\Path");
var parentFolder = new CdmFolderDefinition(manifest.Ctx, "ParentFolder");
parentFolder.Namespace = "TheNamespace";
parentFolder.FolderPath = "ParentFolderPath/";
CdmFolderCollection childFolders = parentFolder.ChildFolders;
var childFolder = new CdmFolderDefinition(manifest.Ctx, "ChildFolder1");
Assert.AreEqual(0, childFolders.Count);
childFolders.Add(childFolder);
Assert.AreEqual(1, childFolders.Count);
childFolders.Remove(childFolder);
Assert.AreEqual(0, childFolders.Count);
}
public void TestFetchChildFolderFromPath()
{
var corpus = new CdmCorpusDefinition();
var rootFolder = new CdmFolderDefinition(corpus.Ctx, "");
var folderPath = "/";
var childFolder = rootFolder.FetchChildFolderFromPath(folderPath, false);
Assert.AreEqual(folderPath, childFolder.FolderPath);
folderPath = "/";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, true);
Assert.AreEqual(folderPath, childFolder.FolderPath);
folderPath = "/core";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, false);
Assert.AreEqual("/", childFolder.FolderPath);
folderPath = "/core";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, true);
Assert.AreEqual($"{folderPath}/", childFolder.FolderPath);
folderPath = "/core/";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, false);
Assert.AreEqual(folderPath, childFolder.FolderPath);
folderPath = "/core/";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, true);
Assert.AreEqual(folderPath, childFolder.FolderPath);
folderPath = "/core/applicationCommon";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, false);
Assert.AreEqual("/core/", childFolder.FolderPath);
folderPath = "/core/applicationCommon";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, true);
Assert.AreEqual($"{folderPath}/", childFolder.FolderPath);
folderPath = "/core/applicationCommon/";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, false);
Assert.AreEqual(folderPath, childFolder.FolderPath);
folderPath = "/core/applicationCommon/";
childFolder = rootFolder.FetchChildFolderFromPath(folderPath, true);
Assert.AreEqual(folderPath, childFolder.FolderPath);
}
