private static DataSource CreateDataSource(CdmManifestDefinition instance)
{
var source = new DataSource();
var t2pm = new TraitToPropertyMap(instance);
var sourceTraits = t2pm.FetchTraitReference(dbLocationTrait);
if (sourceTraits != null && sourceTraits.Arguments != null && sourceTraits.Arguments.Count == 1 &&
sourceTraits.Arguments[0].Name == dbLocationTraitArgName)
{
source.Location = Utils.CorpusPathToSymsPath(sourceTraits.Arguments[0].Value, instance.Ctx.Corpus.Storage);
}
if (source.Location == null)
{
Logger.Error((ResolveContext)instance.Ctx, Tag, nameof(ToDataAsync), instance.AtCorpusPath,
CdmLogCode.ErrPersistSymsStorageSourceTraitError, dbLocationTrait, dbLocationTraitArgName);
return(null);
}
return(source);
}
/// <summary>
/// Loads an entity, resolves it, and then validates the generated attribute contexts
/// </summary>
private async Task LoadEntityForResolutionOptionAndSave(string testName, string entityName, List <string> resOpts)
{
string expectedOutputPath = TestHelper.GetExpectedOutputFolderPath(testsSubpath, testName);
string fileNameSuffix = ProjectionTestUtils.GetResolutionOptionNameSuffix(resOpts);
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
corpus.Storage.Mount("expected", new LocalAdapter(expectedOutputPath));
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json");
CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}");
Assert.IsNotNull(entity);
CdmEntityDefinition resolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, resOpts, true);
Assert.IsNotNull(resolvedEntity);
await ValidateResolvedAttributes(corpus, resolvedEntity, entityName, fileNameSuffix);
AttributeContextUtil.ValidateAttributeContext(corpus, expectedOutputPath, $"{entityName}{fileNameSuffix}", resolvedEntity);
}
/// <summary>
/// Creates a Manifest used for the tests.
/// </summary>
/// <param name="localRootPath">A string used as root path for "local" namespace.</param>
/// <returns>Created Manifest.</returns>
private CdmManifestDefinition GenerateManifest(string localRootPath)
{
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(localRootPath));
// add cdm namespace
cdmCorpus.Storage.Mount("cdm", new LocalAdapter("C:\\Root\\Path"));
var manifest = new CdmManifestDefinition(cdmCorpus.Ctx, "manifest");
manifest.FolderPath = "/";
manifest.Namespace = "local";
return(manifest);
}
public static async Task <LocalEntity> ToData(dynamic documentObjectOrPath, CdmManifestDefinition manifest, ResolveOptions resOpt, CopyOptions options, CdmCorpusContext ctx)
{
if (documentObjectOrPath is string)
{
if (await ctx.Corpus.FetchObjectAsync <CdmEntityDefinition>(documentObjectOrPath) is CdmEntityDefinition cdmEntity)
{
var entity = await EntityPersistence.ToData(cdmEntity, resOpt, options, ctx);
if (cdmEntity.Owner != null && cdmEntity.Owner is CdmDocumentDefinition document)
{
if (document.Imports.Count > 0)
{
entity.Imports = new List <Import>();
foreach (var element in document.Imports)
{
var import = CdmFolder.ImportPersistence.ToData(element, resOpt, options);
// the corpus path in the imports are relative to the document where it was defined.
// when saving in model.json the documents are flattened to the manifest level
// so it is necessary to recalculate the path to be relative to the manifest.
var absolutePath = ctx.Corpus.Storage.CreateAbsoluteCorpusPath(import.CorpusPath, document);
import.CorpusPath = ctx.Corpus.Storage.CreateRelativeCorpusPath(absolutePath, manifest);
entity.Imports.Add(import);
}
}
}
else
{
Logger.Warning(nameof(DocumentPersistence), ctx, $"Entity {cdmEntity.GetName()} is not inside a document or its owner is not a document.");
}
return(entity);
}
else
{
Logger.Error(nameof(DocumentPersistence), ctx, "There was an error while trying to fetch cdm entity doc.");
return(null);
}
}
return(null);
}
public async Task RunAsync(DatabaseOptions options)
{
logger.LogInformation("Reading SQL database ...");
RelationalModel model = relationalModelReader.ReadRelationalModel();
if (!model.Tables.Any())
{
logger.LogWarning("No tables were found ...");
return;
}
logger.LogDebug("Running Annotation Combiner ...");
annotationCombiner.ReadAnnotationsAndCombineWithModel(model);
logger.LogInformation("Generating CDM folder ...");
CdmManifestDefinition manifest = await cdmGenerator.GenerateCdmAsync(model);
logger.LogInformation("Saving CDM folder at {cdmOutput}...", Path.GetFullPath(options.OutputCdmFolder));
await manifest.SaveAsync();
logger.LogInformation("Success!");
}
public void TestManifestCannotAddEntityDefinitionWithoutCreatingDocument()
{
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
var callback = new EventCallback();
var functionWasCalled = false;
var functionParameter1 = CdmStatusLevel.Info;
string functionParameter2 = null;
callback.Invoke = (CdmStatusLevel statusLevel, string message1) =>
{
functionWasCalled = true;
functionParameter1 = statusLevel;
functionParameter2 = message1;
};
cdmCorpus.SetEventCallback(callback);
cdmCorpus.Storage.Mount("local", new LocalAdapter("C:\\Root\\Path"));
var manifest = new CdmManifestDefinition(cdmCorpus.Ctx, "manifest");
manifest.FolderPath = "/";
manifest.Namespace = "local";
var entity = new CdmEntityDefinition(manifest.Ctx, "entityName", null);
var corpus = new CdmCorpusDefinition();
manifest.Entities.Add(entity);
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("Expected entity to have an \"Owner\" document set. Cannot create entity declaration to add to manifest."));
Assert.AreEqual(0, manifest.Entities.Count);
}
/// <summary>
/// Checks if incremental trait is needed from foundations.cdm.json
/// </summary>
private static void ImportFoundationsIfIncrementalPartitionTraitExist(CdmManifestDefinition manifest)
{
if (manifest.Entities == null)
{
return;
}
foreach (var ent in manifest.Entities)
{
if (ent is CdmLocalEntityDeclarationDefinition localEntityDef)
{
if (localEntityDef.IncrementalPartitions?.Count > 0 || localEntityDef.IncrementalPartitionPatterns?.Count > 0)
{
if (manifest.Imports.Item(Constants.FoundationsCorpusPath, checkMoniker: false) == null)
{
manifest.Imports.Add(Constants.FoundationsCorpusPath);
// Find one is enough
break;
}
}
}
}
}
/// <summary>
/// Test that a manifest document is fetched using the syms persistence class.
/// </summary>
internal async Task TestSymsFetchManifest(CdmCorpusDefinition corpus, CdmManifestDefinition manifestExpected, string filename, string threadnumber = "")
{
var manifestReadDatabases = await corpus.FetchObjectAsync <CdmManifestDefinition>($"syms:/databases.manifest.cdm.json");
Assert.IsNotNull(manifestReadDatabases);
Assert.AreEqual("databases.manifest.cdm.json", manifestReadDatabases.ManifestName);
if (!manifestReadDatabases.SubManifests.AllItems.Exists(item => item.ManifestName == manifestExpected.ManifestName))
{
Assert.Fail($"Database {manifestExpected.ManifestName} does not exist.");
}
var manifestActual = await corpus.FetchObjectAsync <CdmManifestDefinition>($"syms:/{manifestExpected.ManifestName}/{manifestExpected.ManifestName}.manifest.cdm.json", manifestReadDatabases, null, true);
await manifestActual.SaveAsAsync($"localActOutput:/{filename}{threadnumber}");
await manifestExpected.SaveAsAsync($"localExpOutput:/{filename}{threadnumber}");
var actualContent = TestHelper.GetActualOutputFileContent(testsSubpath, nameof(TestSymsSavingAndFetchingDocument), filename);
var expectedContent = TestHelper.GetExpectedOutputFileContent(testsSubpath, nameof(TestSymsSavingAndFetchingDocument), filename);
TestHelper.AssertSameObjectWasSerialized(actualContent, expectedContent);
}
private async static void addExistingEntity(CdmCorpusDefinition cdmCorpus, string manifestName, CdmManifestDefinition manifestDefinition)
{
CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestName + ".manifest.cdm.json");
if (manifest != null)
{
foreach (CdmEntityDeclarationDefinition eDef in manifest.Entities)
{
// Create the entity definition instance
var entity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, eDef.EntityName, false);
// Add properties to the entity instance
entity.DisplayName = eDef.EntityName;
entity.Version = "1.0.0";
entity.Description = eDef.EntityName;
//manifestAbstract.Entities.Add(entity.EntityName, $"resolved/{eDef.EntityName}.cdm.json/{eDef.EntityName}");
var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(eDef.EntityPath, manifest);
foreach (CdmTypeAttributeDefinition attribute in entSelected.Attributes)
{
//var attributes = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, attribute.Name, "hasA", attribute.DataType);
entity.Attributes.Add(attribute);
}
// Create the document which contains the entity
var entityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{eDef.EntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
entityDoc.Imports.Add(FoundationJsonPath);
entityDoc.Definitions.Add(entity);
// Add the document to the root of the local documents in the corpus
var localRoot = cdmCorpus.Storage.FetchRootFolder("adls");
localRoot.Documents.Add(entityDoc, entityDoc.Name);
manifestDefinition.Entities.Add(entity);
}
}
}
/// <summary>
/// Convert to SyMs object from Manifest object.
/// </summary>
internal static async Task <SymsManifestContent> ConvertManifestToSyms(CdmManifestDefinition doc, StorageAdapterBase adapter, string path,
ResolveOptions resOpt, CopyOptions options)
{
DatabaseEntity databaseEntity = null;
bool isDeltaSync = true;
IList <TableEntity> existingSymsTables = null;
IList <RelationshipEntity> existingSymsRelationshipEntities = null;
try
{
databaseEntity = JsonConvert.DeserializeObject <DatabaseEntity>(await adapter.ReadAsync(path));
}
catch (Exception e)
{
if (e.Message.Contains("NotFound"))
{
isDeltaSync = false;
}
else
{
throw;
}
}
if (isDeltaSync)
{
var entities = await adapter.ReadAsync($"/{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json/entitydefinition");
existingSymsTables = JsonConvert.DeserializeObject <SymsTableResponse>(entities).Tables;
var realtionships = await adapter.ReadAsync($"/{databaseEntity.Name}/{databaseEntity.Name}.manifest.cdm.json/relationships");
existingSymsRelationshipEntities = JsonConvert.DeserializeObject <SymsRelationshipResponse>(realtionships).Relationships;
}
return(await Persistence.Syms.ManifestPersistence.ToDataAsync(doc, resOpt, options, isDeltaSync, existingSymsTables, existingSymsRelationshipEntities));
}
public async Task TestInvalidJson()
{
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, nameof(TestInvalidJson));
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(testInputPath));
corpus.Storage.DefaultNamespace = "local";
CdmManifestDefinition invalidManifest = null;
try
{
invalidManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/invalidManifest.manifest.cdm.json");
}
catch (Exception)
{
Assert.Fail("Error should not be thrown when input json is invalid.");
}
Assert.IsNull(invalidManifest);
}
public async Task TestReferencedEntityDeclarationResolution()
{
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(TestHelper.SchemaDocumentsPath));
cdmCorpus.Storage.DefaultNamespace = "cdm";
var manifest = new CdmManifestDefinition(cdmCorpus.Ctx, "manifest");
manifest.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
var referencedEntity = new CdmReferencedEntityDeclarationDefinition(cdmCorpus.Ctx, "Address");
referencedEntity.EntityPath = "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/electronicMedicalRecords.manifest.cdm.json/Address";
manifest.Entities.Add(referencedEntity);
cdmCorpus.Storage.FetchRootFolder("cdm").Documents.Add(manifest);
var resolvedManifest = await manifest.CreateResolvedManifestAsync("resolvedManifest", null);
Assert.AreEqual(2, resolvedManifest.Entities.Count);
Assert.AreEqual("core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/resolved/Account.cdm.json/Account", resolvedManifest.Entities[0].EntityPath);
Assert.AreEqual("cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/electronicMedicalRecords.manifest.cdm.json/Address", resolvedManifest.Entities[1].EntityPath);
}
public async Task TestSavingInvalidModelJsonName()
{
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Unmount("cdm");
corpus.Storage.DefaultNamespace = "local";
var manifest = new CdmManifestDefinition(corpus.Ctx, "manifest");
corpus.Storage.FetchRootFolder("local").Documents.Add(manifest);
ConcurrentDictionary <string, string> allDocs = new ConcurrentDictionary <string, string>();
var testAdapter = new TestStorageAdapter(allDocs);
corpus.Storage.SetAdapter("local", testAdapter);
var newManifestFromModelJsonName = "my.model.json";
await manifest.SaveAsAsync(newManifestFromModelJsonName, true);
// TODO: because we can load documents properly now, SaveAsAsync returns false. Will check the value returned from SaveAsAsync() when the problem is solved
Assert.IsFalse(allDocs.ContainsKey($"/{newManifestFromModelJsonName}"));
}
public static async Task <CdmManifestDefinition> FromObject(CdmCorpusContext ctx, Model obj, CdmFolderDefinition folder)
{
#region Prepare extensionDoc
List <CdmTraitDefinition> extensionTraitDefList = new List <CdmTraitDefinition>();
#endregion
#region Set manifest fields
CdmManifestDefinition manifest = ctx.Corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, obj.Name);
// We need to set up folder path and namespace of a manifest to be able to retrieve that object.
folder.Documents.Add(manifest);
if (obj.Imports != null)
{
foreach (var element in obj.Imports)
{
manifest.Imports.Add(CdmFolder.ImportPersistence.FromData(ctx, element));
}
}
if (!manifest.Imports.Any((CdmImport importPresent) => importPresent.CorpusPath == "cdm:/foundations.cdm.json"))
{
manifest.Imports.Add("cdm:/foundations.cdm.json");
}
manifest.Explanation = obj.Description;
manifest.LastFileModifiedTime = obj.ModifiedTime;
manifest.LastChildFileModifiedTime = obj.LastChildFileModifiedTime;
manifest.LastFileStatusCheckTime = obj.LastFileStatusCheckTime;
if (!string.IsNullOrEmpty(obj.DocumentVersion))
{
manifest.DocumentVersion = obj.DocumentVersion;
}
if (obj.Application != null)
{
var applicationTrait = ctx.Corpus.MakeRef <CdmTraitReference>(CdmObjectType.TraitRef, "is.managedBy", false);
applicationTrait.IsFromProperty = true;
var arg = ctx.Corpus.MakeObject <CdmArgumentDefinition>(CdmObjectType.ArgumentDef, "application");
arg.Value = obj.Application;
applicationTrait.Arguments.Add(arg);
manifest.ExhibitsTraits.Add(applicationTrait);
}
if (obj.Version != null)
{
var versionTrait = ctx.Corpus.MakeRef <CdmTraitReference>(CdmObjectType.TraitRef, "is.modelConversion.modelVersion", false);
var arg = ctx.Corpus.MakeObject <CdmArgumentDefinition>(CdmObjectType.ArgumentDef, "version");
arg.Value = obj.Version;
versionTrait.Arguments.Add(arg);
manifest.ExhibitsTraits.Add(versionTrait);
}
if (obj.Culture != null)
{
var cultureTrait = ctx.Corpus.MakeRef <CdmTraitReference>(CdmObjectType.TraitRef, "is.partition.culture", false);
cultureTrait.IsFromProperty = true;
var arg = ctx.Corpus.MakeObject <CdmArgumentDefinition>(CdmObjectType.ArgumentDef, "culture");
arg.Value = obj.Culture;
cultureTrait.Arguments.Add(arg);
manifest.ExhibitsTraits.Add(cultureTrait);
}
if (obj.IsHidden == true)
{
var isHiddenTrait = ctx.Corpus.MakeRef <CdmTraitReference>(CdmObjectType.TraitRef, "is.hidden", true);
isHiddenTrait.IsFromProperty = true;
manifest.ExhibitsTraits.Add(isHiddenTrait);
}
var referenceModels = new Dictionary <string, string>();
if (obj.ReferenceModels != null)
{
var referenceModelsTrait = ctx.Corpus.MakeRef <CdmTraitReference>(CdmObjectType.TraitRef, "is.modelConversion.referenceModelMap", false);
var arg = ctx.Corpus.MakeObject <CdmArgumentDefinition>(CdmObjectType.ArgumentDef, "referenceModelMap");
arg.Value = JToken.FromObject(obj.ReferenceModels);
referenceModelsTrait.Arguments.Add(arg);
manifest.ExhibitsTraits.Add(referenceModelsTrait);
foreach (var referenceModel in obj.ReferenceModels)
{
referenceModels.Add(referenceModel.Id, referenceModel.Location);
}
}
var entitySchemaByName = new Dictionary <string, string>();
if (obj.Entities != null && obj.Entities.Count > 0)
{
foreach (var element in obj.Entities)
{
CdmEntityDeclarationDefinition entity = null;
if ((string)element["$type"] == "LocalEntity")
{
entity = await LocalEntityDeclarationPersistence.FromData(ctx, folder, element.ToObject <LocalEntity>(), extensionTraitDefList, manifest);
}
else if ((string)element["$type"] == "ReferenceEntity")
{
var referenceEntity = element.ToObject <ReferenceEntity>();
if (!referenceModels.ContainsKey(referenceEntity.ModelId))
{
Logger.Error(nameof(ManifestPersistence), ctx, $"Model Id {referenceEntity.ModelId} from {referenceEntity.Name} not found in referenceModels.");
return(null);
}
entity = await ReferencedEntityDeclarationPersistence.FromData(ctx, referenceEntity, referenceModels[referenceEntity.ModelId]);
}
else
{
Logger.Error(nameof(ManifestPersistence), ctx, "There was an error while trying to parse entity type.");
}
if (entity != null)
{
manifest.Entities.Add(entity);
entitySchemaByName.Add(entity.EntityName, entity.EntityPath);
}
else
{
Logger.Error(nameof(ManifestPersistence), ctx, "There was an error while trying to parse entity type.");
}
}
}
if (obj.Relationships != null && obj.Relationships.Count > 0)
{
foreach (var element in obj.Relationships)
{
var relationship = await RelationshipPersistence.FromData(ctx, element, entitySchemaByName);
if (relationship != null)
{
manifest.Relationships.Add(relationship);
}
else
{
Logger.Warning(nameof(ManifestPersistence), ctx, "There was an issue while trying to read relationships from the model.json file.");
}
}
}
await Utils.ProcessAnnotationsFromData(ctx, obj, manifest.ExhibitsTraits);
var localExtensionTraitDefList = new List <CdmTraitDefinition>();
ExtensionHelper.ProcessExtensionFromJson(ctx, obj, manifest.ExhibitsTraits, extensionTraitDefList, localExtensionTraitDefList);
#endregion
#region Use extensionDoc, finalize importDocs
List <CdmImport> importDocs = await ExtensionHelper.StandardImportDetection(ctx, extensionTraitDefList, localExtensionTraitDefList);
ExtensionHelper.AddImportDocsToManifest(ctx, importDocs, manifest);
CreateExtensionDocAndAddToFolderAndImports(ctx, extensionTraitDefList, folder);
#endregion
return(manifest);
}
public async static Task <Model> ToData(CdmManifestDefinition instance, ResolveOptions resOpt, CopyOptions options)
{
var result = new Model()
{
Name = instance.ManifestName,
Description = instance.Explanation,
ModifiedTime = instance.LastFileModifiedTime,
LastChildFileModifiedTime = instance.LastChildFileModifiedTime,
LastFileStatusCheckTime = instance.LastFileStatusCheckTime,
DocumentVersion = instance.DocumentVersion
};
TraitToPropertyMap t2pm = new TraitToPropertyMap(instance);
CdmTraitReference isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
if (isHiddenTrait != null)
{
result.IsHidden = true;
}
CdmTraitReference applicationTrait = t2pm.FetchTraitReference("is.managedBy");
if (applicationTrait != null)
{
result.Application = applicationTrait.Arguments.AllItems[0].Value as string;
}
CdmTraitReference versionTrait = t2pm.FetchTraitReference("is.modelConversion.modelVersion");
if (versionTrait != null)
{
result.Version = versionTrait.Arguments.AllItems[0].Value;
}
else
{
// version property is required. If it doesn't exist set default.
result.Version = "1.0";
}
CdmTraitReference cultureTrait = t2pm.FetchTraitReference("is.partition.culture");
if (cultureTrait != null)
{
result.Culture = cultureTrait.Arguments.AllItems[0].Value as string;
}
Dictionary <string, string> referenceEntityLocations = new Dictionary <string, string>();
Dictionary <string, string> referenceModels = new Dictionary <string, string>();
CdmTraitReference referenceModelsTrait = t2pm.FetchTraitReference("is.modelConversion.referenceModelMap");
if (referenceModelsTrait != null)
{
JArray refModels = referenceModelsTrait.Arguments[0].Value as JArray;
foreach (JObject referenceModel in refModels)
{
var referenceModelId = referenceModel["id"];
var referenceModelIdAsString = referenceModelId.ToString();
var referenceModelLocation = referenceModel["location"];
var referenceModelLocationAsString = referenceModelLocation.ToString();
referenceModels.Add(referenceModelIdAsString, referenceModelLocationAsString);
referenceEntityLocations.Add(referenceModelLocationAsString, referenceModelIdAsString);
}
}
Utils.ProcessTraitsAndAnnotationsToData(instance.Ctx, result, instance.ExhibitsTraits);
if (instance.Entities != null && instance.Entities.Count > 0)
{
List <Task> promises = new List <Task>();
var obtainedEntities = new ConcurrentBag <JToken>();
foreach (var entity in instance.Entities)
{
Task createdPromise = Task.Run(async() =>
{
dynamic element = null;
if (entity.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
element = await LocalEntityDeclarationPersistence.ToData(
entity as CdmLocalEntityDeclarationDefinition,
instance,
resOpt,
options
);
}
else if (entity.ObjectType == CdmObjectType.ReferencedEntityDeclarationDef)
{
element = await ReferencedEntityDeclarationPersistence.ToData(
entity as CdmReferencedEntityDeclarationDefinition,
resOpt,
options
);
var location = instance.Ctx.Corpus.Storage.CorpusPathToAdapterPath(entity.EntityPath);
if (string.IsNullOrEmpty(location))
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"Invalid entity path set in entity {entity.EntityName}");
element = null;
}
if (element is ReferenceEntity referenceEntity)
{
location = location.Slice(0, location.LastIndexOf("/"));
if (referenceEntity.ModelId != null)
{
if (referenceModels.TryGetValue(referenceEntity.ModelId, out var savedLocation) && savedLocation != location)
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"Same ModelId pointing to different locations");
element = null;
}
else if (savedLocation == null)
{
referenceModels[referenceEntity.ModelId] = location;
referenceEntityLocations[location] = referenceEntity.ModelId;
}
}
else if (referenceEntity.ModelId == null && referenceEntityLocations.ContainsKey(location))
{
referenceEntity.ModelId = referenceEntityLocations[location];
}
else
{
referenceEntity.ModelId = Guid.NewGuid().ToString();
referenceModels[referenceEntity.ModelId] = location;
referenceEntityLocations[location] = referenceEntity.ModelId;
}
}
}
if (element != null)
{
obtainedEntities.Add(JToken.FromObject(element));
}
else
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"There was an error while trying to convert {entity.EntityName}'s entity declaration to model json format.");
}
});
try
{
// TODO: Currently function is synchronous. Remove next line to turn it asynchronous.
// Currently some functions called are not thread safe.
await createdPromise;
promises.Add(createdPromise);
}
catch (Exception ex)
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, $"There was an error while trying to convert {entity.EntityName}'s entity declaration to model json format for reason {ex.Message}.");
}
}
await Task.WhenAll(promises);
result.Entities = obtainedEntities.ToList();
}
if (referenceModels.Count > 0)
{
result.ReferenceModels = new List <ReferenceModel>();
foreach (var referenceModel in referenceModels)
{
result.ReferenceModels.Add(new ReferenceModel()
{
Id = referenceModel.Key,
Location = referenceModel.Value
});
}
}
if (instance.Relationships != null && instance.Relationships.Count > 0)
{
result.Relationships = new List <SingleKeyRelationship>();
foreach (var cdmRelationship in instance.Relationships)
{
var relationship = await RelationshipPersistence.ToData(cdmRelationship, resOpt, options);
if (relationship != null)
{
result.Relationships.Add(relationship);
}
else
{
Logger.Error(nameof(ManifestPersistence), instance.Ctx, "There was an error while trying to convert cdm relationship to model.json relationship.");
}
}
}
if (instance.Imports != null && instance.Imports.Count > 0)
{
result.Imports = new List <Import>();
foreach (var element in instance.Imports)
{
result.Imports.Add(CdmFolder.ImportPersistence.ToData(element, resOpt, options));
}
}
return(result);
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../";
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "2-create-manifest"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add each declaration, this example is about medical appointments and care plans
manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address");
manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan");
manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept");
manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact");
manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device");
manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment");
manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter");
manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare");
manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", "");
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add("cdm:/foundations.cdm.json");
Console.WriteLine("Save the documents");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// Define the location of the partition, relative to the manifest
var location = $"local:/{entDef.EntityName}/partition-data.csv";
part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved);
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
// Get the actual location of the partition file from the corpus
string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location);
// Make a fake file with nothing but header for columns
string header = "";
foreach (CdmTypeAttributeDefinition att in entDef.Attributes)
{
if (header != "")
{
header += ",";
}
header += att.Name;
}
Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{ entDef.EntityName}"));
File.WriteAllText(partPath, header);
}
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
}
public async static Task <bool> manifestToSQLMetadata(AdlsContext adlsContext, string manifestName, string localRoot, List <SQLMetadata> metadataList)
{
ManifestHandler manifestHandler = new ManifestHandler(adlsContext, localRoot);
CdmManifestDefinition manifest = await manifestHandler.cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestName + ".manifest.cdm.json");
if (manifest == null)
{
Console.WriteLine($"Manifest: {manifestName } at Location {localRoot} is invalid");
return(false);
}
foreach (var submanifest in manifest.SubManifests)
{
string subManifestName = submanifest.ManifestName;
await manifestToSQLMetadata(adlsContext, subManifestName, localRoot + '/' + subManifestName, metadataList);
}
foreach (CdmEntityDeclarationDefinition eDef in manifest.Entities)
{
string entityName = eDef.EntityName;
string dataLocation;
if (eDef.DataPartitionPatterns.Count > 0)
{
var pattern = eDef.DataPartitionPatterns.First();
dataLocation = localRoot + "/" + pattern.RootLocation + pattern.GlobPattern;
}
else if (eDef.DataPartitions.Count > 0)
{
dataLocation = eDef.DataPartitions[0].Location;
string nameSpace = dataLocation.Substring(0, dataLocation.IndexOf(":") + 1);
if (nameSpace != "")
{
dataLocation = dataLocation.Replace(nameSpace, localRoot);
}
else
{
if (dataLocation.StartsWith('/') || localRoot.EndsWith('/'))
{
dataLocation = localRoot + dataLocation;
}
else
{
dataLocation = localRoot + "/" + dataLocation;
}
}
}
else
{
dataLocation = $"{localRoot}/{entityName}/*.*";
}
string fileName = dataLocation.Substring(dataLocation.LastIndexOf("/") + 1);
string ext = fileName.Substring(fileName.LastIndexOf("."));
dataLocation = dataLocation.Replace(fileName, "*" + ext);
var entSelected = await manifestHandler.cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(eDef.EntityPath, manifest);
string columnDef = string.Join(", ", entSelected.Attributes.Select(i => CdmTypeToSQl((CdmTypeAttributeDefinition)i)));;
metadataList.Add(new SQLMetadata()
{
entityName = entityName, columnDefinition = columnDef, dataLocation = dataLocation
});
}
return(true);
}
/// <summary>
/// Get the text version of all the resolved entities.
/// </summary>
/// <param name="cdmCorpus"> The CDM corpus. </param>
/// <param name="directives"> The directives to use while getting the resolved entities. </param>
/// <param name="manifest"> The manifest to be resolved. </param>
/// <param name="spew"> The object used to store the text to be returned. </param>
/// <returns> The text version of the resolved entities. (it's in a form that facilitates debugging) </returns>
internal static async Task <string> ListAllResolved(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CdmManifestDefinition manifest, StringSpewCatcher spew = null)
{
// make sure the corpus has a set of default artifact attributes
await cdmCorpus.PrepareArtifactAttributesAsync();
var seen = new HashSet <string>();
Func <CdmManifestDefinition, Task> seekEntities = null;
seekEntities = async(CdmManifestDefinition f) =>
{
if (f.Entities != null)
{
if (spew != null)
{
spew.SpewLine(f.FolderPath);
}
foreach (CdmEntityDeclarationDefinition entity in f.Entities)
{
string corpusPath;
CdmEntityDeclarationDefinition ent = entity;
CdmObject currentFile = f;
while (ent is CdmReferencedEntityDeclarationDefinition)
{
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(ent.EntityPath, currentFile);
ent = await cdmCorpus.FetchObjectAsync <CdmReferencedEntityDeclarationDefinition>(corpusPath);
currentFile = ent;
}
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(((CdmLocalEntityDeclarationDefinition)ent).EntityPath, currentFile);
ResolveOptions resOpt = new ResolveOptions()
{
ImportsLoadStrategy = ImportsLoadStrategy.Load
};
CdmEntityDefinition newEnt = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(corpusPath, null, resOpt);
resOpt.WrtDoc = newEnt.InDocument;
resOpt.Directives = directives;
ResolvedEntity resEnt = new ResolvedEntity(resOpt, newEnt);
if (spew != null)
{
resEnt.Spew(resOpt, spew, " ", true);
}
}
}
if (f.SubManifests != null)
{
// folder.SubManifests.ForEach(async f =>
foreach (CdmManifestDeclarationDefinition subManifest in f.SubManifests)
{
string corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(subManifest.Definition, f);
await seekEntities(await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(corpusPath));
}
}
};
await seekEntities(manifest);
if (spew != null)
{
return(spew.GetContent());
}
return("");
}
public async Task <bool> createManifest(EntityList entityList, bool resolveRef = true)
{
bool manifestCreated = false;
string manifestName = entityList.manifestName;
Console.WriteLine("Make placeholder manifest");
// Add the temp manifest to the root of the local adapter in the corpus
var localRoot = cdmCorpus.Storage.FetchRootFolder("adls");
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json");
// Create two entities from scratch, and add some attributes, traits, properties, and relationships in between
Console.WriteLine("Create net new entities");
List <EntityDefinition> EntityDefinitions = entityList.entityDefinitions;
foreach (EntityDefinition entityDefinition in EntityDefinitions)
{
string entityName = entityDefinition.name;
string entityDesciption = entityDefinition.description;
// Create the entity definition instance
var entity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, entityName, false);
// Add properties to the entity instance
entity.DisplayName = entityName;
entity.Version = "1.0.0";
entity.Description = entityDesciption;
List <dynamic> attributes = entityDefinition.attributes;
foreach (var a in attributes)
{
// Add type attributes to the entity instance
entity.Attributes.Add(sqlToCDMAttribute(cdmCorpus, a));
}
// Create the document which contains the entity
var entityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{entityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
entityDoc.Imports.Add(FoundationJsonPath);
entityDoc.Definitions.Add(entity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(entityDoc, entityDoc.Name);
manifestAbstract.Entities.Add(entity);
}
CdmManifestDefinition manifestResolved = await manifestAbstract.CreateResolvedManifestAsync(manifestName, null);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add(FoundationJsonPath);
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
var entityDefinition = EntityDefinitions.Find(x => x.name.Equals(entDef.EntityName, StringComparison.OrdinalIgnoreCase));
if (entityDefinition != null)
{
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data");
eDef.DataPartitions.Add(part);
part.Explanation = "data files";
// We have existing partition files for the custom entities, so we need to make the partition point to the file location
part.Location = $"adls:{entityDefinition.dataPartitionLocation}/{entityDefinition.partitionPattern}";
if (entityDefinition.partitionPattern.Contains("parquet"))
{
part.ExhibitsTraits.Add("is.partition.format.parquet", false);
}
//default is csv
else
{
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "false");
csvTrait.Arguments.Add("delimiter", ",");
}
}
}
Console.WriteLine("Save the documents");
// We can save the documents as manifest.cdm.json format or model.json
// Save as manifest.cdm.json
manifestCreated = await manifestResolved.SaveAsAsync($"{manifestName}.manifest.cdm.json", resolveRef);
// Save as a model.json
// await manifestResolved.SaveAsAsync("model.json", true);
return(manifestCreated);
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
// set callback to receive error and warning logs.
cdmCorpus.SetEventCallback(new EventCallback
{
Invoke = (level, message) =>
{
Console.WriteLine(message);
}
}, CdmStatusLevel.Warning);
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../../";
string sampleEntityName = "Account";
// Mount it as a local adapter.
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "7-search-partition-pattern/sample-data"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the CDM Standards adapter
// Mount it as the 'cdm' adapter, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "tempAbstract.manifest.cdm.json");
// Add an entity named Account from some public standards
Console.WriteLine("Add an entity named Account from some public standards");
var accountDeclarationDefinition = manifestAbstract.Entities.Add(sampleEntityName, "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
// Create a data partition pattern
var dataPartitionPattern = CreatePartitionPatternDefinition(cdmCorpus, "sampleDataPartitionPattern");
dataPartitionPattern.Explanation = "/ capture 4 digits / capture a word / capture one or more digits after the word cohort but before .csv";
Console.WriteLine($" Assign regular expression of the data partition pattern to: {dataPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the data partition pattern to: {dataPartitionPattern.RootLocation}");
// Add the data partition pattern we just created to the entity data partition pattern collection
accountDeclarationDefinition.DataPartitionPatterns.Add(dataPartitionPattern);
// Create incremental partition patterns
var upsertIncrementalPartitionPattern = CreatePartitionPatternDefinition(cdmCorpus, "UpsertPattern", true, true);
AddIncrementalPartitionTrait(upsertIncrementalPartitionPattern, CdmIncrementalPartitionType.Upsert);
Console.WriteLine($"\n Assign regular expression of the first incremental partition pattern to: {upsertIncrementalPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the first incremental partition pattern to: {upsertIncrementalPartitionPattern.RootLocation}");
var deleteIncrementalPartitionPattern = CreatePartitionPatternDefinition(cdmCorpus, "DeletePattern", true);
AddIncrementalPartitionTrait(deleteIncrementalPartitionPattern, CdmIncrementalPartitionType.Delete, "FullDataPattern");
Console.WriteLine($"\n Assign regular expression of the second incremental partition pattern to: {deleteIncrementalPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the second incremental partition pattern to: {deleteIncrementalPartitionPattern.RootLocation}");
// Add the incremental partition patterns we just created to the entity increment partition pattern collection
accountDeclarationDefinition.IncrementalPartitionPatterns.Add(upsertIncrementalPartitionPattern);
accountDeclarationDefinition.IncrementalPartitionPatterns.Add(deleteIncrementalPartitionPattern);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestAbstract.Imports.Add("cdm:/foundations.cdm.json");
// Calling FileStatusCheckAsync to pick up the all data partition files which names match the data partition pattern,
// and add them to the entity in the manifest
await manifestAbstract.FileStatusCheckAsync(PartitionFileStatusCheckType.FullAndIncremental);
// List all data partition locations.
Console.WriteLine($"\nlist of all data partition locations for the entity Account matches the data partition pattern:");
foreach (CdmDataPartitionDefinition dataPartition in accountDeclarationDefinition.DataPartitions)
{
Console.WriteLine($" {dataPartition.Location}");
}
// List all incremental partition locations.
Console.WriteLine($"\nlist of all incremental partition locations for the entity Account matches the incremental partition pattern:");
foreach (CdmDataPartitionDefinition incrementalPartition in accountDeclarationDefinition.IncrementalPartitions)
{
Console.WriteLine($" {incrementalPartition.Location}");
}
Console.WriteLine("Resolve the manifest");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// You can save the doc as a model.json format as an option
// await manifestResolved.SaveAsAsync("model.json", true);
}
public async static Task ListAllResolved(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CdmManifestDefinition manifest, StringSpewCatcher spew = null)
{
ISet <string> seen = new HashSet <string>();
Func <CdmManifestDefinition, Task> seekEntities = null;
seekEntities = async(CdmManifestDefinition f) =>
{
if (f.Entities != null)
{
if (spew != null)
{
spew.SpewLine(f.FolderPath);
}
// manifest.Entities.ForEach(async entity =>
foreach (CdmEntityDeclarationDefinition entity in f.Entities)
{
string corpusPath;
CdmEntityDeclarationDefinition ent = entity;
CdmObject currentFile = f;
while (ent is CdmReferencedEntityDeclarationDefinition)
{
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(ent.EntityPath, currentFile);
ent = await cdmCorpus.FetchObjectAsync <CdmReferencedEntityDeclarationDefinition>(corpusPath);
currentFile = (CdmObject)ent;
}
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(((CdmLocalEntityDeclarationDefinition)ent).EntityPath, currentFile);
var newEnt = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(corpusPath);
ResolveOptions resOpt = new ResolveOptions()
{
WrtDoc = newEnt.InDocument, Directives = directives
};
ResolvedEntity resEnt = new ResolvedEntity(resOpt, newEnt);
if (spew != null)
{
resEnt.Spew(resOpt, spew, " ", true);
}
}
}
if (f.SubManifests != null)
{
// folder.SubManifests.ForEach(async f =>
foreach (CdmManifestDeclarationDefinition subManifest in f.SubManifests)
{
string corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(subManifest.Definition, f);
await seekEntities(await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(corpusPath));
}
}
};
await seekEntities(manifest);
if (spew != null)
{
File.WriteAllText(@"c:\temp\allResolved.txt", spew.GetContent(), Encoding.UTF8);
}
}
/// <summary>
/// Common test code for these test cases
/// </summary>
/// <param name="testName"></param>
/// <param name="entityName"></param>
private void TestRun(string testName, string entityName)
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
string inputFolder = TestHelper.GetInputFolderPath(testsSubpath, testName);
string expectedOutputFolder = TestHelper.GetExpectedOutputFolderPath(testsSubpath, testName);
string actualOutputFolder = TestHelper.GetActualOutputFolderPath(testsSubpath, testName);
if (!Directory.Exists(actualOutputFolder))
{
Directory.CreateDirectory(actualOutputFolder);
}
CdmManifestDefinition manifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"local:/default.manifest.cdm.json").GetAwaiter().GetResult();
Assert.IsNotNull(manifest);
CdmEntityDefinition entity = corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}", manifest).GetAwaiter().GetResult();
Assert.IsNotNull(entity);
CdmEntityDefinition resolvedEntity = ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> {
"referenceOnly"
}).GetAwaiter().GetResult();
AttributeContextUtil.ValidateAttributeContext(corpus, expectedOutputFolder, entityName, resolvedEntity);
corpus.CalculateEntityGraphAsync(manifest).GetAwaiter().GetResult();
manifest.PopulateManifestRelationshipsAsync().GetAwaiter().GetResult();
string actualRelationshipsString = ListRelationships(corpus, entity, actualOutputFolder, entityName);
string relationshipsFilename = $"REL_{entityName}.txt";
File.WriteAllText(Path.Combine(actualOutputFolder, relationshipsFilename), actualRelationshipsString);
string expectedRelationshipsStringFilePath = Path.GetFullPath(Path.Combine(expectedOutputFolder, relationshipsFilename));
string expectedRelationshipsString = File.ReadAllText(expectedRelationshipsStringFilePath);
Assert.AreEqual(expectedRelationshipsString, actualRelationshipsString);
CdmFolderDefinition outputFolder = corpus.Storage.FetchRootFolder("output");
outputFolder.Documents.Add(manifest);
string manifestFileName = $"saved.manifest.cdm.json";
manifest.SaveAsAsync(manifestFileName, saveReferenced: true).GetAwaiter().GetResult();
string actualManifestPath = Path.Combine(actualOutputFolder, manifestFileName);
if (!File.Exists(actualManifestPath))
{
Assert.Fail("Unable to save manifest with relationship");
}
else
{
CdmManifestDefinition savedManifest = corpus.FetchObjectAsync <CdmManifestDefinition>($"output:/{manifestFileName}").GetAwaiter().GetResult();
string actualSavedManifestRel = GetRelationshipStrings(savedManifest.Relationships);
string manifestRelationshipsFilename = $"MANIFEST_REL_{entityName}.txt";
File.WriteAllText(Path.Combine(actualOutputFolder, manifestRelationshipsFilename), actualSavedManifestRel);
string expectedSavedManifestRel = File.ReadAllText(Path.Combine(expectedOutputFolder, manifestRelationshipsFilename));
Assert.AreEqual(expectedSavedManifestRel, actualSavedManifestRel);
}
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../";
string sampleEntityName = "Account";
// Mount is as a local device.
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "7-search-partition-pattern"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "tempAbstract.manifest.cdm.json");
// Add an entity named Account from some public standards
Console.WriteLine("Add an entity named Account from some public standards");
var personDeclarationDefinition = manifestAbstract.Entities.Add(sampleEntityName, "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
// Create a data partition pattern
var dataPartitionPattern = cdmCorpus.MakeObject <CdmDataPartitionPatternDefinition>(CdmObjectType.DataPartitionPatternDef, "sampleDataPartitionPattern", false);
dataPartitionPattern.RootLocation = "local:dataFiles";
dataPartitionPattern.RegularExpression = "/(\\d{4})/(\\w+)/cohort(\\d+)\\.csv$";
Console.WriteLine($" Assign regular expression of the data partition pattern to: {dataPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the data partition pattern to: {dataPartitionPattern.RootLocation}");
dataPartitionPattern.Explanation = "/ capture 4 digits / capture a word / capture one or more digits after the word cohort but before .csv";
dataPartitionPattern.Parameters = new List <string> {
"year", "month", "cohortNumber"
};
// Add the data partition pattern we just created to the entity data partition pattern collection
personDeclarationDefinition.DataPartitionPatterns.Add(dataPartitionPattern);
// Calling FileStatusCheckAsync to pick up the all data partition files which names match the data partition pattern,
// and add them to the entity in the manifest
await manifestAbstract.FileStatusCheckAsync();
// List all data partition locations.
Console.WriteLine($"\nlist of all data partition locations for the entity Account matches the data partition pattern:");
foreach (CdmDataPartitionDefinition dataPartition in personDeclarationDefinition.DataPartitions)
{
Console.WriteLine($" {dataPartition.Location}");
}
Console.WriteLine("Resolve the manifest");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// You can save the doc as a model.json format as an option
// await manifestResolved.SaveAsAsync("model.json", true);
}
private async Task CreateManifest(CdmCorpusDefinition cdmCorpus)
{
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add each declaration, this example is about medical appointments and care plans
manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address");
manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan");
manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept");
manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact");
manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device");
manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment");
manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter");
manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare");
manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", "");
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add("cdm:/foundations.cdm.json");
Console.WriteLine("Save the documents");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// Define the location of the partition, relative to the manifest
var location = $"local:/{entDef.EntityName}/partition-data.csv";
part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved);
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false) as CdmTraitReference;
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
// Get the actual location of the partition file from the corpus
string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location);
// Make a fake file with nothing but header for columns
string header = "";
foreach (CdmTypeAttributeDefinition att in entDef.Attributes)
{
if (header != "")
{
header += ",";
}
header += att.Name;
}
Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{entDef.EntityName}"));
File.WriteAllText(partPath, header);
}
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
}
private async Task <CdmEntityDefinition> SaveResolved(CdmCorpusDefinition corpus, CdmManifestDefinition manifest, string testName, CdmEntityDefinition inputEntity, List <string> resolutionOptions)
{
HashSet <string> roHashSet = new HashSet <string>();
for (int i = 0; i < resolutionOptions.Count; i++)
{
roHashSet.Add(resolutionOptions[i]);
}
string fileNameSuffix = GetResolutionOptionNameSuffix(resolutionOptions);
string resolvedEntityName = $"Resolved_{inputEntity.EntityName}{fileNameSuffix}.cdm.json";
ResolveOptions ro = new ResolveOptions(inputEntity.InDocument)
{
Directives = new AttributeResolutionDirectiveSet(roHashSet)
};
CdmFolderDefinition resolvedFolder = corpus.Storage.FetchRootFolder("output");
CdmEntityDefinition resolvedEntity = await inputEntity.CreateResolvedEntityAsync(resolvedEntityName, ro, resolvedFolder);
return(resolvedEntity);
}
static async Task Main()
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
Console.WriteLine("configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
var pathFromExeToExampleRoot = "../../../../../../";
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "6-create-net-new-entities"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the github adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local documents in the corpus
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "TempAbstract.manifest.cdm.json");
// Create two entities from scratch, and add some attributes, traits, properties, and relationships in between
Console.WriteLine("Create net new entities");
// Create the simplest entity - CustomPerson
// Create the entity definition instance
var personEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomPersonEntityName, false);
// Add type attributes to the entity instance
var personAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Id", "identifiedBy", "entityId");
personEntity.Attributes.Add(personAttributeId);
var personAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomPersonEntityName}Name", "hasA", "string");
personEntity.Attributes.Add(personAttributeName);
// Add properties to the entity instance
personEntity.DisplayName = CustomPersonEntityName;
personEntity.Version = "0.0.1";
personEntity.Description = "This is a custom entity created for the sample.";
// Create the document which contains the entity
var personEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomPersonEntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
personEntityDoc.Imports.Add(FoundationJsonPath);
personEntityDoc.Definitions.Add(personEntity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(personEntityDoc, personEntityDoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(personEntity);
// Create an entity - CustomAccount which has a relationship with the entity CustomPerson
// Create the entity definition instance
var accountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, CustomAccountEntityName, false);
// Add type attributes to the entity instance
var accountAttributeId = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Id", "identifiedBy", "entityId");
accountEntity.Attributes.Add(accountAttributeId);
var accountAttributeName = CreateEntityAttributeWithPurposeAndDataType(cdmCorpus, $"{CustomAccountEntityName}Name", "hasA", "string");
accountEntity.Attributes.Add(accountAttributeName);
// Add properties to the entity instance
accountEntity.DisplayName = CustomAccountEntityName;
accountEntity.Version = "0.0.1";
accountEntity.Description = "This is a custom entity created for the sample.";
// In this sample, every account has one person who owns the account
// the relationship is actually an entity attribute
var attributeExplanation = "The owner of the account, which is a person.";
// You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
// more details of how to use resolution guidance to customize your data
var accountOwnerAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomPersonEntityName, "accountOwner", attributeExplanation);
accountEntity.Attributes.Add(accountOwnerAttribute);
// Create the document which contains the entity
var accountEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{CustomAccountEntityName}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
accountEntityDoc.Imports.Add(FoundationJsonPath);
// the CustomAccount entity has a relationship with the CustomPerson entity, this relationship is defined from its attribute with traits,
// the import to the entity reference CustomPerson's doc is required
accountEntityDoc.Imports.Add($"{CustomPersonEntityName}.cdm.json");
accountEntityDoc.Definitions.Add(accountEntity);
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(accountEntityDoc, accountEntityDoc.Name);
// Add the entity to the manifest
manifestAbstract.Entities.Add(accountEntity);
// Create an entity which extends "Account" from the standard, it contains everything that "Account" has
var extendedStandardAccountEntity = cdmCorpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, ExtendedStandardAccount, false);
// This function with 'true' will make a simple reference to the base
extendedStandardAccountEntity.ExtendsEntity = cdmCorpus.MakeObject <CdmEntityReference>(CdmObjectType.EntityRef, "Account", true);
var attrExplanation = "This is a simple custom account for this sample.";
// Add a relationship from it to the CustomAccount entity, and name the foreign key to SimpleCustomAccount
// You can all CreateSimpleAttributeForRelationshipBetweenTwoEntities() instead, but CreateAttributeForRelationshipBetweenTwoEntities() can show
// more details of how to use resolution guidance to customize your data
var simpleCustomAccountAttribute = CreateAttributeForRelationshipBetweenTwoEntities(cdmCorpus, CustomAccountEntityName, "SimpleCustomAccount", attrExplanation);
extendedStandardAccountEntity.Attributes.Add(simpleCustomAccountAttribute);
var extendedStandardAccountntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"{ExtendedStandardAccount}.cdm.json", false);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
extendedStandardAccountntityDoc.Imports.Add(FoundationJsonPath);
// The ExtendedAccount entity extends from the "Account" entity from standards, the import to the entity Account's doc is required
// it also has a relationship with the CustomAccount entity, the relationship defined from its from its attribute with traits, the import to the entity reference CustomAccount's doc is required
extendedStandardAccountntityDoc.Imports.Add($"{SchemaDocsRoot}/Account.cdm.json");
extendedStandardAccountntityDoc.Imports.Add($"{CustomAccountEntityName}.cdm.json");
// Add the document to the root of the local documents in the corpus
localRoot.Documents.Add(extendedStandardAccountntityDoc, extendedStandardAccountntityDoc.Name);
extendedStandardAccountntityDoc.Definitions.Add(extendedStandardAccountEntity);
// Add the entity to the manifest
manifestAbstract.Entities.Add(extendedStandardAccountEntity);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
Console.WriteLine("Save the docs");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// We have existing partition files for the custom entities, so we need to make the partition point to the file location
part.Location = $"local:/{entDef.EntityName}/partition-data.csv";
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false);
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
}
// We can save the docs as manifest.cdm.json format or model.json
// Save as manifest.cdm.json
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// Save as a model.json
// await manifestResolved.SaveAsAsync("model.json", true);
}
public static async Task <LocalEntity> ToData(CdmLocalEntityDeclarationDefinition instance, CdmManifestDefinition manifest, ResolveOptions resOpt, CopyOptions options)
{
var localEntity = await DocumentPersistence.ToData(instance.EntityPath, manifest, resOpt, options, instance.Ctx);
if (localEntity != null)
{
var t2pm = new TraitToPropertyMap(instance);
var isHiddenTrait = t2pm.FetchTraitReference("is.hidden");
localEntity.Description = instance.Explanation;
localEntity.LastChildFileModifiedTime = instance.LastChildFileModifiedTime;
localEntity.LastFileModifiedTime = instance.LastFileModifiedTime;
localEntity.LastFileStatusCheckTime = instance.LastFileStatusCheckTime;
if (isHiddenTrait != null)
{
localEntity.IsHidden = true;
}
if (t2pm.FetchPropertyValue("cdmSchemas") is List <string> schemas)
{
localEntity.Schemas = schemas;
}
if (instance.DataPartitions != null && instance.DataPartitions.Count > 0)
{
localEntity.Partitions = new List <Partition>();
foreach (var element in instance.DataPartitions)
{
var partition = await DataPartitionPersistence.ToData(element, resOpt, options);
if (partition != null)
{
localEntity.Partitions.Add(partition);
}
else
{
Logger.Error(nameof(LocalEntityDeclarationPersistence), instance.Ctx, "There was an error while trying to convert cdm data partition to model.json partition.");
return(null);
}
}
}
}
return(localEntity);
}
public async Task <bool> createManifest(EntityList entityList, bool createModelJson = false)
{
bool manifestCreated = false;
string manifestName = entityList.manifestName;
// Add to root folder.
var adlsRoot = cdmCorpus.Storage.FetchRootFolder("adls");
List <EntityDefinition> EntityDefinitions = entityList.entityDefinitions;
// Read manifest if exists.
CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestName + ".manifest.cdm.json");
if (manifest == null)
{
// Make the temp manifest and add it to the root of the local documents in the corpus
manifest = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, manifestName);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifest.Imports.Add(FoundationJsonPath);
// Add to root folder.
adlsRoot.Documents.Add(manifest, $"{manifestName}.manifest.cdm.json");
}
else
{
foreach (EntityDefinition entityDefinition in EntityDefinitions)
{
foreach (CdmEntityDeclarationDefinition localEntityDefinition in manifest.Entities.ToList())
{
if (localEntityDefinition.EntityName.Equals(entityDefinition.name, StringComparison.OrdinalIgnoreCase))
{
manifest.Entities.Remove(currObject: localEntityDefinition);
}
}
}
}
foreach (EntityDefinition entityDefinition in EntityDefinitions)
{
var cdmEntityDefinition = this.CreateCdmEntityDefinition(entityDefinition);
var cdmEntityDocument = this.CreateDocumentDefinition(cdmEntityDefinition);
// Add Imports to the entity document.
cdmEntityDocument.Imports.Add(FoundationJsonPath);
// Add the document to the root of the local documents in the corpus.
adlsRoot.Documents.Add(cdmEntityDocument, cdmEntityDocument.Name);
// Add the entity to the manifest.
manifest.Entities.Add(cdmEntityDefinition);
this.addPartition(manifest, entityDefinition);
}
Console.WriteLine("Save the documents");
// We can save the documents as manifest.cdm.json format or model.json
// Save as manifest.cdm.json
manifestCreated = await manifest.SaveAsAsync($"{manifestName}.manifest.cdm.json", true);
// Save as a model.json
if (createModelJson)
{
await manifest.SaveAsAsync("model.json", true);
}
return(manifestCreated);
}
static async Task ExploreManifest(CdmCorpusDefinition cdmCorpus, string manifestPath)
{
Console.WriteLine($"\nLoading manifest {manifestPath} ...");
CdmManifestDefinition manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(manifestPath);
if (manifest == null)
{
Console.WriteLine($"Unable to load manifest {manifestPath}. Please inspect error log for additional details.");
return;
}
// ------------------------------------------------------------------------------------------------------------
// List all the entities found in the manifest and allow the user to choose which entity to explore.
while (true)
{
int index = 1;
if (manifest.Entities.Count > 0)
{
Console.WriteLine("List of all entities:");
foreach (var entDec in manifest.Entities)
{
// Print entity declarations.
// Assume there are only local entities in this manifest for simplicity.
Console.Write(" " + index.ToString().PadRight(3));
Console.Write(" " + entDec.EntityName.PadRight(35));
Console.WriteLine(" " + entDec.EntityPath);
index++;
}
}
if (manifest.SubManifests.Count > 0)
{
Console.WriteLine("List of all sub-manifests:");
foreach (var manifestDecl in manifest.SubManifests)
{
// Print sub-manifest declarations.
Console.Write(" " + index.ToString().PadRight(3));
Console.Write(" " + manifestDecl.ManifestName.PadRight(35));
Console.WriteLine(" " + manifestDecl.Definition);
index++;
}
}
if (index == 1)
{
Console.Write("No Entities or Sub-manifest found. Press [enter] to exit.");
Console.ReadLine();
break;
}
Console.Write("Enter a number to show details for that Entity or Sub-manifest (press [enter] to exit): ");
// Get the user's choice.
string input = Console.ReadLine();
if (string.IsNullOrEmpty(input))
{
break;
}
// Make sure the user's input is a number.
int num = 0;
if (!int.TryParse(input, out num))
{
Console.WriteLine("\nEnter a number.");
Console.WriteLine();
continue;
}
// User can select an entry that is a sub-manifest
if (num > manifest.Entities.Count)
{
int subNum = num - manifest.Entities.Count - 1;
// Re-enter this method supplying the absolute path of the submanifest definition (relative to the current manifest)
await ExploreManifest(cdmCorpus, cdmCorpus.Storage.CreateAbsoluteCorpusPath(manifest.SubManifests[subNum].Definition, manifest));
continue;
}
index = 1;
foreach (var entityDec in manifest.Entities)
{
if (index == num)
{
Console.WriteLine("Reading the entity schema and resolving with the standard docs, first one may take a second ...");
// From the path to the entity, get the actual schema description.
// Take the local relative path in this doc and make sure it works.
var entSelected = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(entityDec.EntityPath, manifest); // gets the entity object from the doc
while (true)
{
// List all the metadata properties of this entity that can be explored.
Console.WriteLine($"\nMetadata properties for the entity {entityDec.EntityName}:");
Console.WriteLine(" 1: Attributes");
Console.WriteLine(" 2: Traits");
Console.WriteLine(" 3: Properties");
Console.WriteLine(" 4: Data partition locations");
Console.WriteLine(" 5: Relationships");
Console.Write("Enter a number to show details for that metadata property (press [enter] to explore other entities): ");
// Get the user's choice.
input = Console.ReadLine();
if (string.IsNullOrEmpty(input))
{
Console.WriteLine();
break;
}
// Make sure the user's input is a number.
int choice = 0;
if (int.TryParse(input, out choice))
{
switch (choice)
{
// List the entity's attributes.
case 1:
ListAttributes(entSelected);
break;
// List the entity's traits.
case 2:
ListTraits(entSelected);
break;
// List the entity's properties.
case 3:
ListProperties(entSelected, entityDec);
break;
// List the entity's data partition locations.
case 4:
ListDataPartitionLocations(cdmCorpus, entityDec);
break;
// List the entity's relationships.
case 5:
if (manifest.Relationships != null && manifest.Relationships.Count > 0)
{
// The manifest file contains pre-calculated entity relationships, so we can read them directly.
ListRelationshipsFromManifest(manifest, entSelected);
}
else
{
// The manifest file doesn't contain relationships, so we have to compute the relationships first.
await cdmCorpus.CalculateEntityGraphAsync(manifest);
ListRelationships(cdmCorpus, entSelected);
}
break;
default:
Console.WriteLine("\nEnter a number between 1-5.");
break;
}
}
else
{
Console.WriteLine("\nEnter a number.");
}
}
}
index++;
}
}
}
/// <summary>
/// Test that a document is saved using the syms persistence class.
/// </summary>
internal async Task TestSymsSaveManifest(CdmManifestDefinition manifest)
{
Assert.IsTrue(await manifest.SaveAsAsync($"syms:/{manifest.ManifestName}/{manifest.ManifestName}.manifest.cdm.json"));
}
