public static void PersistCorpus(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CopyOptions options = null)
{
if (cdmCorpus?.ChildFolders?.Count == 1)
{
PersistCorpusFolder(cdmCorpus.RootPath, cdmCorpus.ChildFolders[0], directives, options);
}
}
public async Task ResolveCorpus()
{
Assert.IsTrue(Directory.Exists(Path.GetFullPath(SchemaDocsRoot)), "SchemaDocsRoot not found!!!");
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
Console.WriteLine("reading source files");
var watch = Stopwatch.StartNew();
cdmCorpus.Storage.Mount("local", new LocalAdapter(SchemaDocsRoot));
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(TestHelper.CdmStandardSchemaPath);
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
await EntityResolutionTests.ListAllResolved(cdmCorpus, directives, manifest);
watch.Stop();
Assert.Performance(70000, watch.ElapsedMilliseconds);
}
public async static Task ListAllResolved(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CdmManifestDefinition manifest, StringSpewCatcher spew = null)
{
ISet <string> seen = new HashSet <string>();
Func <CdmManifestDefinition, Task> seekEntities = null;
seekEntities = async(CdmManifestDefinition f) =>
{
if (f.Entities != null)
{
if (spew != null)
{
spew.SpewLine(f.FolderPath);
}
// manifest.Entities.ForEach(async entity =>
foreach (CdmEntityDeclarationDefinition entity in f.Entities)
{
string corpusPath;
CdmEntityDeclarationDefinition ent = entity;
CdmObject currentFile = f;
while (ent is CdmReferencedEntityDeclarationDefinition)
{
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(ent.EntityPath, currentFile);
ent = await cdmCorpus.FetchObjectAsync <CdmReferencedEntityDeclarationDefinition>(corpusPath);
currentFile = (CdmObject)ent;
}
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(((CdmLocalEntityDeclarationDefinition)ent).EntityPath, currentFile);
var newEnt = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(corpusPath);
ResolveOptions resOpt = new ResolveOptions()
{
WrtDoc = newEnt.InDocument, Directives = directives
};
ResolvedEntity resEnt = new ResolvedEntity(resOpt, newEnt);
if (spew != null)
{
resEnt.Spew(resOpt, spew, " ", true);
}
}
}
if (f.SubManifests != null)
{
// folder.SubManifests.ForEach(async f =>
foreach (CdmManifestDeclarationDefinition subManifest in f.SubManifests)
{
string corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(subManifest.Definition, f);
await seekEntities(await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(corpusPath));
}
}
};
await seekEntities(manifest);
if (spew != null)
{
File.WriteAllText(@"c:\temp\allResolved.txt", spew.GetContent(), Encoding.UTF8);
}
}
/// <summary>
/// Resolve the entities in the given manifest.
/// </summary>
/// <param name="testsSubPath">Tests sub-folder name</param>
/// <param name="testName">The name of the test. It is used to decide the path of input / output files. </param>
/// <parameter name="manifestName">The name of the manifest to be used. </parameter>
/// <returns> The resolved entities. </returns>
internal static async Task <string> ResolveEnvironment(string testsSubPath, string testName, string manifestName)
{
var cdmCorpus = TestHelper.GetLocalCorpus(testsSubPath, testName);
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{manifestName}.manifest.cdm.json");
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
return(await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher()));
}
public async Task ResolveEntities()
{
var cdmCorpus = new CdmCorpusDefinition();
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestResolveEntities");
cdmCorpus.RootPath = testInputPath;
cdmCorpus.Storage.Mount("local", new LocalAdapter(testInputPath));
cdmCorpus.Storage.DefaultNamespace = "local";
var entities = await this.GetAllEntities(cdmCorpus);
var entityResolutionTimes = new List <Tuple <string, long> >();
foreach (var data in entities)
{
var entity = data.Item1;
var doc = data.Item2;
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var resOpt = new ResolveOptions {
WrtDoc = doc, Directives = directives
};
var watch = Stopwatch.StartNew();
await entity.CreateResolvedEntityAsync($"{entity.GetName()}_", resOpt); watch.Stop();
entityResolutionTimes.Add(Tuple.Create(entity.AtCorpusPath, watch.ElapsedMilliseconds));
}
entityResolutionTimes.Sort((lhs, rhs) =>
{
var diff = rhs.Item2 - lhs.Item2;
return(diff == 0 ? 0 : diff < 0 ? -1 : 1);
});
foreach (var data in entityResolutionTimes)
{
Trace.WriteLine($"{data.Item1}:{data.Item2}");
}
Assert.Performance(1000, entityResolutionTimes[0].Item2);
var total = entityResolutionTimes.Sum(data => data.Item2);
Assert.Performance(5000, total);
}
/// <summary>
/// Resolve the entities in the given manifest.
/// </summary>
/// <param name="testName">The name of the test. It is used to decide the path of input / output files. </param>
/// <parameter name="manifestName">The name of the manifest to be used. </parameter>
/// <returns> The resolved entities. </returns>
private async Task <string> ResolveEnvironment(string testName, string manifestName)
{
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
var testLocalAdapter = this.CreateStorageAdapterConfigForTest(testName);
cdmCorpus.Storage.Mount("local", testLocalAdapter);
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{manifestName}.manifest.cdm.json");
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
return(await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher()));
}
public async Task TestResolveTestCorpus()
{
Assert.IsTrue(Directory.Exists(Path.GetFullPath(SchemaDocsPath)), "SchemaDocsRoot not found!!!");
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(SchemaDocsPath));
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(TestHelper.CdmStandardSchemaPath) as CdmManifestDefinition;
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var allResolved = await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher());
Assert.IsTrue(!string.IsNullOrWhiteSpace(allResolved));
}
/// <summary>
/// Creates a resolved copy of the manifest.
/// newEntityDocumentNameFormat specifies a pattern to use when creating documents for resolved entites.
/// The default is "{f}resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest.
/// Every instance of the string {n} is replaced with the entity name from the source manifest.
/// Every instance of the string {f} is replaced with the folder path from the source manifest to the source entity
/// (if there is one that is possible as a relative location, else nothing).
/// </summary>
/// <param name="newManifestName"></param>
/// <param name="newEntityDocumentNameFormat"></param>
/// <param name="Directives"></param>
/// <returns></returns>
public async Task <CdmManifestDefinition> CreateResolvedManifestAsync(string newManifestName, string newEntityDocumentNameFormat, AttributeResolutionDirectiveSet Directives = null)
{
using (Logger.EnterScope(nameof(CdmManifestDefinition), Ctx, nameof(CreateResolvedManifestAsync)))
{
if (this.Entities == null)
{
return(null);
}
if (this.Folder == null)
{
Logger.Error(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, CdmLogCode.ErrResolveManifestFailed, this.ManifestName);
return(null);
}
if (newEntityDocumentNameFormat == null)
{
newEntityDocumentNameFormat = "{f}resolved/{n}.cdm.json";
}
else if (newEntityDocumentNameFormat == "") // for back compat
{
newEntityDocumentNameFormat = "{n}.cdm.json";
}
else if (!newEntityDocumentNameFormat.Contains("{n}")) // for back compat
{
newEntityDocumentNameFormat = newEntityDocumentNameFormat + "/{n}.cdm.json";
}
string sourceManifestPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.AtCorpusPath, this);
string sourceManifestFolderPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.Folder.AtCorpusPath, this);
int resolvedManifestPathSplit = newManifestName.LastIndexOf("/") + 1;
CdmFolderDefinition resolvedManifestFolder;
if (resolvedManifestPathSplit > 0)
{
var resolvedManifestPath = newManifestName.Substring(0, resolvedManifestPathSplit);
var newFolderPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(resolvedManifestPath, this);
resolvedManifestFolder = await this.Ctx.Corpus.FetchObjectAsync <CdmFolderDefinition>(newFolderPath);
if (resolvedManifestFolder == null)
{
Logger.Error(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, CdmLogCode.ErrResolveFolderNotFound, newFolderPath);
return(null);
}
newManifestName = newManifestName.Substring(resolvedManifestPathSplit);
}
else
{
resolvedManifestFolder = this.Owner as CdmFolderDefinition;
}
Logger.Debug(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, $"resolving manifest {sourceManifestPath}");
// Using the references present in the resolved entities, get an entity
// create an imports doc with all the necessary resolved entity references and then resolve it
// sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name
if (newManifestName.ToLowerInvariant().EndsWith(".manifest.cdm.json"))
{
newManifestName = newManifestName.Substring(0, newManifestName.Length - ".manifest.cdm.json".Length);
}
var resolvedManifest = new CdmManifestDefinition(this.Ctx, newManifestName);
// bring over any imports in this document or other bobbles
resolvedManifest.Schema = this.Schema;
resolvedManifest.Explanation = this.Explanation;
resolvedManifest.DocumentVersion = this.DocumentVersion;
foreach (CdmImport imp in this.Imports)
{
resolvedManifest.Imports.Add((CdmImport)imp.Copy());
}
// add the new document to the folder
if (resolvedManifestFolder.Documents.Add(resolvedManifest) == null)
{
// when would this happen?
return(null);
}
foreach (var entity in this.Entities)
{
var entDef = await this.GetEntityFromReference(entity, this);
if (entDef == null)
{
Logger.Error(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, CdmLogCode.ErrResolveEntityRefError);
return(null);
}
if (entDef.InDocument.Folder == null)
{
Logger.Error(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, CdmLogCode.ErrDocIsNotFolder, entDef.EntityName);
return(null);
}
// get the path from this manifest to the source entity. this will be the {f} replacement value
string sourceEntityFullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(entDef.InDocument.Folder.AtCorpusPath, this);
string f = "";
if (sourceEntityFullPath.StartsWith(sourceManifestFolderPath))
{
f = sourceEntityFullPath.Substring(sourceManifestFolderPath.Length);
}
string newDocumentFullPath = newEntityDocumentNameFormat.Replace("{n}", entDef.EntityName);
newDocumentFullPath = newDocumentFullPath.Replace("{f}", f);
newDocumentFullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(newDocumentFullPath, this);
int newDocumentPathSplit = newDocumentFullPath.LastIndexOf("/") + 1;
string newDocumentPath = newDocumentFullPath.Substring(0, newDocumentPathSplit);
string newDocumentName = newDocumentFullPath.Substring(newDocumentPathSplit);
// make sure the new folder exists
var folder = await this.Ctx.Corpus.FetchObjectAsync <CdmFolderDefinition>(newDocumentPath) as CdmFolderDefinition;
if (folder == null)
{
Logger.Error(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, CdmLogCode.ErrResolveFolderNotFound, newDocumentPath);
return(null);
}
// Next create the resolved entity
AttributeResolutionDirectiveSet withDirectives = Directives != null ? Directives : this.Ctx.Corpus.DefaultResolutionDirectives;
var resOpt = new ResolveOptions
{
WrtDoc = entDef.InDocument,
Directives = withDirectives?.Copy()
};
Logger.Debug(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, $"resolving entity {sourceEntityFullPath} to document {newDocumentFullPath}");
var resolvedEntity = await entDef.CreateResolvedEntityAsync(entDef.EntityName, resOpt, folder, newDocumentName);
if (resolvedEntity == null)
{
// Fail all resolution, if any one entity resolution fails
return(null);
}
var result = entity.Copy(resOpt) as CdmEntityDeclarationDefinition;
if (result.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
result.EntityPath = this.Ctx.Corpus.Storage.CreateRelativeCorpusPath(resolvedEntity.AtCorpusPath, resolvedManifest) ?? result.AtCorpusPath;
}
resolvedManifest.Entities.Add(result);
}
Logger.Debug(this.Ctx as ResolveContext, Tag, nameof(CreateResolvedManifestAsync), this.AtCorpusPath, "calculating relationships");
// calculate the entity graph for just this manifest and any submanifests
await this.Ctx.Corpus.CalculateEntityGraphAsync(resolvedManifest);
// stick results into the relationships list for the manifest
// only put in relationships that are between the entities that are used in the manifest
await resolvedManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.Exclusive);
// needed until Matt's changes with collections where I can propigate
resolvedManifest.IsDirty = true;
return(resolvedManifest);
}
}
public static void PersistCorpusFolder(string rootPath, CdmFolderDefinition cdmFolder, AttributeResolutionDirectiveSet directiveSet, CopyOptions options = null)
{
if (cdmFolder != null)
{
string folderPath = rootPath + cdmFolder.FolderPath;
Directory.CreateDirectory(folderPath);
if (cdmFolder.Documents != null)
{
cdmFolder.Documents.AllItems.ForEach(doc =>
{
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = doc, Directives = directiveSet
};
PersistDocument(rootPath, resOpt, options);
});
}
if (cdmFolder.ChildFolders != null)
{
cdmFolder.ChildFolders.AllItems.ForEach(f =>
{
PersistCorpusFolder(rootPath, f, directiveSet, options);
});
}
}
}
/// Creates a resolved copy of the manifest.
/// newEntityDocumentNameFormat specifies a pattern to use when creating documents for resolved entites.
/// The default is "{f}resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest.
/// Every instance of the string {n} is replaced with the entity name from the source manifest.
/// Every instance of the string {f} is replaced with the folder path from the source manifest to the source entity
/// (if there is one that is possible as a relative location, else nothing).
public async Task <CdmManifestDefinition> CreateResolvedManifestAsync(string newManifestName, string newEntityDocumentNameFormat, AttributeResolutionDirectiveSet Directives = null)
{
if (this.Entities == null)
{
return(null);
}
if (this.Folder == null)
{
Logger.Error(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $"Cannot resolve the manifest '{this.ManifestName}' because it has not been added to a folder", nameof(CreateResolvedManifestAsync));
return(null);
}
if (newEntityDocumentNameFormat == null)
{
newEntityDocumentNameFormat = "{f}resolved/{n}.cdm.json";
}
else if (newEntityDocumentNameFormat == "") // for back compat
{
newEntityDocumentNameFormat = "{n}.cdm.json";
}
else if (!newEntityDocumentNameFormat.Contains("{n}")) // for back compat
{
newEntityDocumentNameFormat = newEntityDocumentNameFormat + "/{n}.cdm.json";
}
string sourceManifestPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.AtCorpusPath, this);
string sourceManifestFolderPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(this.Folder.AtCorpusPath, this);
int resolvedManifestPathSplit = newManifestName.LastIndexOf("/") + 1;
CdmFolderDefinition resolvedManifestFolder;
if (resolvedManifestPathSplit > 0)
{
var resolvedManifestPath = newManifestName.Substring(0, resolvedManifestPathSplit);
var newFolderPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(resolvedManifestPath, this);
resolvedManifestFolder = await this.Ctx.Corpus.FetchObjectAsync <CdmFolderDefinition>(newFolderPath);
if (resolvedManifestFolder == null)
{
Logger.Error(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $"New folder for manifest not found {newFolderPath}", nameof(CreateResolvedManifestAsync));
return(null);
}
newManifestName = newManifestName.Substring(resolvedManifestPathSplit);
}
else
{
resolvedManifestFolder = this.Owner as CdmFolderDefinition;
}
Logger.Debug(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $"resolving manifest {sourceManifestPath}", nameof(CreateResolvedManifestAsync));
// Using the references present in the resolved entities, get an entity
// create an imports doc with all the necessary resolved entity references and then resolve it
var resolvedManifest = new CdmManifestDefinition(this.Ctx, newManifestName);
// add the new document to the folder
if (resolvedManifestFolder.Documents.Add(resolvedManifest) == null)
{
// when would this happen?
return(null);
}
foreach (var entity in this.Entities)
{
var entDef = await this.GetEntityFromReference(entity, this);
if (entDef == null)
{
Logger.Error(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $"Unable to get entity from reference", nameof(CreateResolvedManifestAsync));
return(null);
}
if (entDef.InDocument.Folder == null)
{
Logger.Error(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $"The document containing the entity '{entDef.EntityName}' is not in a folder", nameof(CreateResolvedManifestAsync));
return(null);
}
// get the path from this manifest to the source entity. this will be the {f} replacement value
string sourceEntityFullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(entDef.InDocument.Folder.AtCorpusPath, this);
string f = "";
if (sourceEntityFullPath.StartsWith(sourceManifestFolderPath))
{
f = sourceEntityFullPath.Substring(sourceManifestFolderPath.Length);
}
string newDocumentFullPath = newEntityDocumentNameFormat.Replace("{n}", entDef.EntityName);
newDocumentFullPath = newDocumentFullPath.Replace("{f}", f);
newDocumentFullPath = this.Ctx.Corpus.Storage.CreateAbsoluteCorpusPath(newDocumentFullPath, this);
int newDocumentPathSplit = newDocumentFullPath.LastIndexOf("/") + 1;
string newDocumentPath = newDocumentFullPath.Substring(0, newDocumentPathSplit);
string newDocumentName = newDocumentFullPath.Substring(newDocumentPathSplit);
// make sure the new folder exists
var folder = await this.Ctx.Corpus.FetchObjectAsync <CdmFolderDefinition>(newDocumentPath) as CdmFolderDefinition;
if (folder == null)
{
Logger.Error(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $"New folder not found {newDocumentPath}", nameof(CreateResolvedManifestAsync));
return(null);
}
// Next create the resolved entity
AttributeResolutionDirectiveSet withDirectives = Directives != null ? Directives : this.Ctx.Corpus.DefaultResolutionDirectives;
var resOpt = new ResolveOptions
{
WrtDoc = entDef.InDocument,
Directives = withDirectives?.Copy()
};
Logger.Debug(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $" resolving entity {sourceEntityFullPath} to document {newDocumentFullPath}", nameof(CreateResolvedManifestAsync));
var resolvedEntity = await entDef.CreateResolvedEntityAsync(entDef.EntityName, resOpt, folder, newDocumentName);
if (resolvedEntity == null)
{
// Fail all resolution, if any one entity resolution fails
return(null);
}
var result = entity.Copy(resOpt) as CdmEntityDeclarationDefinition;
if (result.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
result.EntityPath = this.Ctx.Corpus.Storage.CreateRelativeCorpusPath(resolvedEntity.AtCorpusPath, resolvedManifest) ?? result.AtCorpusPath;
}
resolvedManifest.Entities.Add(result);
}
Logger.Debug(nameof(CdmManifestDefinition), this.Ctx as ResolveContext, $" calculating relationships", nameof(CreateResolvedManifestAsync));
// calculate the entity graph for just this manifest and any submanifests
await this.Ctx.Corpus.CalculateEntityGraphAsync(resolvedManifest);
// stick results into the relationships list for the manifest
// only put in relationships that are between the entities that are used in the manifest
await resolvedManifest.PopulateManifestRelationshipsAsync(CdmRelationshipDiscoveryStyle.Exclusive);
// needed until Matt's changes with collections where I can propigate
resolvedManifest.IsDirty = true;
return(resolvedManifest);
}
/// <summary>
/// Get the text version of all the resolved entities.
/// </summary>
/// <param name="cdmCorpus"> The CDM corpus. </param>
/// <param name="directives"> The directives to use while getting the resolved entities. </param>
/// <param name="manifest"> The manifest to be resolved. </param>
/// <param name="spew"> The object used to store the text to be returned. </param>
/// <returns> The text version of the resolved entities. (it's in a form that facilitates debugging) </returns>
internal static async Task <string> ListAllResolved(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives, CdmManifestDefinition manifest, StringSpewCatcher spew = null)
{
// make sure the corpus has a set of default artifact attributes
await cdmCorpus.PrepareArtifactAttributesAsync();
var seen = new HashSet <string>();
Func <CdmManifestDefinition, Task> seekEntities = null;
seekEntities = async(CdmManifestDefinition f) =>
{
if (f.Entities != null)
{
if (spew != null)
{
spew.SpewLine(f.FolderPath);
}
foreach (CdmEntityDeclarationDefinition entity in f.Entities)
{
string corpusPath;
CdmEntityDeclarationDefinition ent = entity;
CdmObject currentFile = f;
while (ent is CdmReferencedEntityDeclarationDefinition)
{
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(ent.EntityPath, currentFile);
ent = await cdmCorpus.FetchObjectAsync <CdmReferencedEntityDeclarationDefinition>(corpusPath);
currentFile = ent;
}
corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(((CdmLocalEntityDeclarationDefinition)ent).EntityPath, currentFile);
ResolveOptions resOpt = new ResolveOptions()
{
ImportsLoadStrategy = ImportsLoadStrategy.Load
};
CdmEntityDefinition newEnt = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(corpusPath, null, resOpt);
resOpt.WrtDoc = newEnt.InDocument;
resOpt.Directives = directives;
ResolvedEntity resEnt = new ResolvedEntity(resOpt, newEnt);
if (spew != null)
{
resEnt.Spew(resOpt, spew, " ", true);
}
}
}
if (f.SubManifests != null)
{
// folder.SubManifests.ForEach(async f =>
foreach (CdmManifestDeclarationDefinition subManifest in f.SubManifests)
{
string corpusPath = cdmCorpus.Storage.CreateAbsoluteCorpusPath(subManifest.Definition, f);
await seekEntities(await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>(corpusPath));
}
}
};
await seekEntities(manifest);
if (spew != null)
{
return(spew.GetContent());
}
return("");
}
public async Task ResolveEntitiesWrt()
{
var cdmCorpus = new CdmCorpusDefinition();
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestResolveEntitiesWrt");
((CdmCorpusDefinition)cdmCorpus).RootPath = testInputPath;
cdmCorpus.Storage.Mount("local", new LocalAdapter(testInputPath));
cdmCorpus.Storage.DefaultNamespace = "local";
var entities = await this.GetAllEntities(cdmCorpus);
var incomingReferences = new Dictionary <CdmEntityDefinition, List <CdmEntityDefinition> >();
foreach (var data in entities)
{
var entity = data.Item1;
incomingReferences[entity] = new List <CdmEntityDefinition>();
}
// Start by populating all the incoming references to the entities
foreach (var data in entities)
{
var entity = data.Item1;
var doc = data.Item2;
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var resOpt = new ResolveOptions {
WrtDoc = doc, Directives = directives
};
var resolvedEntity = await entity.CreateResolvedEntityAsync($"{entity.GetName()}_", resOpt);
var references = await this.GetEntityReferences(resolvedEntity, resOpt, cdmCorpus);
if (references.Count > 0)
{
foreach (var reference in references)
{
incomingReferences[reference].Add(entity);
}
}
}
// Next resolve the entity with all of it's incoming references and save the times
var entityResolutionTimes = new List <Tuple <string, long> >();
foreach (var data in entities)
{
var entity = data.Item1;
var doc = data.Item2;
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var resOpt = new ResolveOptions {
WrtDoc = doc, Directives = directives
};
var watch = Stopwatch.StartNew();
var references = incomingReferences[entity];
foreach (var entRef in references)
{
await entRef.CreateResolvedEntityAsync($"{entRef.GetName()}_", resOpt);
}
watch.Stop();
entityResolutionTimes.Add(Tuple.Create(entity.AtCorpusPath, watch.ElapsedMilliseconds));
}
entityResolutionTimes.Sort((lhs, rhs) =>
{
var diff = rhs.Item2 - lhs.Item2;
return(diff == 0 ? 0 : diff < 0 ? -1 : 1);
});
foreach (var data in entityResolutionTimes)
{
Trace.WriteLine($"{data.Item1}:{data.Item2}");
}
Assert.Performance(4000, entityResolutionTimes[0].Item2);
var total = entityResolutionTimes.Sum(data => data.Item2);
Assert.Performance(5000, total);
}
static async Task Main(string[] args)
{
string pathToDocRoot;
string docGroup;
CdmManifestDefinition manifest;
string testEnt;
bool testCorpus = false;
bool resolveEnt = false;
bool spewAll = true;
bool rePersist = false;
CdmCorpusDefinition cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
if (testCorpus)
{
pathToDocRoot = "../../../../../../../../CDM.Tools.Internal/TestCorpus";
//pathToDocRoot = "../perfTestCorpus";
//docGroup = "E2EResolution";
//docGroup = "POVResolution";
//docGroup = "MiniDyn";
//docGroup = "composites";
//docGroup = "KnowledgeGraph";
//docGroup = "overrides";
docGroup = "webClicks";
//testEnt = "/E2EResolution/E2EArrayOne.cdm.json/E2EArrayOne";
//testEnt = "/MiniDyn/sub/Lead.cdm.json/Lead";
// testEnt = "/POVResolution/sub1/Main.cdm.json/Main";
testEnt = "local:/MiniDyn/Account.cdm.json/Account";
}
else
{
pathToDocRoot = "../../../../../../../../schemaDocuments";
testEnt = "local:/core/applicationCommon/foundationCommon/crmCommon/Account.cdm.json/Account";
docGroup = "standards";
}
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathToDocRoot));
manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{docGroup}.manifest.cdm.json");
string version = "";
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Progress);
Console.WriteLine("reading source files");
if (resolveEnt)
{
// AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet<string> { "normalized", "xstructured", "referenceOnly" });
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var ent = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(testEnt);
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = ent.InDocument, Directives = directives
};
var x = await ent.CreateResolvedEntityAsync("RESOLVED_KILL", resOpt);
resOpt.WrtDoc = x.InDocument;
CommonDataModelLoader.PersistDocument(cdmCorpus.RootPath, resOpt, new CopyOptions {
StringRefs = false, RemoveSingleRowLocalizedTableTraits = true
});
}
if (spewAll)
{
Console.WriteLine("list all resolved");
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "xstructured", "referenceOnly"
});
await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher());
}
if (rePersist)
{
Console.WriteLine("persist corpus");
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "xstructured", "referenceOnly"
});
}
//ListAllTraits(cdmCorpus);
Console.WriteLine("done");
Console.ReadLine();
}
public static void ListAllTraits(CdmCorpusDefinition cdmCorpus, AttributeResolutionDirectiveSet directives)
{
ISet <string> seen = new HashSet <string>();
Action <CdmFolderDefinition> seekTraits = null;
seekTraits = (folder) =>
{
if (!string.IsNullOrEmpty(folder.Name) && folder.Documents != null && folder.Documents.Count > 0)
{
if (folder.Documents != null)
{
folder.Documents.AllItems.ForEach(doc =>
{
if (doc.Definitions?.Count > 0)
{
foreach (var def in doc.Definitions)
{
if (def.ObjectType == CdmObjectType.EntityDef)
{
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = doc, Directives = directives
};
var ent = (def as CdmEntityDefinition);
var rtsEnt = ent.FetchResolvedTraits(resOpt);
rtsEnt.Set.ForEach(rt =>
{
string rtName = rt.TraitName;
if (!seen.Contains(rtName))
{
Console.WriteLine(rtName);
seen.Add(rtName);
}
});
var ras = ent.FetchResolvedAttributes(resOpt);
ras.Set.ForEach(ra =>
{
var rtsAtt = ra.ResolvedTraits;
rtsAtt.Set.ForEach(rt =>
{
string rtName = rt.TraitName;
if (!seen.Contains(rtName))
{
Console.WriteLine(rtName);
seen.Add(rtName);
}
});
});
}
}
}
});
}
}
;
if (folder.ChildFolders != null)
{
folder.ChildFolders.AllItems.ForEach(f =>
{
if (seekTraits != null)
{
seekTraits(f);
}
});
}
};
seekTraits(cdmCorpus);
}
