public async Task TestResolveTestCorpus()
{
Assert.IsTrue(Directory.Exists(Path.GetFullPath(SchemaDocsPath)), "SchemaDocsRoot not found!!!");
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(SchemaDocsPath));
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/standards.manifest.cdm.json") as CdmManifestDefinition;
var directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var allResolved = await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher());
Assert.IsTrue(!string.IsNullOrWhiteSpace(allResolved));
}
public async Task TestReadingIsPrimaryKeyConstructedFromPurpose()
{
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestReadingIsPrimaryKeyConstructedFromPurpose");
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(testInputPath));
corpus.Storage.DefaultNamespace = "local";
CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>("local:/TeamMembership.cdm.json/TeamMembership");
CdmAttributeGroupReference attributeGroupRef = (CdmAttributeGroupReference)entity.Attributes[0];
CdmAttributeGroupDefinition attributeGroup = (CdmAttributeGroupDefinition)attributeGroupRef.ExplicitReference;
CdmTypeAttributeDefinition typeAttribute = (CdmTypeAttributeDefinition)attributeGroup.Members[0];
Assert.AreEqual("identifiedBy", typeAttribute.Purpose.NamedReference);
Assert.IsTrue((bool)typeAttribute.IsPrimaryKey);
}
public async Task TestLoadsAndSetsTimesCorrectly()
{
var inputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestLoadsAndSetsTimesCorrectly");
var timeBeforeLoad = DateTime.Now;
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("someNamespace", new LocalAdapter(inputPath));
cdmCorpus.Storage.Mount("local", new LocalAdapter(inputPath));
cdmCorpus.Storage.Unmount("cdm");
cdmCorpus.Storage.DefaultNamespace = "local";
var cdmManifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("someNamespace:/default.manifest.cdm.json");
var statusTimeAtLoad = cdmManifest.LastFileStatusCheckTime;
// hard coded because the time comes from inside the file
Assert.AreEqual(TimeUtils.GetFormattedDateString(statusTimeAtLoad), "2019-02-01T15:36:19.410Z");
Assert.IsNotNull(cdmManifest._fileSystemModifiedTime);
Assert.IsTrue(cdmManifest._fileSystemModifiedTime < timeBeforeLoad);
System.Threading.Thread.Sleep(100);
await cdmManifest.FileStatusCheckAsync();
Assert.IsTrue(cdmManifest.LastFileStatusCheckTime > timeBeforeLoad);
Assert.IsTrue(cdmManifest.LastFileStatusCheckTime > statusTimeAtLoad);
Assert.AreEqual(cdmManifest.SubManifests.Count, 1);
Assert.IsTrue(cdmManifest.SubManifests.AllItems[0].LastFileStatusCheckTime > timeBeforeLoad);
Assert.AreEqual(cdmManifest.Entities.Count, 1);
Assert.AreEqual(cdmManifest.Entities.AllItems[0].DataPartitions.Count, 1);
var entity = cdmManifest.Entities.AllItems[0];
var subManifest = cdmManifest.SubManifests.AllItems[0] as CdmManifestDeclarationDefinition;
var maxTime = TimeUtils.MaxTime(entity.LastFileModifiedTime, subManifest.LastFileModifiedTime);
Assert.AreEqual(TimeUtils.GetFormattedDateString(cdmManifest.LastChildFileModifiedTime), TimeUtils.GetFormattedDateString(maxTime));
}
/// <summary>
/// Creates a Manifest used for the tests.
/// </summary>
/// <param name="localRootPath">A string used as root path for "local" namespace.</param>
/// <returns>Created Manifest.</returns>
private CdmManifestDefinition GenerateManifest(string localRootPath)
{
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(localRootPath));
// add cdm namespace
cdmCorpus.Storage.Mount("cdm", new LocalAdapter("C:\\Root\\Path"));
var manifest = new CdmManifestDefinition(cdmCorpus.Ctx, "manifest");
manifest.FolderPath = "/";
manifest.Namespace = "local";
return(manifest);
}
public void TestNonExistingAdapterNamespace()
{
var localAdapter = this.CreateStorageAdapterForTest("TestNonExistingAdapterNamespace");
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new Utilities.EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
// Register it as a 'local' adapter.
cdmCorpus.Storage.Mount("erp", localAdapter);
// Set local as our default.
cdmCorpus.Storage.DefaultNamespace = "erp";
// Load a manifest that is trying to import from 'cdm' namespace.
// The manifest does't exist since the import couldn't get resolved,
// so the error message will be logged and the null value will be propagated back to a user.
Assert.IsNull(cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("erp.missingImportManifest.cdm").Result);
}
public async Task TestAllOperations()
{
string testName = "TestAllOperations";
string entityName = testName;
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
if (!message.Contains("Projection operation not implemented yet."))
{
Assert.Fail($"'Some unexpected failure - {message}!");
}
}
}, CdmStatusLevel.Error);
await ProjectionTestUtils.LoadEntityForResolutionOptionAndSave(corpus, testName, testsSubpath, entityName, new List <string> {
"referenceOnly"
});
}
public async Task TestWithNesting()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId);
var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy");
// Test resolving a manifest (added to a folder) with an entity in it, this should collect messages from
// CreateResolvedManifestAsync and CreateResolvedEntityAsync functions
corpus.Storage.FetchRootFolder("local").Documents.Add(manifest);
var entity1 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity1");
var someAttrib1 = corpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "someAttrib1", false);
someAttrib1.DataType = corpus.MakeRef <CdmDataTypeReference>(CdmObjectType.DataTypeRef, "entityId", true);
entity1.Attributes.Add(someAttrib1);
var entity1Doc = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "MyEntity1.cdm.json");
entity1Doc.Definitions.Add(entity1);
corpus.Storage.FetchRootFolder("local").Documents.Add(entity1Doc);
manifest.Entities.Add(entity1);
await manifest.CreateResolvedManifestAsync("new dummy 2", null);
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
// We check that there first event recorded was an error from the nested function
Assert.IsTrue(corpus.Ctx.Events[0]["level"] == "Error", "The first recorded event level should have been 'Error'");
Assert.IsTrue(corpus.Ctx.Events[0]["message"] == "Unable to resolve the reference 'entityId' to a known object",
"The first recorded event message should have specified that 'entityId' was not resolved");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Keep for debugging
//corpus.Ctx.Events.ForEach(logEntry => {
// logEntry.ToList().ForEach(logEntryPair => Console.WriteLine($"{logEntryPair.Key}={logEntryPair.Value}"));
// Console.WriteLine();
//});
}
public async Task TestLoadingInvalidModelJsonAndOdiJsonName()
{
var testName = "TestLoadingInvalidModelJsonAndOdiJsonName";
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, testName);
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(testInputPath));
corpus.Storage.DefaultNamespace = "local";
// We are trying to load a file with an invalid name, so FetchObjectAsync() should just return null.
var invalidModelJson = await corpus.FetchObjectAsync <CdmManifestDefinition>("test.model.json");
Assert.IsNull(invalidModelJson);
var invalidOdiJson = await corpus.FetchObjectAsync <CdmManifestDefinition>("test.odi.json");
Assert.IsNull(invalidOdiJson);
}
public void TestManifestCannotAddEntityDefinitionWithoutCreatingDocument()
{
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
var callback = new EventCallback();
var functionWasCalled = false;
var functionParameter1 = CdmStatusLevel.Info;
string functionParameter2 = null;
callback.Invoke = (CdmStatusLevel statusLevel, string message1) =>
{
functionWasCalled = true;
functionParameter1 = statusLevel;
functionParameter2 = message1;
};
cdmCorpus.SetEventCallback(callback);
cdmCorpus.Storage.Mount("local", new LocalAdapter("C:\\Root\\Path"));
var manifest = new CdmManifestDefinition(cdmCorpus.Ctx, "manifest");
manifest.FolderPath = "/";
manifest.Namespace = "local";
var entity = new CdmEntityDefinition(manifest.Ctx, "entityName", null);
var corpus = new CdmCorpusDefinition();
manifest.Entities.Add(entity);
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("Expected entity to have an \"Owner\" document set. Cannot create entity declaration to add to manifest."));
Assert.AreEqual(0, manifest.Entities.Count);
}
public async Task TestAddAttrGrpOnEntAttrProj()
{
string testName = nameof(TestAddAttrGrpOnEntAttrProj);
string entityName = "NewPerson";
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, testName);
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
if (!message.Contains("CdmOperationAddArtifactAttribute | Operation AddArtifactAttribute is not supported on an attribute group yet."))
{
Assert.Fail($"Some unexpected failure - {message}!");
}
}
}, CdmStatusLevel.Warning);
CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}");
CdmEntityDefinition resolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> {
});
}
public async Task TestFileStatusCheckOnNullLocation()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestFileStatusCheckOnNullLocation");
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
Assert.AreEqual(statusLevel, CdmStatusLevel.Error, "Error level message should have been reported");
Assert.IsTrue(
message == "StorageManager | The object path cannot be null or empty. | CreateAbsoluteCorpusPath" ||
message == "CdmCorpusDefinition | The object path cannot be null or empty. | GetLastModifiedTimeAsyncFromPartitionPath",
"Unexpected error message received");
}
}, CdmStatusLevel.Warning);
// Create manifest
var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "TestModel");
corpus.Storage.FetchRootFolder("local").Documents.Add(manifest);
// Create entity
var entDoc = corpus.Storage.FetchRootFolder("local").Documents.Add("MyEntityDoc.cdm.json");
var entDef = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity");
entDoc.Definitions.Add(entDef);
var entDecl = manifest.Entities.Add(entDef);
// Create partition
var part = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "MyPartition");
entDecl.DataPartitions.Add(part);
// This should not throw exception
await manifest.FileStatusCheckAsync();
}
public async Task TestEntityWithSameImportsAsync()
{
var localAdapter = this.CreateStorageAdapterForTest("TestEntityWithSameImportsAsync");
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new Utilities.EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", localAdapter);
var doc = await cdmCorpus.FetchObjectAsync <CdmDocumentDefinition>("local:/multipleImports.cdm.json");
Assert.IsNotNull(doc);
Assert.AreEqual(doc.Imports.Count, 2);
var firstImport = (doc.Imports.AllItems[0] as CdmImport).Doc;
Assert.AreEqual(firstImport.Name, "missingImport.cdm.json");
Assert.AreEqual(firstImport.Imports.Count, 1);
var secondImport = (doc.Imports.AllItems[1] as CdmImport).Doc;
Assert.AreEqual(secondImport.Name, "notMissing.cdm.json");
}
public void TestStorageManagerEvents()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Info, DummyCorrelationId);
corpus.Storage.Mount("dummy", null);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullAdapter);
corpus.Storage.Unmount("nothing");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.WarnStorageRemoveAdapterFailed);
// No errors/warnings expected here
corpus.Storage.FetchRootFolder(null);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace);
corpus.Storage.AdapterPathToCorpusPath("Test");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageInvalidAdapterPath);
corpus.Storage.CorpusPathToAdapterPath("unknown:/Test");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageAdapterNotFound);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNamespaceNotRegistered);
corpus.Storage.CreateAbsoluteCorpusPath(null);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath);
corpus.Storage.CreateRelativeCorpusPath(null);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath);
}
public async Task TestInvalidJson()
{
var testInputPath = TestHelper.GetInputFolderPath(testsSubpath, nameof(TestInvalidJson));
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(testInputPath));
corpus.Storage.DefaultNamespace = "local";
CdmManifestDefinition invalidManifest = null;
try
{
invalidManifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("local:/invalidManifest.manifest.cdm.json");
}
catch (Exception)
{
Assert.Fail("Error should not be thrown when input json is invalid.");
}
Assert.IsNull(invalidManifest);
}
public async Task TestSavingInvalidModelJsonName()
{
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Unmount("cdm");
corpus.Storage.DefaultNamespace = "local";
var manifest = new CdmManifestDefinition(corpus.Ctx, "manifest");
corpus.Storage.FetchRootFolder("local").Documents.Add(manifest);
ConcurrentDictionary <string, string> allDocs = new ConcurrentDictionary <string, string>();
var testAdapter = new TestStorageAdapter(allDocs);
corpus.Storage.SetAdapter("local", testAdapter);
var newManifestFromModelJsonName = "my.model.json";
await manifest.SaveAsAsync(newManifestFromModelJsonName, true);
// TODO: because we can load documents properly now, SaveAsAsync returns false. Will check the value returned from SaveAsAsync() when the problem is solved
Assert.IsFalse(allDocs.ContainsKey($"/{newManifestFromModelJsonName}"));
}
/// <summary>
/// Gets local corpus.
/// </summary>
/// <returns>The <see cref="CdmCorpusDef"/>. </returns>
protected CdmCorpusDefinition GetLocalCorpus(string testFilesRoot)
{
Assert.IsTrue(Directory.Exists(Path.GetFullPath(SchemaDocsRoot)), "SchemaDocsRoot not found!!!");
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(testFilesRoot));
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsRoot));
var hosts = new Dictionary <string, string>();
hosts.Add("contoso", "http://contoso.com");
cdmCorpus.Storage.Mount("remote", new RemoteAdapter()
{
Hosts = hosts
});
return(cdmCorpus);
}
public async Task TestStartGTEndOrdinal()
{
string testName = "TestStartGTEndOrdinal";
string entityName = "ThreeMusketeers";
CdmCorpusDefinition corpus = ProjectionTestUtils.GetLocalCorpus(testsSubpath, testName);
// A warning should be logged when startOrdinal > endOrdinal
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
if (!message.Contains("startOrdinal 2 should not be greater than endOrdinal 0"))
{
Assert.Fail($"'Some unexpected failure - {message}!");
}
}
}, CdmStatusLevel.Warning);
foreach (List <string> resOpt in resOptsCombinations)
{
await ProjectionTestUtils.LoadEntityForResolutionOptionAndSave(corpus, testName, testsSubpath, entityName, resOpt);
}
CdmEntityDefinition entity = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/{entityName}.cdm.json/{entityName}");
CdmEntityDefinition resolvedEntity = await ProjectionTestUtils.GetResolvedEntity(corpus, entity, new List <string> {
});
// Original set of attributes: ["name", "age", "address"]
// Expand 2...0, renameFormat = {m}{o}
// No array expansion happens here so the input just passes through
Assert.AreEqual(3, resolvedEntity.Attributes.Count);
Assert.AreEqual("name", (resolvedEntity.Attributes[0] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("age", (resolvedEntity.Attributes[1] as CdmTypeAttributeDefinition).Name);
Assert.AreEqual("address", (resolvedEntity.Attributes[2] as CdmTypeAttributeDefinition).Name);
}
/// <summary>
/// Test automatic mounting of adls adapter in syms if does not exist.
/// </summary>
internal async Task TestSymsSmartADLSAdapterMountLogic()
{
var symsAdapter = SymsTestHelper.CreateAdapterWithClientId();
var corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("syms", symsAdapter);
var adlsAdapter1 = SymsTestHelper.CreateADLSAdapterWithClientIdWithSharedKey(1);
var adlsAdapter2 = SymsTestHelper.CreateADLSAdapterWithClientIdWithSharedKey(2);
int countAdapterCountBefore = corpus.Storage.NamespaceAdapters.Count;
var manifestReadDatabases = await corpus.FetchObjectAsync <CdmManifestDefinition>($"syms:/databases.manifest.cdm.json");
var manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>($"syms:/{manifestReadDatabases.SubManifests[0].ManifestName}/{manifestReadDatabases.SubManifests[0].ManifestName}.manifest.cdm.json", manifestReadDatabases, null, true);
int countAdapterCountAfter = corpus.Storage.NamespaceAdapters.Count;
Assert.AreEqual(countAdapterCountBefore + 2, countAdapterCountAfter);
Assert.IsNotNull(corpus.Storage.AdapterPathToCorpusPath($"https://{adlsAdapter1.Hostname}{adlsAdapter1.Root}"));
Assert.IsNotNull(corpus.Storage.AdapterPathToCorpusPath($"https://{adlsAdapter2.Hostname}{adlsAdapter2.Root}"));
}
/// <summary>
/// This method runs the tests with a set expected attributes & attribute context values and validated the actual result
/// </summary>
/// <param name="testName"></param>
/// <param name="sourceEntityName"></param>
/// <param name="expectedContext_*">expected attribute context object - for each resolution option combination</param>
/// <param name="expected_*">expected attribute object - for each resolution option combination</param>
/// <returns></returns>
protected static async Task RunTestWithValues(
string testName,
string sourceEntityName,
AttributeContextExpectedValue expectedContext_default,
AttributeContextExpectedValue expectedContext_normalized,
AttributeContextExpectedValue expectedContext_referenceOnly,
AttributeContextExpectedValue expectedContext_structured,
AttributeContextExpectedValue expectedContext_normalized_structured,
AttributeContextExpectedValue expectedContext_referenceOnly_normalized,
AttributeContextExpectedValue expectedContext_referenceOnly_structured,
AttributeContextExpectedValue expectedContext_referenceOnly_normalized_structured,
List <AttributeExpectedValue> expected_default,
List <AttributeExpectedValue> expected_normalized,
List <AttributeExpectedValue> expected_referenceOnly,
List <AttributeExpectedValue> expected_structured,
List <AttributeExpectedValue> expected_normalized_structured,
List <AttributeExpectedValue> expected_referenceOnly_normalized,
List <AttributeExpectedValue> expected_referenceOnly_structured,
List <AttributeExpectedValue> expected_referenceOnly_normalized_structured
)
{
try
{
string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName);
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("localInput", new LocalAdapter(testInputPath));
corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath));
corpus.Storage.DefaultNamespace = "localInput";
CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"localInput:/{sourceEntityName}.cdm.json/{sourceEntityName}") as CdmEntityDefinition;
Assert.IsTrue(srcEntityDef != null);
var resOpt = new ResolveOptions
{
WrtDoc = srcEntityDef.InDocument,
Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
})
};
CdmEntityDefinition resolvedEntityDef = null;
string outputEntityName = string.Empty;
string outputEntityFileName = string.Empty;
string entityFileName = string.Empty;
if (expectedContext_default != null && expected_default != null)
{
entityFileName = "default";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_default, expected_default, resolvedEntityDef);
}
if (expectedContext_normalized != null && expected_normalized != null)
{
entityFileName = "normalized";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_normalized, expected_normalized, resolvedEntityDef);
}
if (expectedContext_referenceOnly != null && expected_referenceOnly != null)
{
entityFileName = "referenceOnly";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_referenceOnly, expected_referenceOnly, resolvedEntityDef);
}
if (expectedContext_structured != null && expected_structured != null)
{
entityFileName = "structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"structured"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_structured, expected_structured, resolvedEntityDef);
}
if (expectedContext_normalized_structured != null && expected_normalized_structured != null)
{
entityFileName = "normalized_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_normalized_structured, expected_normalized_structured, resolvedEntityDef);
}
if (expectedContext_referenceOnly_normalized != null && expected_referenceOnly_normalized != null)
{
entityFileName = "referenceOnly_normalized";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_referenceOnly_normalized, expected_referenceOnly_normalized, resolvedEntityDef);
}
if (expectedContext_referenceOnly_structured != null && expected_referenceOnly_structured != null)
{
entityFileName = "referenceOnly_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "structured"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_referenceOnly_structured, expected_referenceOnly_structured, resolvedEntityDef);
}
if (expectedContext_referenceOnly_normalized_structured != null && expected_referenceOnly_normalized_structured != null)
{
entityFileName = "referenceOnly_normalized_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized", "structured"
});
outputEntityName = $"{sourceEntityName}_Resolved_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt);
ValidateOutputWithValues(expectedContext_referenceOnly_normalized_structured, expected_referenceOnly_normalized_structured, resolvedEntityDef);
}
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public async Task TestWithoutNesting()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId);
// Test fetching an object from invalid namespace results in at least one error message in the recorder
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNamespaceNotRegistered);
// Test fetching a good object, this should leave event recorder empty
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json");
TestNoLogsState(corpus);
// Test saving a manifest to invalid namespace results in at least one error message in the recorder
var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy");
await manifest.SaveAsAsync("foo:/bar", true);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrValdnMissingDoc);
// Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder
await manifest.CreateResolvedManifestAsync("new dummy", null);
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrResolveManifestFailed);
// Test resolving an entity without WRT doc, this should yield at least one error message in the recorder
var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2");
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrDocWrtDocNotfound);
// Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder
await manifest.FileStatusCheckAsync();
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace);
// Repeat the same test but with status level 'None', no events should be recorded
corpus.Ctx.ReportAtLevel = CdmStatusLevel.None;
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
TestNoLogsState(corpus);
// Test checking file status on a data partition
// We're at log level 'Progress', so we get the EnterScope/LeaveScope messages too
corpus.Ctx.ReportAtLevel = CdmStatusLevel.Progress;
var part = corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "part");
await part.FileStatusCheckAsync();
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPathNullObjectPath);
// Test checking file status on a data partition pattern
var refDoc = corpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "RefEntDoc");
var partPattern = corpus.MakeObject <CdmDataPartitionPatternDefinition>(CdmObjectType.DataPartitionPatternDef, "partPattern");
partPattern.InDocument = refDoc;
await partPattern.FileStatusCheckAsync();
TestBasicLogsState(corpus);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrStorageNullNamespace);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrDocAdapterNotFound);
// Test calculating relationships - no errors/warnings
await corpus.CalculateEntityGraphAsync(manifest);
TestBasicLogsState(corpus);
// Test populating relationships in manifest - no errors/warnings
await manifest.PopulateManifestRelationshipsAsync();
TestBasicLogsState(corpus);
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
// set callback to receive error and warning logs.
cdmCorpus.SetEventCallback(new EventCallback
{
Invoke = (level, message) =>
{
Console.WriteLine(message);
}
}, CdmStatusLevel.Warning);
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../../";
string sampleEntityName = "Account";
// Mount it as a local adapter.
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "7-search-partition-pattern/sample-data"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the CDM Standards adapter
// Mount it as the 'cdm' adapter, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract, "tempAbstract.manifest.cdm.json");
// Add an entity named Account from some public standards
Console.WriteLine("Add an entity named Account from some public standards");
var accountDeclarationDefinition = manifestAbstract.Entities.Add(sampleEntityName, "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
// Create a data partition pattern
var dataPartitionPattern = CreatePartitionPatternDefinition(cdmCorpus, "sampleDataPartitionPattern");
dataPartitionPattern.Explanation = "/ capture 4 digits / capture a word / capture one or more digits after the word cohort but before .csv";
Console.WriteLine($" Assign regular expression of the data partition pattern to: {dataPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the data partition pattern to: {dataPartitionPattern.RootLocation}");
// Add the data partition pattern we just created to the entity data partition pattern collection
accountDeclarationDefinition.DataPartitionPatterns.Add(dataPartitionPattern);
// Create incremental partition patterns
var upsertIncrementalPartitionPattern = CreatePartitionPatternDefinition(cdmCorpus, "UpsertPattern", true, true);
AddIncrementalPartitionTrait(upsertIncrementalPartitionPattern, CdmIncrementalPartitionType.Upsert);
Console.WriteLine($"\n Assign regular expression of the first incremental partition pattern to: {upsertIncrementalPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the first incremental partition pattern to: {upsertIncrementalPartitionPattern.RootLocation}");
var deleteIncrementalPartitionPattern = CreatePartitionPatternDefinition(cdmCorpus, "DeletePattern", true);
AddIncrementalPartitionTrait(deleteIncrementalPartitionPattern, CdmIncrementalPartitionType.Delete, "FullDataPattern");
Console.WriteLine($"\n Assign regular expression of the second incremental partition pattern to: {deleteIncrementalPartitionPattern.RegularExpression}");
Console.WriteLine($" Assign root location of the second incremental partition pattern to: {deleteIncrementalPartitionPattern.RootLocation}");
// Add the incremental partition patterns we just created to the entity increment partition pattern collection
accountDeclarationDefinition.IncrementalPartitionPatterns.Add(upsertIncrementalPartitionPattern);
accountDeclarationDefinition.IncrementalPartitionPatterns.Add(deleteIncrementalPartitionPattern);
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestAbstract.Imports.Add("cdm:/foundations.cdm.json");
// Calling FileStatusCheckAsync to pick up the all data partition files which names match the data partition pattern,
// and add them to the entity in the manifest
await manifestAbstract.FileStatusCheckAsync(PartitionFileStatusCheckType.FullAndIncremental);
// List all data partition locations.
Console.WriteLine($"\nlist of all data partition locations for the entity Account matches the data partition pattern:");
foreach (CdmDataPartitionDefinition dataPartition in accountDeclarationDefinition.DataPartitions)
{
Console.WriteLine($" {dataPartition.Location}");
}
// List all incremental partition locations.
Console.WriteLine($"\nlist of all incremental partition locations for the entity Account matches the incremental partition pattern:");
foreach (CdmDataPartitionDefinition incrementalPartition in accountDeclarationDefinition.IncrementalPartitions)
{
Console.WriteLine($" {incrementalPartition.Location}");
}
Console.WriteLine("Resolve the manifest");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", null);
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
// You can save the doc as a model.json format as an option
// await manifestResolved.SaveAsAsync("model.json", true);
}
static async Task Main(string[] args)
{
string pathToDocRoot;
string docGroup;
CdmManifestDefinition manifest;
string testEnt;
bool testCorpus = false;
bool resolveEnt = false;
bool spewAll = true;
bool rePersist = false;
CdmCorpusDefinition cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
if (testCorpus)
{
pathToDocRoot = "../../../../../../../../CDM.Tools.Internal/TestCorpus";
//pathToDocRoot = "../perfTestCorpus";
//docGroup = "E2EResolution";
//docGroup = "POVResolution";
//docGroup = "MiniDyn";
//docGroup = "composites";
//docGroup = "KnowledgeGraph";
//docGroup = "overrides";
docGroup = "webClicks";
//testEnt = "/E2EResolution/E2EArrayOne.cdm.json/E2EArrayOne";
//testEnt = "/MiniDyn/sub/Lead.cdm.json/Lead";
// testEnt = "/POVResolution/sub1/Main.cdm.json/Main";
testEnt = "local:/MiniDyn/Account.cdm.json/Account";
}
else
{
pathToDocRoot = "../../../../../../../../schemaDocuments";
testEnt = "local:/core/applicationCommon/foundationCommon/crmCommon/Account.cdm.json/Account";
docGroup = "standards";
}
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathToDocRoot));
manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"local:/{docGroup}.manifest.cdm.json");
string version = "";
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Progress);
Console.WriteLine("reading source files");
if (resolveEnt)
{
// AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet<string> { "normalized", "xstructured", "referenceOnly" });
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "referenceOnly"
});
var ent = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(testEnt);
ResolveOptions resOpt = new ResolveOptions {
WrtDoc = ent.InDocument, Directives = directives
};
var x = await ent.CreateResolvedEntityAsync("RESOLVED_KILL", resOpt);
resOpt.WrtDoc = x.InDocument;
CommonDataModelLoader.PersistDocument(cdmCorpus.RootPath, resOpt, new CopyOptions {
StringRefs = false, RemoveSingleRowLocalizedTableTraits = true
});
}
if (spewAll)
{
Console.WriteLine("list all resolved");
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "xstructured", "referenceOnly"
});
await ListAllResolved(cdmCorpus, directives, manifest, new StringSpewCatcher());
}
if (rePersist)
{
Console.WriteLine("persist corpus");
AttributeResolutionDirectiveSet directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "xstructured", "referenceOnly"
});
}
//ListAllTraits(cdmCorpus);
Console.WriteLine("done");
Console.ReadLine();
}
static async Task Main(string[] args)
{
var pathFromExeToExampleRoot = "../../../../../../";
// ------------------------------------------------------------------------------------------------------------
// Instantiate corpus and set up the default namespace to be local
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.Storage.DefaultNamespace = "local";
// set callback to receive error and warning logs.
cdmCorpus.SetEventCallback(new EventCallback
{
Invoke = (level, message) =>
{
Console.WriteLine(message);
}
}, CdmStatusLevel.Warning);
// ------------------------------------------------------------------------------------------------------------
// Set up adapters for managing access to different files-system locations
// Fake cdm, normaly use the CDM Standards adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
cdmCorpus.Storage.Mount("local", new LocalAdapter(modelJsonRoot));
// Example how to mount to the ADLS - make sure the hostname and root entered here are also changed
// in the example.model.json file we load in the next section
cdmCorpus.Storage.Mount("adls",
new ADLSAdapter(
"<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
"/<FILESYSTEM-NAME>", // Root.
"72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
"<CLIENT-ID>", // Client ID.
"<CLIENT-SECRET>" // Client secret.
));
// ------------------------------------------------------------------------------------------------------------
// Load a model.json file from local FS
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/model.json");
// ------------------------------------------------------------------------------------------------------------
// Explore entities and partitions defined in the model
Console.WriteLine("Listing entity declarations:");
foreach (CdmEntityDeclarationDefinition decl in manifest.Entities)
{
Console.WriteLine(" " + decl.EntityName);
// TODO: This can be rewritten in a different way since data partition gives null for referenced entities, suggestions are welcome.
if (decl.ObjectType == CdmObjectType.LocalEntityDeclarationDef)
{
foreach (CdmDataPartitionDefinition dataPart in decl.DataPartitions)
{
Console.WriteLine(" " + dataPart.Location);
}
}
}
// ------------------------------------------------------------------------------------------------------------
// Make changes to the model
// Create a new document where the new entity's definition will be stored
var newEntityDoc = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, "NewEntity.cdm.json", false);
newEntityDoc.Imports.Add("cdm:/foundations.cdm.json");
cdmCorpus.Storage.FetchRootFolder("local").Documents.Add(newEntityDoc);
var newEntity = newEntityDoc.Definitions.Add(CdmObjectType.EntityDef, "NewEntity") as CdmEntityDefinition;
// Define new string attribute and add it to the entity definition
var newAttribute = cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(CdmObjectType.TypeAttributeDef, "NewAttribute", false);
newAttribute.DataFormat = CdmDataFormat.String;
newEntity.Attributes.Add(newAttribute);
// Call will create EntityDeclarationDefinition based on entity definition and add it to manifest.Entities
var newEntityDecl = manifest.Entities.Add(newEntity);
// Define a partition and add it to the local declaration
var newPartition = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, "NewPartition", false);
newPartition.Location = "adls:/NewPartition.csv";
newEntityDecl.DataPartitions.Add(newPartition);
// ------------------------------------------------------------------------------------------------------------
// Save the file to ADLSg2 - we achieve that by adding the manifest to the root folder of
// the ADLS file-system and performing a save on the manifest
CdmFolderDefinition adlsFolder = cdmCorpus.Storage.FetchRootFolder("adls");
adlsFolder.Documents.Add(manifest);
await manifest.SaveAsAsync("model.json", true);
}
static async Task Main(string[] args)
{
// Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
var cdmCorpus = new CdmCorpusDefinition();
// set callback to receive error and warning logs.
cdmCorpus.SetEventCallback(new EventCallback
{
Invoke = (level, message) =>
{
Console.WriteLine(message);
}
}, CdmStatusLevel.Warning);
Console.WriteLine("Configure storage adapters");
// Configure storage adapters to point at the target local manifest location and at the fake public standards
string pathFromExeToExampleRoot = "../../../../../../";
cdmCorpus.Storage.Mount("local", new LocalAdapter(pathFromExeToExampleRoot + "2-create-manifest/sample-data"));
cdmCorpus.Storage.DefaultNamespace = "local"; // local is our default. so any paths that start out navigating without a device tag will assume local
// Fake cdm, normaly use the CDM Standards adapter
// Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
cdmCorpus.Storage.Mount("cdm", new LocalAdapter(pathFromExeToExampleRoot + "example-public-standards"));
// Example how to mount to the ADLS.
// cdmCorpus.Storage.Mount("adls",
// new ADLSAdapter(
// "<ACCOUNT-NAME>.dfs.core.windows.net", // Hostname.
// "/<FILESYSTEM-NAME>", // Root.
// "72f988bf-86f1-41af-91ab-2d7cd011db47", // Tenant ID.
// "<CLIENT-ID>", // Client ID.
// "<CLIENT-SECRET>" // Client secret.
// ));
Console.WriteLine("Make placeholder manifest");
// Make the temp manifest and add it to the root of the local documents in the corpus
CdmManifestDefinition manifestAbstract = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "tempAbstract");
// Add each declaration, this example is about medical appointments and care plans
manifestAbstract.Entities.Add("Account", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account");
manifestAbstract.Entities.Add("Address", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address");
manifestAbstract.Entities.Add("CarePlan", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan");
manifestAbstract.Entities.Add("CodeableConcept", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept");
manifestAbstract.Entities.Add("Contact", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact");
manifestAbstract.Entities.Add("Device", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device");
manifestAbstract.Entities.Add("EmrAppointment", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment");
manifestAbstract.Entities.Add("Encounter", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter");
manifestAbstract.Entities.Add("EpisodeOfCare", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare");
manifestAbstract.Entities.Add("Location", "cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location");
// Add the temp manifest to the root of the local documents in the corpus.
var localRoot = cdmCorpus.Storage.FetchRootFolder("local");
localRoot.Documents.Add(manifestAbstract);
// Create the resolved version of everything in the root folder too
Console.WriteLine("Resolve the placeholder");
var manifestResolved = await manifestAbstract.CreateResolvedManifestAsync("default", "");
// Add an import to the foundations doc so the traits about partitons will resolve nicely
manifestResolved.Imports.Add("cdm:/foundations.cdm.json");
Console.WriteLine("Save the documents");
foreach (CdmEntityDeclarationDefinition eDef in manifestResolved.Entities)
{
// Get the entity being pointed at
var localEDef = eDef;
var entDef = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(localEDef.EntityPath, manifestResolved);
// Make a fake partition, just to demo that
var part = cdmCorpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef, $"{entDef.EntityName}-data-description");
localEDef.DataPartitions.Add(part);
part.Explanation = "not real data, just for demo";
// Define the location of the partition, relative to the manifest
var location = $"local:/{entDef.EntityName}/partition-data.csv";
part.Location = cdmCorpus.Storage.CreateRelativeCorpusPath(location, manifestResolved);
// Add trait to partition for csv params
var csvTrait = part.ExhibitsTraits.Add("is.partition.format.CSV", false) as CdmTraitReference;
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", ",");
// Get the actual location of the partition file from the corpus
string partPath = cdmCorpus.Storage.CorpusPathToAdapterPath(location);
// Make a fake file with nothing but header for columns
string header = "";
foreach (CdmTypeAttributeDefinition att in entDef.Attributes)
{
if (header != "")
{
header += ",";
}
header += att.Name;
}
Directory.CreateDirectory(cdmCorpus.Storage.CorpusPathToAdapterPath($"local:/{entDef.EntityName}"));
File.WriteAllText(partPath, header);
}
await manifestResolved.SaveAsAsync($"{manifestResolved.ManifestName}.manifest.cdm.json", true);
}
public void TestPathRootInvalidFolderPath()
{
var corpus = new CdmCorpusDefinition();
var callback = new EventCallback();
var functionWasCalled = false;
var functionParameter1 = CdmStatusLevel.Info;
string functionParameter2 = null, functionParameter3 = null;
callback.Invoke = (CdmStatusLevel statusLevel, string message1) =>
{
functionWasCalled = true;
functionParameter1 = statusLevel;
functionParameter2 = message1;
};
corpus.SetEventCallback(callback);
var absolutePath = corpus.Storage.CreateAbsoluteCorpusPath("Abc",
new CdmManifestDefinition(null, null)
{
Namespace = "cdm", FolderPath = "./Mnp"
});
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("The path should not start with ./"));
functionWasCalled = false;
absolutePath = corpus.Storage.CreateAbsoluteCorpusPath("Abc",
new CdmManifestDefinition(null, null)
{
Namespace = "cdm", FolderPath = "/./Mnp"
});
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("The path should not contain /./"));
functionWasCalled = false;
absolutePath = corpus.Storage.CreateAbsoluteCorpusPath("Abc",
new CdmManifestDefinition(null, null)
{
Namespace = "cdm", FolderPath = "../Mnp"
});
functionParameter2 = functionParameter2.Split("|")[1].Trim();
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("The path should not contain ../"));
functionWasCalled = false;
absolutePath = corpus.Storage.CreateAbsoluteCorpusPath("Abc",
new CdmManifestDefinition(null, null)
{
Namespace = "cdm", FolderPath = "Mnp/./Qrs"
});
functionParameter2 = functionParameter2.Split("|")[1].Trim();
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("The path should not contain /./"));
functionWasCalled = false;
absolutePath = corpus.Storage.CreateAbsoluteCorpusPath("Abc",
new CdmManifestDefinition(null, null)
{
Namespace = "cdm", FolderPath = "Mnp/../Qrs"
});
functionParameter2 = functionParameter2.Split("|")[1].Trim();
Assert.IsTrue(functionWasCalled);
Assert.AreEqual(CdmStatusLevel.Error, functionParameter1);
Assert.IsTrue(functionParameter2.Contains("The path should not contain ../"));
}
public async Task TestPartitionPatternWithGlob()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestPartitionPatternWithGlob");
int patternWithGlobAndRegex = 0;
corpus.SetEventCallback(new EventCallback
{
Invoke = (CdmStatusLevel statusLevel, string message) =>
{
if (message.Contains("CdmDataPartitionPatternDefinition | The Data Partition Pattern contains both a glob pattern (/testfile.csv) and a regular expression (/subFolder/testSubFile.csv) set, the glob pattern will be used. | FileStatusCheckAsync"))
{
patternWithGlobAndRegex++;
}
}
}, CdmStatusLevel.Warning);
CdmManifestDefinition manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("pattern.manifest.cdm.json");
await manifest.FileStatusCheckAsync();
// one pattern object contains both glob and regex
Assert.AreEqual(1, patternWithGlobAndRegex);
int index = 0;
// make sure '.' in glob is not converted to '.' in regex
CdmLocalEntityDeclarationDefinition dotIsEscaped = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(dotIsEscaped.DataPartitionPatterns[0].GlobPattern, "test.ile.csv");
Assert.AreEqual(dotIsEscaped.DataPartitions.Count, 0);
index++;
// star pattern should match anything in the root folder
CdmLocalEntityDeclarationDefinition onlyStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(onlyStar.DataPartitionPatterns[0].GlobPattern, "*");
Assert.AreEqual(onlyStar.DataPartitions.Count, 1);
Assert.AreEqual(onlyStar.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// star can match nothing
CdmLocalEntityDeclarationDefinition starNoMatch = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(starNoMatch.DataPartitionPatterns[0].GlobPattern, "/testfile*.csv");
Assert.AreEqual(starNoMatch.DataPartitions.Count, 1);
Assert.AreEqual(starNoMatch.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// star at root level
// this should match any files at root level, none in subfolders
CdmLocalEntityDeclarationDefinition starAtRoot = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(starAtRoot.DataPartitionPatterns[0].GlobPattern, "/*.csv");
Assert.AreEqual(starAtRoot.DataPartitions.Count, 1);
Assert.AreEqual(starAtRoot.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// star at deeper level
CdmLocalEntityDeclarationDefinition starAtDeeperLevel = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(starAtDeeperLevel.DataPartitionPatterns[0].GlobPattern, "/*/*.csv");
Assert.AreEqual(starAtDeeperLevel.DataPartitions.Count, 1);
Assert.AreEqual(starAtDeeperLevel.DataPartitions[0].Location, "/partitions/subFolder/testSubFile.csv");
index++;
// pattern that ends with star
CdmLocalEntityDeclarationDefinition endsWithStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(endsWithStar.DataPartitionPatterns[0].GlobPattern, "/testfile*");
Assert.AreEqual(endsWithStar.DataPartitions.Count, 1);
Assert.AreEqual(endsWithStar.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// globstar (**) on its own matches
CdmLocalEntityDeclarationDefinition globStar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(globStar.DataPartitionPatterns[0].GlobPattern, "**");
Assert.AreEqual(2, globStar.DataPartitions.Count);
Assert.AreEqual(1, globStar.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, globStar.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
index++;
// globstar at the beginning of the pattern
CdmLocalEntityDeclarationDefinition beginsWithGlobstar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(beginsWithGlobstar.DataPartitionPatterns[0].GlobPattern, "/**.csv");
Assert.AreEqual(1, beginsWithGlobstar.DataPartitions.Count);
Assert.AreEqual(beginsWithGlobstar.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// globstar at the end of the pattern
CdmLocalEntityDeclarationDefinition endsWithGlobstar = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(endsWithGlobstar.DataPartitionPatterns[0].GlobPattern, "/**");
Assert.AreEqual(endsWithGlobstar.DataPartitions.Count, 2);
Assert.AreEqual(1, endsWithGlobstar.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, endsWithGlobstar.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
index++;
// globstar matches zero or more folders
CdmLocalEntityDeclarationDefinition zeroOrMoreFolders = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(zeroOrMoreFolders.DataPartitionPatterns[0].GlobPattern, "/**/*.csv");
Assert.AreEqual(2, zeroOrMoreFolders.DataPartitions.Count);
Assert.AreEqual(1, zeroOrMoreFolders.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, zeroOrMoreFolders.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
index++;
// globstar matches zero or more folders without starting slash
CdmLocalEntityDeclarationDefinition zeroOrMoreNoStartingSlash = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(zeroOrMoreNoStartingSlash.DataPartitionPatterns[0].GlobPattern, "/**/*.csv");
Assert.AreEqual(2, zeroOrMoreNoStartingSlash.DataPartitions.Count);
Assert.AreEqual(1, zeroOrMoreNoStartingSlash.DataPartitions.Where(x =>
x.Location == "/partitions/testfile.csv"
).ToList().Count);
Assert.AreEqual(1, zeroOrMoreNoStartingSlash.DataPartitions.Where(x =>
x.Location == "/partitions/subFolder/testSubFile.csv"
).ToList().Count);
index++;
// question mark in the middle of a pattern
CdmLocalEntityDeclarationDefinition questionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(questionMark.DataPartitionPatterns[0].GlobPattern, "/test?ile.csv");
Assert.AreEqual(1, questionMark.DataPartitions.Count);
Assert.AreEqual(questionMark.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// question mark at the beginning of a pattern
CdmLocalEntityDeclarationDefinition beginsWithQuestionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(beginsWithQuestionMark.DataPartitionPatterns[0].GlobPattern, "/?estfile.csv");
Assert.AreEqual(1, beginsWithQuestionMark.DataPartitions.Count);
Assert.AreEqual(beginsWithQuestionMark.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// question mark at the end of a pattern
CdmLocalEntityDeclarationDefinition endsWithQuestionMark = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(endsWithQuestionMark.DataPartitionPatterns[0].GlobPattern, "/testfile.cs?");
Assert.AreEqual(1, endsWithQuestionMark.DataPartitions.Count);
Assert.AreEqual(endsWithQuestionMark.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// backslash in glob can match slash
CdmLocalEntityDeclarationDefinition backslashInPattern = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(backslashInPattern.DataPartitionPatterns[0].GlobPattern, "\\testfile.csv");
Assert.AreEqual(1, backslashInPattern.DataPartitions.Count);
Assert.AreEqual(backslashInPattern.DataPartitions[0].Location, "/partitions/testfile.csv");
index++;
// pattern object includes glob pattern and regular expression
CdmLocalEntityDeclarationDefinition globAndRegex = (CdmLocalEntityDeclarationDefinition)manifest.Entities[index];
Assert.AreEqual(globAndRegex.DataPartitionPatterns[0].GlobPattern, "/testfile.csv");
Assert.AreEqual(globAndRegex.DataPartitionPatterns[0].RegularExpression, "/subFolder/testSubFile.csv");
Assert.AreEqual(1, globAndRegex.DataPartitions.Count);
// matching this file means the glob pattern was (correctly) used
Assert.AreEqual(globAndRegex.DataPartitions[0].Location, "/partitions/testfile.csv");
}
/// <summary>
/// This method runs the tests with a set expected attributes & attribute context values and validated the actual result
/// </summary>
/// <param name="testName"></param>
/// <param name="sourceEntityName"></param>
/// <param name="expectedContext_*">expected attribute context object - for each resolution option combination</param>
/// <param name="expected_*">expected attribute object - for each resolution option combination</param>
/// <returns></returns>
protected static async Task RunTestWithValues(
string testName,
string sourceEntityName,
AttributeContextExpectedValue expectedContext_default,
AttributeContextExpectedValue expectedContext_normalized,
AttributeContextExpectedValue expectedContext_referenceOnly,
AttributeContextExpectedValue expectedContext_structured,
AttributeContextExpectedValue expectedContext_normalized_structured,
AttributeContextExpectedValue expectedContext_referenceOnly_normalized,
AttributeContextExpectedValue expectedContext_referenceOnly_structured,
AttributeContextExpectedValue expectedContext_referenceOnly_normalized_structured,
List <AttributeExpectedValue> expected_default,
List <AttributeExpectedValue> expected_normalized,
List <AttributeExpectedValue> expected_referenceOnly,
List <AttributeExpectedValue> expected_structured,
List <AttributeExpectedValue> expected_normalized_structured,
List <AttributeExpectedValue> expected_referenceOnly_normalized,
List <AttributeExpectedValue> expected_referenceOnly_structured,
List <AttributeExpectedValue> expected_referenceOnly_normalized_structured
)
{
try
{
string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName);
string testActualPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName);
string testExpectedPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName);
string corpusPath = testInputPath.Substring(0, testInputPath.Length - "/Input".Length);
testActualPath = Path.GetFullPath(testActualPath);
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("local", new LocalAdapter(corpusPath));
corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath));
corpus.Storage.DefaultNamespace = "local";
string outFolderPath = corpus.Storage.AdapterPathToCorpusPath(testActualPath) + "/"; // interesting 'bug'
CdmFolderDefinition outFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>(outFolderPath);
CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"local:/Input/{sourceEntityName}.cdm.json/{sourceEntityName}");
Assert.IsTrue(srcEntityDef != null);
var resOpt = new ResolveOptions
{
WrtDoc = srcEntityDef.InDocument,
Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
})
};
CdmEntityDefinition resolvedEntityDef = null;
string outputEntityName = string.Empty;
string outputEntityFileName = string.Empty;
string entityFileName = string.Empty;
if (expectedContext_default != null && expected_default != null)
{
entityFileName = "d";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_normalized != null && expected_normalized != null)
{
entityFileName = "n";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_referenceOnly != null && expected_referenceOnly != null)
{
entityFileName = "ro";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_structured != null && expected_structured != null)
{
entityFileName = "s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_normalized_structured != null && expected_normalized_structured != null)
{
entityFileName = "n_s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_referenceOnly_normalized != null && expected_referenceOnly_normalized != null)
{
entityFileName = "ro_n";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_referenceOnly_structured != null && expected_referenceOnly_structured != null)
{
entityFileName = "ro_s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
if (expectedContext_referenceOnly_normalized_structured != null && expected_referenceOnly_normalized_structured != null)
{
entityFileName = "ro_n_s";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized", "structured"
});
outputEntityName = $"{sourceEntityName}_R_{entityFileName}";
outputEntityFileName = $"{outputEntityName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityName, resOpt, outFolder);
await SaveActualEntityAndValidateWithExpected(Path.Combine(testExpectedPath, outputEntityFileName), resolvedEntityDef);
}
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public async Task TestWithoutNesting()
{
CdmCorpusDefinition corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestEventList");
corpus.SetEventCallback(eventCallback, CdmStatusLevel.Warning, DummyCorrelationId);
// Test fetching an object from invalid namespace results in at least one error message in the recorder
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("foo:/bar");
Assert.IsNotNull(corpus.Ctx.Events, "Ctx.Events should not be null");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded when fetching object with incorrect path");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test fetching a good object, this should leave event recorder empty
_ = await corpus.FetchObjectAsync <CdmDocumentDefinition>("local:/default.manifest.cdm.json");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count == 0, "There should have been no events recorded when fetching object with correct path");
// Test saving a manifest to invalid namespace results in at least one error message in the recorder
var manifest = corpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "dummy");
await manifest.SaveAsAsync("foo:/bar", true);
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test resolving a manifest not added to a folder, this should yield at least one error message in the recorder
await manifest.CreateResolvedManifestAsync("new dummy", null);
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test resolving an entity without WRT doc, this should yield at least one error message in the recorder
var entity2 = corpus.MakeObject <CdmEntityDefinition>(CdmObjectType.EntityDef, "MyEntity2");
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Test invoking FileStatusCheckAsync on the manifest, this should yield at least one error message in the recorder
await manifest.FileStatusCheckAsync();
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count > 0, "There should have been at least one event recorded");
Assert.IsTrue(corpus.Ctx.Events[0].ContainsKey("timestamp"), "The recorded event should have had a timestamp key");
Assert.IsTrue(corpus.Ctx.Events[0]["correlationId"] == DummyCorrelationId, "The recorded event should have had a correlationId key with the dummy value");
// Repeat the same test but with status level 'None', no events should be recorded
corpus.Ctx.ReportAtLevel = CdmStatusLevel.None;
await entity2.CreateResolvedEntityAsync("MyEntity2-Resolved");
Assert.IsFalse(corpus.Ctx.Events.IsRecording, "Recording should be disabled at the end of API call");
Assert.IsTrue(corpus.Ctx.Events.Count == 0, "There should have been no events recorded when fetching object with correct path");
}
private static async Task RunTest(string testName, string sourceEntityName)
{
try
{
string testInputPath = TestHelper.GetInputFolderPath(TestsSubpath, testName);
string testExpectedOutputPath = TestHelper.GetExpectedOutputFolderPath(TestsSubpath, testName);
string testActualOutputPath = TestHelper.GetActualOutputFolderPath(TestsSubpath, testName);
CdmCorpusDefinition corpus = new CdmCorpusDefinition();
corpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
corpus.Storage.Mount("localInput", new LocalAdapter(testInputPath));
corpus.Storage.Mount("localExpectedOutput", new LocalAdapter(testExpectedOutputPath));
corpus.Storage.Mount("localActualOutput", new LocalAdapter(testActualOutputPath));
corpus.Storage.Mount("cdm", new LocalAdapter(SchemaDocsPath));
corpus.Storage.DefaultNamespace = "localInput";
CdmEntityDefinition srcEntityDef = await corpus.FetchObjectAsync <CdmEntityDefinition>($"localInput:/{sourceEntityName}.cdm.json/{sourceEntityName}") as CdmEntityDefinition;
Assert.IsTrue(srcEntityDef != null);
var resOpt = new ResolveOptions
{
WrtDoc = srcEntityDef.InDocument,
Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
})
};
CdmFolderDefinition actualOutputFolder = await corpus.FetchObjectAsync <CdmFolderDefinition>("localActualOutput:/") as CdmFolderDefinition;
CdmEntityDefinition resolvedEntityDef = null;
string outputEntityFileName = string.Empty;
string entityFileName = string.Empty;
entityFileName = "default";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "normalized";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly_normalized";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "normalized_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"normalized", "structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
entityFileName = "referenceOnly_normalized_structured";
resOpt.Directives = new AttributeResolutionDirectiveSet(new HashSet <string> {
"referenceOnly", "normalized", "structured"
});
outputEntityFileName = $"{sourceEntityName}_Resolved_{entityFileName}.cdm.json";
resolvedEntityDef = await srcEntityDef.CreateResolvedEntityAsync(outputEntityFileName, resOpt, actualOutputFolder);
if (await resolvedEntityDef.InDocument.SaveAsAsync(outputEntityFileName, true, new CopyOptions()))
{
ValidateOutput(outputEntityFileName, testExpectedOutputPath, testActualOutputPath);
}
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public async Task TestRefreshesDataPartitionPatterns()
{
var inputPath = TestHelper.GetInputFolderPath(testsSubpath, "TestRefreshDataPartitionPatterns");
var actualLastModTime = new FileInfo(Path.GetFullPath(inputPath)).LastWriteTimeUtc;
var cdmCorpus = new CdmCorpusDefinition();
cdmCorpus.SetEventCallback(new EventCallback {
Invoke = CommonDataModelLoader.ConsoleStatusReport
}, CdmStatusLevel.Warning);
cdmCorpus.Storage.Mount("local", new LocalAdapter(inputPath));
cdmCorpus.Storage.DefaultNamespace = "local";
var cdmManifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/patternManifest.manifest.cdm.json");
var partitionEntity = cdmManifest.Entities.AllItems[0];
Assert.AreEqual(partitionEntity.DataPartitions.Count, 1);
var timeBeforeLoad = DateTime.Now;
await cdmManifest.FileStatusCheckAsync();
// file status check should check patterns and add two more partitions that match the pattern
// should not re-add already existing partitions
// Mac and Windows behave differently when listing file content, so we don't want to be strict about partition file order
int totalExpectedPartitionsFound = 0;
foreach (CdmDataPartitionDefinition partition in partitionEntity.DataPartitions.AllItems)
{
switch (partition.Location)
{
case "partitions/existingPartition.csv":
totalExpectedPartitionsFound++;
break;
case "partitions/someSubFolder/someSubPartition.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.SpecializedSchema, "test special schema");
Assert.IsTrue(partition.LastFileStatusCheckTime > timeBeforeLoad);
// inherits the exhibited traits from pattern
Assert.AreEqual(partition.ExhibitsTraits.Count, 1);
Assert.AreEqual(partition.ExhibitsTraits.AllItems[0].NamedReference, "is");
Assert.AreEqual(partition.Arguments.Count, 1);
Assert.IsTrue(partition.Arguments.ContainsKey("testParam1"));
List <string> argArray = partition.Arguments["testParam1"];
Assert.AreEqual(argArray.Count, 1);
Assert.AreEqual(argArray[0], "/someSubFolder/someSub");
break;
case "partitions/newPartition.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 1);
break;
case "partitions/2018/folderCapture.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 1);
Assert.AreEqual(partition.Arguments.ContainsKey("year"), true);
Assert.AreEqual(partition.Arguments["year"][0], "2018");
break;
case "partitions/2018/8/15/folderCapture.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 3);
Assert.AreEqual(partition.Arguments.ContainsKey("year"), true);
Assert.AreEqual(partition.Arguments["year"][0], "2018");
Assert.AreEqual(partition.Arguments.ContainsKey("month"), true);
Assert.AreEqual(partition.Arguments["month"][0], "8");
Assert.AreEqual(partition.Arguments.ContainsKey("day"), true);
Assert.AreEqual(partition.Arguments["day"][0], "15");
break;
case "partitions/2018/8/15/folderCaptureRepeatedGroup.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 1);
Assert.AreEqual(partition.Arguments.ContainsKey("day"), true);
Assert.AreEqual(partition.Arguments["day"][0], "15");
break;
case "partitions/testTooFew.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 0);
break;
case "partitions/testTooMany.csv":
totalExpectedPartitionsFound++;
Assert.AreEqual(partition.Arguments.Count, 0);
break;
}
}
Assert.AreEqual(totalExpectedPartitionsFound, 8);
}
