private static async Task RunSpecialCharactersTest(ADLSAdapter adapter)
{
var corpus = new CdmCorpusDefinition();
corpus.Storage.Mount("adls", adapter);
corpus.Storage.DefaultNamespace = "adls";
try
{
var manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("default.manifest.cdm.json");
await manifest.FileStatusCheckAsync();
Assert.AreEqual(1, manifest.Entities.Count);
Assert.AreEqual(2, manifest.Entities[0].DataPartitions.Count);
Assert.AreEqual(
"TestEntity-With=Special Characters/year=2020/TestEntity-partition-With=Special Characters-0.csv",
manifest.Entities[0].DataPartitions[0].Location);
Assert.AreEqual(
"TestEntity-With=Special Characters/year=2020/TestEntity-partition-With=Special Characters-1.csv",
manifest.Entities[0].DataPartitions[1].Location);
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public async Task TestImportsRelativePath()
{
// the corpus path in the imports are relative to the document where it was defined.
// when saving in model.json the documents are flattened to the manifest level
// so it is necessary to recalculate the path to be relative to the manifest.
var corpus = this.GetLocalCorpus("notImportantLocation");
var folder = corpus.Storage.FetchRootFolder("local");
var manifest = new CdmManifestDefinition(corpus.Ctx, "manifest");
var entityDeclaration = manifest.Entities.Add("EntityName", "EntityName/EntityName.cdm.json/EntityName");
folder.Documents.Add(manifest);
var entityFolder = folder.ChildFolders.Add("EntityName");
var document = new CdmDocumentDefinition(corpus.Ctx, "EntityName.cdm.json");
document.Imports.Add("subfolder/EntityName.cdm.json");
document.Definitions.Add("EntityName");
entityFolder.Documents.Add(document);
var subFolder = entityFolder.ChildFolders.Add("subfolder");
subFolder.Documents.Add("EntityName.cdm.json");
var data = await ManifestPersistence.ToData(manifest, null, null);
Assert.AreEqual(1, data.Entities.Count);
var imports = data.Entities[0]["cdm:imports"].ToObject <List <Import> >();
Assert.AreEqual(1, imports.Count);
Assert.AreEqual("EntityName/subfolder/EntityName.cdm.json", imports[0].CorpusPath);
}
public void TestLoadingAndSavingEndpointInConfig()
{
// Mount from config
var config = TestHelper.GetInputFileContent(testSubpath, nameof(TestLoadingAndSavingEndpointInConfig), "config.json");
var corpus = new CdmCorpusDefinition();
corpus.Storage.MountFromConfig(config);
Assert.Null(((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter1")).Endpoint);
Assert.AreEqual(AzureCloudEndpoint.AzurePublic, ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter2")).Endpoint);
Assert.AreEqual(AzureCloudEndpoint.AzureChina, ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter3")).Endpoint);
Assert.AreEqual(AzureCloudEndpoint.AzureGermany, ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter4")).Endpoint);
Assert.AreEqual(AzureCloudEndpoint.AzureUsGovernment, ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter5")).Endpoint);
try
{
var configSnakeCase = TestHelper.GetInputFileContent(testSubpath, nameof(TestLoadingAndSavingEndpointInConfig), "config-SnakeCase.json");
var corpusSnakeCase = new CdmCorpusDefinition();
corpusSnakeCase.Storage.MountFromConfig(configSnakeCase);
Assert.Fail("Expected RuntimeException for config.json using endpoint value in snake case.");
}
catch (Exception ex)
{
String message = "Endpoint value should be a string of an enumeration value from the class AzureCloudEndpoint in Pascal case.";
Assert.AreEqual(message, ex.Message);
}
}
public async Task TestAvoidRetryCodes()
{
AdlsTestHelper.CheckADLSEnvironment();
var adlsAdapter = AdlsTestHelper.CreateAdapterWithSharedKey();
adlsAdapter.NumberOfRetries = 3;
var corpus = new CdmCorpusDefinition();
corpus.Storage.Mount("adls", adlsAdapter);
var count = 0;
corpus.SetEventCallback(new EventCallback
{
Invoke = (status, message) =>
{
if (message.Contains("Response for request "))
{
count++;
}
}
}, CdmStatusLevel.Progress);
await corpus.FetchObjectAsync <CdmDocumentDefinition>("adls:/inexistentFile.cdm.json");
Assert.AreEqual(1, count);
}
public async Task TestLoadingAndSavingDateAndTimeDataTypes()
{
var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestLoadingAndSavingDateAndTimeDataTypes));
// Load the manifest and resolve it
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local:/default.manifest.cdm.json");
var resolvedManifest = await manifest.CreateResolvedManifestAsync("resolved", null);
// Convert loaded manifest to model.json
var modelJson = await ManifestPersistence.ToData(resolvedManifest, null, null);
// Verify that the attributes' data types were correctly persisted as "date" and "time"
Assert.AreEqual("date", modelJson.Entities[0]["attributes"][0]["dataType"].ToString());
Assert.AreEqual("time", modelJson.Entities[0]["attributes"][1]["dataType"].ToString());
// Now check that these attributes' data types are still "date" and "time" when loading the model.json back to manifest
// We first need to create a second adapter to the input folder to fool the OM into thinking it's different
// This is because there's a bug that currently prevents us from saving and then loading a model.json under the same namespace
cdmCorpus.Storage.Mount("local2", new LocalAdapter(TestHelper.GetInputFolderPath(testsSubpath, nameof(TestLoadingAndSavingDateAndTimeDataTypes))));
var manifestFromModelJson = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("local2:/model.json");
var entity = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>(manifestFromModelJson.Entities[0].EntityPath, manifestFromModelJson);
// Verify that the attributes' data types were correctly loaded as "date" and "time"
Assert.AreEqual(CdmDataFormat.Date, (entity.Attributes[0] as CdmTypeAttributeDefinition).DataFormat);
Assert.AreEqual(CdmDataFormat.Time, (entity.Attributes[1] as CdmTypeAttributeDefinition).DataFormat);
}
public async Task TestLoadingCdmFolderResultAndModelJsonToData()
{
var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestLoadingCdmFolderResultAndModelJsonToData));
var watch = Stopwatch.StartNew();
var cdmManifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>($"result.model{PersistenceLayer.ManifestExtension}", cdmCorpus.Storage.FetchRootFolder("local"));
watch.Stop();
Assert.Performance(1000, watch.ElapsedMilliseconds, "Loading from data");
watch.Restart();
var obtainedModelJson = await ManifestPersistence.ToData(cdmManifest, null, null);
watch.Stop();
Assert.Performance(1000, watch.ElapsedMilliseconds, "Parsing to data");
// remove empty description from entities as they interfere with test.
obtainedModelJson.Entities.ForEach(RemoveDescriptionFromEntityIfEmpty);
obtainedModelJson.Description = null;
Assert.IsNull(cdmManifest.Imports.Item(Constants.FoundationsCorpusPath, checkMoniker: false));
Assert.AreEqual(1, obtainedModelJson.Imports.Count);
Assert.AreEqual(Constants.FoundationsCorpusPath, obtainedModelJson.Imports[0].CorpusPath);
this.HandleOutput(nameof(TestLoadingCdmFolderResultAndModelJsonToData), PersistenceLayer.ModelJsonExtension, obtainedModelJson);
}
public void TestInitializeHostnameAndRoot()
{
var host1 = "storageaccount.dfs.core.windows.net";
var adlsAdapter1 = new ADLSAdapter(host1, "root-without-slash", string.Empty);
Assert.AreEqual("storageaccount.dfs.core.windows.net", adlsAdapter1.Hostname);
Assert.AreEqual("/root-without-slash", adlsAdapter1.Root);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/root-without-slash/a/1.csv";
var corpusPath1 = adlsAdapter1.CreateCorpusPath(adapterPath1);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual(adapterPath1, adlsAdapter1.CreateAdapterPath(corpusPath1));
var adlsAdapter1WithFolders = new ADLSAdapter(host1, "root-without-slash/folder1/folder2", string.Empty);
Assert.AreEqual("/root-without-slash/folder1/folder2", adlsAdapter1WithFolders.Root);
var adapterPath2 = "https://storageaccount.dfs.core.windows.net/root-without-slash/folder1/folder2/a/1.csv";
var corpusPath2 = adlsAdapter1WithFolders.CreateCorpusPath(adapterPath2);
Assert.AreEqual("/a/1.csv", corpusPath2);
Assert.AreEqual(adapterPath2, adlsAdapter1WithFolders.CreateAdapterPath(corpusPath2));
var adlsAdapter2 = new ADLSAdapter(host1, "/root-starts-with-slash", string.Empty);
Assert.AreEqual("/root-starts-with-slash", adlsAdapter2.Root);
var adlsAdapter2WithFolders = new ADLSAdapter(host1, "/root-starts-with-slash/folder1/folder2", string.Empty);
Assert.AreEqual("/root-starts-with-slash/folder1/folder2", adlsAdapter2WithFolders.Root);
var adlsAdapter3 = new ADLSAdapter(host1, "root-ends-with-slash/", string.Empty);
Assert.AreEqual("/root-ends-with-slash", adlsAdapter3.Root);
var adlsAdapter3WithFolders = new ADLSAdapter(host1, "root-ends-with-slash/folder1/folder2/", string.Empty);
Assert.AreEqual("/root-ends-with-slash/folder1/folder2", adlsAdapter3WithFolders.Root);
var adlsAdapter4 = new ADLSAdapter(host1, "/root-with-slashes/", string.Empty);
Assert.AreEqual("/root-with-slashes", adlsAdapter4.Root);
var adlsAdapter4WithFolders = new ADLSAdapter(host1, "/root-with-slashes/folder1/folder2/", string.Empty);
Assert.AreEqual("/root-with-slashes/folder1/folder2", adlsAdapter4WithFolders.Root);
// Mount from config
var config = TestHelper.GetInputFileContent(testSubpath, nameof(TestInitializeHostnameAndRoot), "config.json");
var corpus = new CdmCorpusDefinition();
corpus.Storage.MountFromConfig(config);
Assert.AreEqual("/root-without-slash", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter1")).Root);
Assert.AreEqual("/root-without-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter2")).Root);
Assert.AreEqual("/root-starts-with-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter3")).Root);
Assert.AreEqual("/root-ends-with-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter4")).Root);
Assert.AreEqual("/root-with-slashes/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter5")).Root);
}
public async Task TestIncorrectModelLocation()
{
var expectedLogCodes = new HashSet <CdmLogCode> {
CdmLogCode.ErrStorageInvalidAdapterPath, CdmLogCode.ErrPersistModelJsonEntityParsingError, CdmLogCode.ErrPersistModelJsonRefEntityInvalidLocation
};
var corpus = TestHelper.GetLocalCorpus(testsSubpath, "TestIncorrectModelLocation", expectedCodes: expectedLogCodes);
var manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("model.json");
Assert.NotNull(manifest);
Assert.AreEqual(0, manifest.Entities.Count);
TestHelper.AssertCdmLogCodeEquality(corpus, CdmLogCode.ErrPersistModelJsonRefEntityInvalidLocation, true);
}
public void TestEndpointMissingOnConfig()
{
var config = new JObject
{
{ "hostname", "hostname.dfs.core.windows.net" },
{ "root", "root" },
{ "tenant", "tenant" },
{ "clientId", "clientId" }
};
var adlsAdapter = new ADLSAdapter();
adlsAdapter.UpdateConfig(config.ToString());
Assert.AreEqual(AzureCloudEndpoint.AzurePublic, adlsAdapter.Endpoint);
}
public async Task TestLoadingAndSavingCdmTraits()
{
var cdmCorpus = TestHelper.GetLocalCorpus(testsSubpath, nameof(TestLoadingAndSavingCdmTraits));
var manifest = await cdmCorpus.FetchObjectAsync <CdmManifestDefinition>("model.json");
var entity = await cdmCorpus.FetchObjectAsync <CdmEntityDefinition>("someEntity.cdm.json/someEntity");
Assert.NotNull(entity.ExhibitsTraits.Item("is.CDM.entityVersion"));
var manifestData = await ManifestPersistence.ToData(manifest, new ResolveOptions(manifest.InDocument), new CopyOptions());
var versionAnnotation = (manifestData.Entities[0]["annotations"][0]).ToObject <Annotation>();
Assert.AreEqual("<version>", versionAnnotation.Value);
}
public void TestFormattedHostname()
{
var config = new JObject
{
{ "hostname", "hostname.dfs.core.windows.net" },
{ "root", "root" },
{ "tenant", "tenant" },
{ "clientId", "clientId" }
};
var adlsAdapter = new ADLSAdapter();
adlsAdapter.UpdateConfig(config.ToString());
var corpusPath = adlsAdapter.CreateCorpusPath("https://hostname.dfs.core.windows.net/root/partitions/data.csv");
Assert.AreEqual("/partitions/data.csv", corpusPath);
}
public void TestSerializerOrderMetadataProperty()
{
var originalModelJson = TestHelper.GetInputFileContent(testsSubpath, "TestSerializerOrderMetadataProperty", "SerializerTesting-entity.json");
var deserialized = Deserialize <Entity>(originalModelJson);
var serialized = Serialize(deserialized);
if (doesWriteTestDebuggingFiles)
{
TestHelper.WriteActualOutputFileContent(testsSubpath, "TestSerializerOrderMetadataProperty", "SerializerTesting -entity.json", serialized);
}
TestHelper.AssertSameObjectWasSerialized(originalModelJson, serialized);
originalModelJson.Replace("\r\n", "\n");
serialized.Replace("\r\n", "\n");
// "$type" appears on same position.
Assert.AreEqual(originalModelJson.IndexOf('$'), serialized.IndexOf('$'));
}
public void TestHostnameWithLeadingProtocol()
{
var host1 = "https://storageaccount.dfs.core.windows.net";
var adlsAdapter1 = new ADLSAdapter(host1, "root-without-slash", string.Empty);
var adapterPath = "https://storageaccount.dfs.core.windows.net/root-without-slash/a/1.csv";
var corpusPath1 = adlsAdapter1.CreateCorpusPath(adapterPath);
Assert.AreEqual("https://storageaccount.dfs.core.windows.net", adlsAdapter1.Hostname);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual(adapterPath, adlsAdapter1.CreateAdapterPath(corpusPath1));
var host2 = "HttPs://storageaccount.dfs.core.windows.net";
var adlsAdapter2 = new ADLSAdapter(host2, "root-without-slash", string.Empty);
var corpusPath2 = adlsAdapter2.CreateCorpusPath(adapterPath);
Assert.AreEqual("HttPs://storageaccount.dfs.core.windows.net", adlsAdapter2.Hostname);
Assert.AreEqual("/a/1.csv", corpusPath2);
Assert.AreEqual(adapterPath, adlsAdapter2.CreateAdapterPath(corpusPath2));
try
{
var host3 = "http://storageaccount.dfs.core.windows.net";
var adlsAdapter3 = new ADLSAdapter(host3, "root-without-slash", string.Empty);
Assert.Fail("Expected Exception for using a http:// hostname.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is ArgumentException);
}
try
{
var host4 = "https://bar:baz::]/foo/";
var adlsAdapter4 = new ADLSAdapter(host4, "root-without-slash", string.Empty);
Assert.Fail("Expected Exception for using and invalid hostname.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is ArgumentException);
}
}
public async Task TestSettingModelJsonEntityDescription()
{
var cdmCorpus = new CdmCorpusDefinition();
var cdmManifest = cdmCorpus.MakeObject <CdmManifestDefinition>(CdmObjectType.ManifestDef, "test");
var document = cdmCorpus.MakeObject <CdmDocumentDefinition>(CdmObjectType.DocumentDef, $"entity{PersistenceLayer.CdmExtension}");
var folder = cdmCorpus.Storage.FetchRootFolder("local");
folder.Documents.Add(document);
var entity = document.Definitions.Add(CdmObjectType.EntityDef, "entity") as CdmEntityDefinition;
entity.Description = "test description";
cdmManifest.Entities.Add(entity);
folder.Documents.Add(cdmManifest);
var obtainedModelJson = await ManifestPersistence.ToData(cdmManifest, null, null);
Assert.AreEqual("test description", obtainedModelJson.Entities[0]["description"].ToString());
}
public void TestEntityTimeDateReadInLocalFormat()
{
var originalModelJson = TestHelper.GetInputFileContent(testsSubpath, "TestEntityTimeDateReadInLocalFormat", "SerializerTesting-entity2.json");
var deserialized = Deserialize <Entity>(originalModelJson);
var expectedSerializedLastChildFileModifiedTime = "\"2018-12-19T02:05:03.2374986+00:00\"";
var expectedSerializedLastFileModifiedTime = "\"2018-12-19T05:05:03.2374986+00:00\"";
var expectedSerializedLastFileStatusCheckTime = "\"2018-12-19T21:35:03.2374986+00:00\"";
var lastChildFileModifiedTime = deserialized?.LastChildFileModifiedTime;
var lastFileModifiedTime = deserialized?.LastFileModifiedTime;
var lastFileStatusCheckTime = deserialized?.LastFileStatusCheckTime;
Assert.AreEqual(expectedSerializedLastChildFileModifiedTime, Serialize(lastChildFileModifiedTime));
Assert.AreEqual(expectedSerializedLastFileModifiedTime, Serialize(lastFileModifiedTime));
Assert.AreEqual(expectedSerializedLastFileStatusCheckTime, Serialize(lastFileStatusCheckTime));
string serialized = Serialize(deserialized);
Assert.IsTrue(serialized.Contains(expectedSerializedLastChildFileModifiedTime));
Assert.IsTrue(serialized.Contains(expectedSerializedLastFileModifiedTime));
Assert.IsTrue(serialized.Contains(expectedSerializedLastFileStatusCheckTime));
}
public void TestCreateCorpusAndAdapterPathInAdlsAdapter()
{
var host1 = "storageaccount.dfs.core.windows.net";
var root = "/fs";
var adlsAdapter = new ADLSAdapter(host1, root, string.Empty);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/fs/a/1.csv";
var adapterPath2 = "https://storageaccount.dfs.core.windows.net:443/fs/a/2.csv";
var adapterPath3 = "https://storageaccount.blob.core.windows.net/fs/a/3.csv";
var adapterPath4 = "https://storageaccount.blob.core.windows.net:443/fs/a/4.csv";
var corpusPath1 = adlsAdapter.CreateCorpusPath(adapterPath1);
var corpusPath2 = adlsAdapter.CreateCorpusPath(adapterPath2);
var corpusPath3 = adlsAdapter.CreateCorpusPath(adapterPath3);
var corpusPath4 = adlsAdapter.CreateCorpusPath(adapterPath4);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual("/a/2.csv", corpusPath2);
Assert.AreEqual("/a/3.csv", corpusPath3);
Assert.AreEqual("/a/4.csv", corpusPath4);
Assert.AreEqual(adapterPath1, adlsAdapter.CreateAdapterPath(corpusPath1));
Assert.AreEqual(adapterPath2, adlsAdapter.CreateAdapterPath(corpusPath2));
Assert.AreEqual(adapterPath3, adlsAdapter.CreateAdapterPath(corpusPath3));
Assert.AreEqual(adapterPath4, adlsAdapter.CreateAdapterPath(corpusPath4));
// Check that an adapter path is correctly created from a corpus path with any namespace
var corpusPathWithNamespace1 = "adls:/test.json";
var corpusPathWithNamespace2 = "mylake:/test.json";
var expectedAdapterPath = "https://storageaccount.dfs.core.windows.net/fs/test.json";
Assert.AreEqual(expectedAdapterPath, adlsAdapter.CreateAdapterPath(corpusPathWithNamespace1));
Assert.AreEqual(expectedAdapterPath, adlsAdapter.CreateAdapterPath(corpusPathWithNamespace2));
// Check that an adapter path is correctly created from a corpus path with colons
var corpusPathWithColons = "namespace:/a/path:with:colons/some-file.json";
Assert.AreEqual("https://storageaccount.dfs.core.windows.net/fs/a/path%3Awith%3Acolons/some-file.json", adlsAdapter.CreateAdapterPath(corpusPathWithColons));
Assert.AreEqual("/a/path:with:colons/some-file.json", adlsAdapter.CreateCorpusPath("https://storageaccount.dfs.core.windows.net/fs/a/path%3Awith%3Acolons/some-file.json"));
Assert.AreEqual("/a/path:with:colons/some-file.json", adlsAdapter.CreateCorpusPath("https://storageaccount.dfs.core.windows.net/fs/a/path%3awith%3acolons/some-file.json"));
// Check other special characters
Assert.AreEqual("https://storageaccount.dfs.core.windows.net/fs/a/path%20with%3Dspecial%3Dcharacters/some-file.json", adlsAdapter.CreateAdapterPath("namespace:/a/path with=special=characters/some-file.json"));
Assert.AreEqual("/a/path with=special=characters/some-file.json", adlsAdapter.CreateCorpusPath("https://storageaccount.dfs.core.windows.net/fs/a/path%20with%3dspecial%3dcharacters/some-file.json"));
Assert.AreEqual("/a/path with=special=characters/some-file.json", adlsAdapter.CreateCorpusPath("https://storageaccount.dfs.core.windows.net/fs/a/path%20with%3dspecial%3Dcharacters/some-file.json"));
// Check that an adapter path is null if the corpus path provided is null
Assert.IsNull(adlsAdapter.CreateAdapterPath(null));
var host2 = "storageaccount.blob.core.windows.net:8888";
adlsAdapter = new ADLSAdapter(host2, root, string.Empty);
var adapterPath5 = "https://storageaccount.blob.core.windows.net:8888/fs/a/5.csv";
var adapterPath6 = "https://storageaccount.dfs.core.windows.net:8888/fs/a/6.csv";
var adapterPath7 = "https://storageaccount.blob.core.windows.net/fs/a/7.csv";
Assert.AreEqual("/a/5.csv", adlsAdapter.CreateCorpusPath(adapterPath5));
Assert.AreEqual("/a/6.csv", adlsAdapter.CreateCorpusPath(adapterPath6));
Assert.AreEqual(null, adlsAdapter.CreateCorpusPath(adapterPath7));
}