public void TestCreateCorpusAndAdapterPathInAdlsAdapter()
{
var host1 = "storageaccount.dfs.core.windows.net";
var root = "/fs";
var adlsAdapter = new ADLSAdapter(host1, root, string.Empty);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/fs/a/1.csv";
var adapterPath2 = "https://storageaccount.dfs.core.windows.net:443/fs/a/2.csv";
var adapterPath3 = "https://storageaccount.blob.core.windows.net/fs/a/3.csv";
var adapterPath4 = "https://storageaccount.blob.core.windows.net:443/fs/a/4.csv";
var corpusPath1 = adlsAdapter.CreateCorpusPath(adapterPath1);
var corpusPath2 = adlsAdapter.CreateCorpusPath(adapterPath2);
var corpusPath3 = adlsAdapter.CreateCorpusPath(adapterPath3);
var corpusPath4 = adlsAdapter.CreateCorpusPath(adapterPath4);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual("/a/2.csv", corpusPath2);
Assert.AreEqual("/a/3.csv", corpusPath3);
Assert.AreEqual("/a/4.csv", corpusPath4);
Assert.AreEqual(adapterPath1, adlsAdapter.CreateAdapterPath(corpusPath1));
Assert.AreEqual(adapterPath2, adlsAdapter.CreateAdapterPath(corpusPath2));
Assert.AreEqual(adapterPath3, adlsAdapter.CreateAdapterPath(corpusPath3));
Assert.AreEqual(adapterPath4, adlsAdapter.CreateAdapterPath(corpusPath4));
// Check that an adapter path is correctly created from a corpus path with any namespace
var corpusPathWithNamespace1 = "adls:/test.json";
var corpusPathWithNamespace2 = "mylake:/test.json";
var expectedAdapterPath = "https://storageaccount.dfs.core.windows.net/fs/test.json";
Assert.AreEqual(expectedAdapterPath, adlsAdapter.CreateAdapterPath(corpusPathWithNamespace1));
Assert.AreEqual(expectedAdapterPath, adlsAdapter.CreateAdapterPath(corpusPathWithNamespace2));
// Check that an adapter path is correctly created from a corpus path with colons
var corpusPathWithColons = "namespace:/a/path:with:colons/some-file.json";
Assert.AreEqual("https://storageaccount.dfs.core.windows.net/fs/a/path:with:colons/some-file.json", adlsAdapter.CreateAdapterPath(corpusPathWithColons));
// Check that an adapter path is null if the corpus path provided is null
Assert.IsNull(adlsAdapter.CreateAdapterPath(null));
var host2 = "storageaccount.blob.core.windows.net:8888";
adlsAdapter = new ADLSAdapter(host2, root, string.Empty);
var adapterPath5 = "https://storageaccount.blob.core.windows.net:8888/fs/a/5.csv";
var adapterPath6 = "https://storageaccount.dfs.core.windows.net:8888/fs/a/6.csv";
var adapterPath7 = "https://storageaccount.blob.core.windows.net/fs/a/7.csv";
Assert.AreEqual("/a/5.csv", adlsAdapter.CreateCorpusPath(adapterPath5));
Assert.AreEqual("/a/6.csv", adlsAdapter.CreateCorpusPath(adapterPath6));
Assert.AreEqual(null, adlsAdapter.CreateCorpusPath(adapterPath7));
}
public void TestInitializeHostnameAndRoot()
{
var host1 = "storageaccount.dfs.core.windows.net";
var adlsAdapter1 = new ADLSAdapter(host1, "root-without-slash", string.Empty);
Assert.AreEqual("storageaccount.dfs.core.windows.net", adlsAdapter1.Hostname);
Assert.AreEqual("/root-without-slash", adlsAdapter1.Root);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/root-without-slash/a/1.csv";
var corpusPath1 = adlsAdapter1.CreateCorpusPath(adapterPath1);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual(adapterPath1, adlsAdapter1.CreateAdapterPath(corpusPath1));
var adlsAdapter1WithFolders = new ADLSAdapter(host1, "root-without-slash/folder1/folder2", string.Empty);
Assert.AreEqual("/root-without-slash/folder1/folder2", adlsAdapter1WithFolders.Root);
var adapterPath2 = "https://storageaccount.dfs.core.windows.net/root-without-slash/folder1/folder2/a/1.csv";
var corpusPath2 = adlsAdapter1WithFolders.CreateCorpusPath(adapterPath2);
Assert.AreEqual("/a/1.csv", corpusPath2);
Assert.AreEqual(adapterPath2, adlsAdapter1WithFolders.CreateAdapterPath(corpusPath2));
var adlsAdapter2 = new ADLSAdapter(host1, "/root-starts-with-slash", string.Empty);
Assert.AreEqual("/root-starts-with-slash", adlsAdapter2.Root);
var adlsAdapter2WithFolders = new ADLSAdapter(host1, "/root-starts-with-slash/folder1/folder2", string.Empty);
Assert.AreEqual("/root-starts-with-slash/folder1/folder2", adlsAdapter2WithFolders.Root);
var adlsAdapter3 = new ADLSAdapter(host1, "root-ends-with-slash/", string.Empty);
Assert.AreEqual("/root-ends-with-slash", adlsAdapter3.Root);
var adlsAdapter3WithFolders = new ADLSAdapter(host1, "root-ends-with-slash/folder1/folder2/", string.Empty);
Assert.AreEqual("/root-ends-with-slash/folder1/folder2", adlsAdapter3WithFolders.Root);
var adlsAdapter4 = new ADLSAdapter(host1, "/root-with-slashes/", string.Empty);
Assert.AreEqual("/root-with-slashes", adlsAdapter4.Root);
var adlsAdapter4WithFolders = new ADLSAdapter(host1, "/root-with-slashes/folder1/folder2/", string.Empty);
Assert.AreEqual("/root-with-slashes/folder1/folder2", adlsAdapter4WithFolders.Root);
// Mount from config
var config = TestHelper.GetInputFileContent(testSubpath, nameof(TestInitializeHostnameAndRoot), "config.json");
var corpus = new CdmCorpusDefinition();
corpus.Storage.MountFromConfig(config);
Assert.AreEqual("/root-without-slash", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter1")).Root);
Assert.AreEqual("/root-without-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter2")).Root);
Assert.AreEqual("/root-starts-with-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter3")).Root);
Assert.AreEqual("/root-ends-with-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter4")).Root);
Assert.AreEqual("/root-with-slashes/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter5")).Root);
}
public void TestCreateCorpusAndAdapterPathInAdlsAdapter()
{
var host1 = "storageaccount.dfs.core.windows.net";
var root = "/fs";
var adlsAdapter = new ADLSAdapter(host1, root, string.Empty);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/fs/a/1.csv";
var adapterPath2 = "https://storageaccount.dfs.core.windows.net:443/fs/a/2.csv";
var adapterPath3 = "https://storageaccount.blob.core.windows.net/fs/a/3.csv";
var adapterPath4 = "https://storageaccount.blob.core.windows.net:443/fs/a/4.csv";
var corpusPath1 = adlsAdapter.CreateCorpusPath(adapterPath1);
var corpusPath2 = adlsAdapter.CreateCorpusPath(adapterPath2);
var corpusPath3 = adlsAdapter.CreateCorpusPath(adapterPath3);
var corpusPath4 = adlsAdapter.CreateCorpusPath(adapterPath4);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual("/a/2.csv", corpusPath2);
Assert.AreEqual("/a/3.csv", corpusPath3);
Assert.AreEqual("/a/4.csv", corpusPath4);
Assert.AreEqual(adapterPath1, adlsAdapter.CreateAdapterPath(corpusPath1));
Assert.AreEqual(adapterPath2, adlsAdapter.CreateAdapterPath(corpusPath2));
Assert.AreEqual(adapterPath3, adlsAdapter.CreateAdapterPath(corpusPath3));
Assert.AreEqual(adapterPath4, adlsAdapter.CreateAdapterPath(corpusPath4));
var host2 = "storageaccount.blob.core.windows.net:8888";
adlsAdapter = new ADLSAdapter(host2, root, string.Empty);
var adapterPath5 = "https://storageaccount.blob.core.windows.net:8888/fs/a/5.csv";
var adapterPath6 = "https://storageaccount.dfs.core.windows.net:8888/fs/a/6.csv";
var adapterPath7 = "https://storageaccount.blob.core.windows.net/fs/a/7.csv";
Assert.AreEqual("/a/5.csv", adlsAdapter.CreateCorpusPath(adapterPath5));
Assert.AreEqual("/a/6.csv", adlsAdapter.CreateCorpusPath(adapterPath6));
Assert.AreEqual(null, adlsAdapter.CreateCorpusPath(adapterPath7));
}
public void TestFormattedHostname()
{
var config = new JObject
{
{ "hostname", "hostname.dfs.core.windows.net" },
{ "root", "root" },
{ "tenant", "tenant" },
{ "clientId", "clientId" }
};
var adlsAdapter = new ADLSAdapter();
adlsAdapter.UpdateConfig(config.ToString());
var corpusPath = adlsAdapter.CreateCorpusPath("https://hostname.dfs.core.windows.net/root/partitions/data.csv");
Assert.AreEqual("/partitions/data.csv", corpusPath);
}
public void TestHostnameWithLeadingProtocol()
{
var host1 = "https://storageaccount.dfs.core.windows.net";
var adlsAdapter1 = new ADLSAdapter(host1, "root-without-slash", string.Empty);
var adapterPath = "https://storageaccount.dfs.core.windows.net/root-without-slash/a/1.csv";
var corpusPath1 = adlsAdapter1.CreateCorpusPath(adapterPath);
Assert.AreEqual("https://storageaccount.dfs.core.windows.net", adlsAdapter1.Hostname);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual(adapterPath, adlsAdapter1.CreateAdapterPath(corpusPath1));
var host2 = "HttPs://storageaccount.dfs.core.windows.net";
var adlsAdapter2 = new ADLSAdapter(host2, "root-without-slash", string.Empty);
var corpusPath2 = adlsAdapter2.CreateCorpusPath(adapterPath);
Assert.AreEqual("HttPs://storageaccount.dfs.core.windows.net", adlsAdapter2.Hostname);
Assert.AreEqual("/a/1.csv", corpusPath2);
Assert.AreEqual(adapterPath, adlsAdapter2.CreateAdapterPath(corpusPath2));
try
{
var host3 = "http://storageaccount.dfs.core.windows.net";
var adlsAdapter3 = new ADLSAdapter(host3, "root-without-slash", string.Empty);
Assert.Fail("Expected Exception for using a http:// hostname.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is ArgumentException);
}
try
{
var host4 = "https://bar:baz::]/foo/";
var adlsAdapter4 = new ADLSAdapter(host4, "root-without-slash", string.Empty);
Assert.Fail("Expected Exception for using and invalid hostname.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is ArgumentException);
}
}