private static async Task RunFileEnumTest(ADLSAdapter adapter)
{
using (adapter.CreateFileQueryCacheContext())
{
List <string> files1 = await adapter.FetchAllFilesAsync("/FileEnumTest/");
List <string> files2 = await adapter.FetchAllFilesAsync("/FileEnumTest");
List <string> files3 = await adapter.FetchAllFilesAsync("FileEnumTest/");
List <string> files4 = await adapter.FetchAllFilesAsync("FileEnumTest");
// expect 100 files to be enumerated
Assert.IsTrue(files1.Count == 100 && files2.Count == 100 && files3.Count == 100 && files4.Count == 100);
// these calls should be fast due to cache
var watch = Stopwatch.StartNew();
for (int i = 0; i < files1.Count; i++)
{
Assert.IsTrue(files1[i] == files2[i] && files1[i] == files3[i] && files1[i] == files4[i]);
await adapter.ComputeLastModifiedTimeAsync(files1[i]);
}
watch.Stop();
Assert.Performance(10, watch.ElapsedMilliseconds, "Cached file modified times");
}
}
public async Task ADLSWriteClientIdLargeFileContentsNoEmptyFileLeft()
{
AdlsTestHelper.CheckADLSEnvironment();
string filename = "largefilecheck_CSharp.txt";
string writeContents = new string('A', 100000000);
ADLSAdapter adapter = AdlsTestHelper.CreateAdapterWithClientId();
adapter.Ctx = new ResolveContext(null, null);
adapter.Ctx.FeatureFlags.Add("ADLSAdapter_deleteEmptyFile", true);
try
{
await adapter.WriteAsync(filename, writeContents);
}
catch (Exception e)
{ }
try
{
await adapter.ReadAsync(filename);
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("The specified path does not exist"));
}
}
private static async Task RunSpecialCharactersTest(ADLSAdapter adapter)
{
var corpus = new CdmCorpusDefinition();
corpus.Storage.Mount("adls", adapter);
corpus.Storage.DefaultNamespace = "adls";
try
{
var manifest = await corpus.FetchObjectAsync <CdmManifestDefinition>("default.manifest.cdm.json");
await manifest.FileStatusCheckAsync();
Assert.AreEqual(1, manifest.Entities.Count);
Assert.AreEqual(2, manifest.Entities[0].DataPartitions.Count);
Assert.AreEqual(
"TestEntity-With=Special Characters/year=2020/TestEntity-partition-With=Special Characters-0.csv",
manifest.Entities[0].DataPartitions[0].Location);
Assert.AreEqual(
"TestEntity-With=Special Characters/year=2020/TestEntity-partition-With=Special Characters-1.csv",
manifest.Entities[0].DataPartitions[1].Location);
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}
public void TestInitializeHostnameAndRoot()
{
var host1 = "storageaccount.dfs.core.windows.net";
var adlsAdapter1 = new ADLSAdapter(host1, "root-without-slash", string.Empty);
Assert.AreEqual("storageaccount.dfs.core.windows.net", adlsAdapter1.Hostname);
Assert.AreEqual("/root-without-slash", adlsAdapter1.Root);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/root-without-slash/a/1.csv";
var corpusPath1 = adlsAdapter1.CreateCorpusPath(adapterPath1);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual(adapterPath1, adlsAdapter1.CreateAdapterPath(corpusPath1));
var adlsAdapter1WithFolders = new ADLSAdapter(host1, "root-without-slash/folder1/folder2", string.Empty);
Assert.AreEqual("/root-without-slash/folder1/folder2", adlsAdapter1WithFolders.Root);
var adapterPath2 = "https://storageaccount.dfs.core.windows.net/root-without-slash/folder1/folder2/a/1.csv";
var corpusPath2 = adlsAdapter1WithFolders.CreateCorpusPath(adapterPath2);
Assert.AreEqual("/a/1.csv", corpusPath2);
Assert.AreEqual(adapterPath2, adlsAdapter1WithFolders.CreateAdapterPath(corpusPath2));
var adlsAdapter2 = new ADLSAdapter(host1, "/root-starts-with-slash", string.Empty);
Assert.AreEqual("/root-starts-with-slash", adlsAdapter2.Root);
var adlsAdapter2WithFolders = new ADLSAdapter(host1, "/root-starts-with-slash/folder1/folder2", string.Empty);
Assert.AreEqual("/root-starts-with-slash/folder1/folder2", adlsAdapter2WithFolders.Root);
var adlsAdapter3 = new ADLSAdapter(host1, "root-ends-with-slash/", string.Empty);
Assert.AreEqual("/root-ends-with-slash", adlsAdapter3.Root);
var adlsAdapter3WithFolders = new ADLSAdapter(host1, "root-ends-with-slash/folder1/folder2/", string.Empty);
Assert.AreEqual("/root-ends-with-slash/folder1/folder2", adlsAdapter3WithFolders.Root);
var adlsAdapter4 = new ADLSAdapter(host1, "/root-with-slashes/", string.Empty);
Assert.AreEqual("/root-with-slashes", adlsAdapter4.Root);
var adlsAdapter4WithFolders = new ADLSAdapter(host1, "/root-with-slashes/folder1/folder2/", string.Empty);
Assert.AreEqual("/root-with-slashes/folder1/folder2", adlsAdapter4WithFolders.Root);
// Mount from config
var config = TestHelper.GetInputFileContent(testSubpath, nameof(TestInitializeHostnameAndRoot), "config.json");
var corpus = new CdmCorpusDefinition();
corpus.Storage.MountFromConfig(config);
Assert.AreEqual("/root-without-slash", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter1")).Root);
Assert.AreEqual("/root-without-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter2")).Root);
Assert.AreEqual("/root-starts-with-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter3")).Root);
Assert.AreEqual("/root-ends-with-slash/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter4")).Root);
Assert.AreEqual("/root-with-slashes/folder1/folder2", ((ADLSAdapter)corpus.Storage.FetchAdapter("adlsadapter5")).Root);
}
private static async Task RunWriteReadTest(ADLSAdapter adapter)
{
string filename = $"WriteReadTest/{Environment.GetEnvironmentVariable("USERNAME")}_{Environment.GetEnvironmentVariable("COMPUTERNAME")}_CSharp.txt";
string writeContents = $"{DateTimeOffset.Now}\n{filename}";
await adapter.WriteAsync(filename, writeContents);
string readContents = await adapter.ReadAsync(filename);
Assert.IsTrue(string.Equals(writeContents, readContents));
}
private static async Task RunCheckFileTimeTest(ADLSAdapter adapter)
{
DateTimeOffset?offset1 = await adapter.ComputeLastModifiedTimeAsync("/FileTimeTest/CheckFileTime.txt");
DateTimeOffset?offset2 = await adapter.ComputeLastModifiedTimeAsync("FileTimeTest/CheckFileTime.txt");
Assert.IsTrue(offset1.HasValue);
Assert.IsTrue(offset2.HasValue);
Assert.IsTrue(offset1.Value == offset2.Value);
Assert.IsTrue(offset1.Value < DateTimeOffset.Now);
}
public void TestCreateCorpusAndAdapterPathInAdlsAdapter()
{
var host1 = "storageaccount.dfs.core.windows.net";
var root = "/fs";
var adlsAdapter = new ADLSAdapter(host1, root, string.Empty);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/fs/a/1.csv";
var adapterPath2 = "https://storageaccount.dfs.core.windows.net:443/fs/a/2.csv";
var adapterPath3 = "https://storageaccount.blob.core.windows.net/fs/a/3.csv";
var adapterPath4 = "https://storageaccount.blob.core.windows.net:443/fs/a/4.csv";
var corpusPath1 = adlsAdapter.CreateCorpusPath(adapterPath1);
var corpusPath2 = adlsAdapter.CreateCorpusPath(adapterPath2);
var corpusPath3 = adlsAdapter.CreateCorpusPath(adapterPath3);
var corpusPath4 = adlsAdapter.CreateCorpusPath(adapterPath4);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual("/a/2.csv", corpusPath2);
Assert.AreEqual("/a/3.csv", corpusPath3);
Assert.AreEqual("/a/4.csv", corpusPath4);
Assert.AreEqual(adapterPath1, adlsAdapter.CreateAdapterPath(corpusPath1));
Assert.AreEqual(adapterPath2, adlsAdapter.CreateAdapterPath(corpusPath2));
Assert.AreEqual(adapterPath3, adlsAdapter.CreateAdapterPath(corpusPath3));
Assert.AreEqual(adapterPath4, adlsAdapter.CreateAdapterPath(corpusPath4));
// Check that an adapter path is correctly created from a corpus path with any namespace
var corpusPathWithNamespace1 = "adls:/test.json";
var corpusPathWithNamespace2 = "mylake:/test.json";
var expectedAdapterPath = "https://storageaccount.dfs.core.windows.net/fs/test.json";
Assert.AreEqual(expectedAdapterPath, adlsAdapter.CreateAdapterPath(corpusPathWithNamespace1));
Assert.AreEqual(expectedAdapterPath, adlsAdapter.CreateAdapterPath(corpusPathWithNamespace2));
// Check that an adapter path is correctly created from a corpus path with colons
var corpusPathWithColons = "namespace:/a/path:with:colons/some-file.json";
Assert.AreEqual("https://storageaccount.dfs.core.windows.net/fs/a/path:with:colons/some-file.json", adlsAdapter.CreateAdapterPath(corpusPathWithColons));
// Check that an adapter path is null if the corpus path provided is null
Assert.IsNull(adlsAdapter.CreateAdapterPath(null));
var host2 = "storageaccount.blob.core.windows.net:8888";
adlsAdapter = new ADLSAdapter(host2, root, string.Empty);
var adapterPath5 = "https://storageaccount.blob.core.windows.net:8888/fs/a/5.csv";
var adapterPath6 = "https://storageaccount.dfs.core.windows.net:8888/fs/a/6.csv";
var adapterPath7 = "https://storageaccount.blob.core.windows.net/fs/a/7.csv";
Assert.AreEqual("/a/5.csv", adlsAdapter.CreateCorpusPath(adapterPath5));
Assert.AreEqual("/a/6.csv", adlsAdapter.CreateCorpusPath(adapterPath6));
Assert.AreEqual(null, adlsAdapter.CreateCorpusPath(adapterPath7));
}
public static ADLSAdapter CreateAdapterWithSharedKey(string rootRelativePath = null)
{
string hostname = Environment.GetEnvironmentVariable("ADLS_HOSTNAME");
string rootPath = Environment.GetEnvironmentVariable("ADLS_ROOTPATH");
string sharedKey = Environment.GetEnvironmentVariable("ADLS_SHAREDKEY");
Assert.IsFalse(String.IsNullOrEmpty(hostname), "ADLS_ENDPOINT not set");
Assert.IsFalse(String.IsNullOrEmpty(rootPath), "ADLS_ROOTPATH not set");
Assert.IsFalse(String.IsNullOrEmpty(sharedKey), "ADLS_SHAREDKEY not set");
ADLSAdapter adapter = new ADLSAdapter(hostname, GetFullRootPath(rootPath, rootRelativePath), sharedKey);
return(adapter);
}
public static ADLSAdapter CreateADLSAdapterWithClientIdWithSharedKey(int adapterNum, string rootRelativePath = null)
{
string hostname = Environment.GetEnvironmentVariable($"SYMS_TEST_ADLS{adapterNum}_HOSTNAME");
string rootPath = Environment.GetEnvironmentVariable($"SYMS_TEST_ADLS{adapterNum}_ROOTPATH");
string sharedkey = Environment.GetEnvironmentVariable($"SYMS_TEST_ADLS{adapterNum}_SHAREDKEY");
Assert.IsFalse(String.IsNullOrEmpty(hostname), $"SYMS_TEST_ADLS{adapterNum}_ENDPOINT not set");
Assert.IsFalse(String.IsNullOrEmpty(rootPath), $"SYMS_TEST_ADLS{adapterNum}_ROOTPATH not set");
Assert.IsFalse(String.IsNullOrEmpty(sharedkey), $"SYMS_TEST_ADLS{adapterNum}_SHAREDKEY not set");
ADLSAdapter adapter = new ADLSAdapter(hostname, rootPath, sharedkey);
return(adapter);
}
public void TestEndpointMissingOnConfig()
{
var config = new JObject
{
{ "hostname", "hostname.dfs.core.windows.net" },
{ "root", "root" },
{ "tenant", "tenant" },
{ "clientId", "clientId" }
};
var adlsAdapter = new ADLSAdapter();
adlsAdapter.UpdateConfig(config.ToString());
Assert.AreEqual(AzureCloudEndpoint.AzurePublic, adlsAdapter.Endpoint);
}
/// <summary>
/// Create the cdm message processor
/// </summary>
/// <param name="config"></param>
/// <param name="logger"></param>
/// <param name="storage"></param>
public CdmMessageProcessor(ICdmClientConfig config,
ILogger logger, IAdlsStorage storage)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_config = config ?? throw new ArgumentNullException(nameof(config));
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
_lock = new SemaphoreSlim(1, 1);
_samplesCacheSize = 0;
_cacheUploadTimer = new Timer(CacheTimer_ElapesedAsync);
_cacheUploadTriggered = false;
_cacheUploadInterval = TimeSpan.FromSeconds(20);
_samplesCache = new Dictionary <string, List <MonitoredItemSampleModel> >();
_dataSetsCache = new Dictionary <string, List <DataSetMessageModel> >();
_cdmCorpus = new CdmCorpusDefinition();
_cdmCorpus.SetEventCallback(new EventCallback {
Invoke = (level, msg) => {
switch (level)
{
case CdmStatusLevel.Error:
_logger.Error("CDM message: {0}", msg);
break;
case CdmStatusLevel.Warning:
_logger.Warning("CDM message: {0}", msg);
break;
case CdmStatusLevel.Progress:
_logger.Verbose("CDM message: {0}", msg);
break;
case CdmStatusLevel.Info:
_logger.Debug("CDM message: {0}", msg);
break;
}
}
});
_adapter = new ADLSAdapter($"{config.ADLSg2HostName}",
$"/{config.ADLSg2ContainerName}/{config.RootFolder}", config.TenantId, config.AppId, config.AppSecret);
_cdmCorpus.Storage.Mount("adls", _adapter);
var gitAdapter = new GithubAdapter();
_cdmCorpus.Storage.Mount("cdm", gitAdapter);
_cdmCorpus.Storage.DefaultNamespace = "adls";
}
public void TestFormattedHostname()
{
var config = new JObject
{
{ "hostname", "hostname.dfs.core.windows.net" },
{ "root", "root" },
{ "tenant", "tenant" },
{ "clientId", "clientId" }
};
var adlsAdapter = new ADLSAdapter();
adlsAdapter.UpdateConfig(config.ToString());
var corpusPath = adlsAdapter.CreateCorpusPath("https://hostname.dfs.core.windows.net/root/partitions/data.csv");
Assert.AreEqual("/partitions/data.csv", corpusPath);
}
public static ADLSAdapter CreateAdapterWithClientId(string rootRelativePath = null)
{
string hostname = Environment.GetEnvironmentVariable("ADLS_HOSTNAME");
string rootPath = Environment.GetEnvironmentVariable("ADLS_ROOTPATH");
string tenant = Environment.GetEnvironmentVariable("ADLS_TENANT");
string clientId = Environment.GetEnvironmentVariable("ADLS_CLIENTID");
string clientSecret = Environment.GetEnvironmentVariable("ADLS_CLIENTSECRET");
Assert.IsFalse(String.IsNullOrEmpty(hostname), "ADLS_ENDPOINT not set");
Assert.IsFalse(String.IsNullOrEmpty(rootPath), "ADLS_ROOTPATH not set");
Assert.IsFalse(String.IsNullOrEmpty(tenant), "ADLS_TENANT not set");
Assert.IsFalse(String.IsNullOrEmpty(clientId), "ADLS_CLIENTID not set");
Assert.IsFalse(String.IsNullOrEmpty(clientSecret), "ADLS_CLIENTSECRET not set");
ADLSAdapter adapter = new ADLSAdapter(hostname, GetFullRootPath(rootPath, rootRelativePath), tenant, clientId, clientSecret);
return(adapter);
}
public void TestHostnameWithLeadingProtocol()
{
var host1 = "https://storageaccount.dfs.core.windows.net";
var adlsAdapter1 = new ADLSAdapter(host1, "root-without-slash", string.Empty);
var adapterPath = "https://storageaccount.dfs.core.windows.net/root-without-slash/a/1.csv";
var corpusPath1 = adlsAdapter1.CreateCorpusPath(adapterPath);
Assert.AreEqual("https://storageaccount.dfs.core.windows.net", adlsAdapter1.Hostname);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual(adapterPath, adlsAdapter1.CreateAdapterPath(corpusPath1));
var host2 = "HttPs://storageaccount.dfs.core.windows.net";
var adlsAdapter2 = new ADLSAdapter(host2, "root-without-slash", string.Empty);
var corpusPath2 = adlsAdapter2.CreateCorpusPath(adapterPath);
Assert.AreEqual("HttPs://storageaccount.dfs.core.windows.net", adlsAdapter2.Hostname);
Assert.AreEqual("/a/1.csv", corpusPath2);
Assert.AreEqual(adapterPath, adlsAdapter2.CreateAdapterPath(corpusPath2));
try
{
var host3 = "http://storageaccount.dfs.core.windows.net";
var adlsAdapter3 = new ADLSAdapter(host3, "root-without-slash", string.Empty);
Assert.Fail("Expected Exception for using a http:// hostname.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is ArgumentException);
}
try
{
var host4 = "https://bar:baz::]/foo/";
var adlsAdapter4 = new ADLSAdapter(host4, "root-without-slash", string.Empty);
Assert.Fail("Expected Exception for using and invalid hostname.");
}
catch (Exception ex)
{
Assert.IsTrue(ex is ArgumentException);
}
}
public void TestConfigAndUpdateConfigWithoutAuthenticationDetails()
{
var adlsAdapter = new ADLSAdapter();
try
{
var config = new JObject
{
{ "root", "root" },
{ "hostname", "hostname" },
{ "tenant", "tenant" },
{ "clientId", "clientId" }
};
adlsAdapter.UpdateConfig(config.ToString());
adlsAdapter.Secret = "secret";
adlsAdapter.SharedKey = "sharedKey";
}
catch
{
Assert.Fail("AdlsAdapter initialized without secret shouldn't throw exception when updating config.");
}
}
public void TestCreateCorpusAndAdapterPathInAdlsAdapter()
{
var host1 = "storageaccount.dfs.core.windows.net";
var root = "/fs";
var adlsAdapter = new ADLSAdapter(host1, root, string.Empty);
var adapterPath1 = "https://storageaccount.dfs.core.windows.net/fs/a/1.csv";
var adapterPath2 = "https://storageaccount.dfs.core.windows.net:443/fs/a/2.csv";
var adapterPath3 = "https://storageaccount.blob.core.windows.net/fs/a/3.csv";
var adapterPath4 = "https://storageaccount.blob.core.windows.net:443/fs/a/4.csv";
var corpusPath1 = adlsAdapter.CreateCorpusPath(adapterPath1);
var corpusPath2 = adlsAdapter.CreateCorpusPath(adapterPath2);
var corpusPath3 = adlsAdapter.CreateCorpusPath(adapterPath3);
var corpusPath4 = adlsAdapter.CreateCorpusPath(adapterPath4);
Assert.AreEqual("/a/1.csv", corpusPath1);
Assert.AreEqual("/a/2.csv", corpusPath2);
Assert.AreEqual("/a/3.csv", corpusPath3);
Assert.AreEqual("/a/4.csv", corpusPath4);
Assert.AreEqual(adapterPath1, adlsAdapter.CreateAdapterPath(corpusPath1));
Assert.AreEqual(adapterPath2, adlsAdapter.CreateAdapterPath(corpusPath2));
Assert.AreEqual(adapterPath3, adlsAdapter.CreateAdapterPath(corpusPath3));
Assert.AreEqual(adapterPath4, adlsAdapter.CreateAdapterPath(corpusPath4));
var host2 = "storageaccount.blob.core.windows.net:8888";
adlsAdapter = new ADLSAdapter(host2, root, string.Empty);
var adapterPath5 = "https://storageaccount.blob.core.windows.net:8888/fs/a/5.csv";
var adapterPath6 = "https://storageaccount.dfs.core.windows.net:8888/fs/a/6.csv";
var adapterPath7 = "https://storageaccount.blob.core.windows.net/fs/a/7.csv";
Assert.AreEqual("/a/5.csv", adlsAdapter.CreateCorpusPath(adapterPath5));
Assert.AreEqual("/a/6.csv", adlsAdapter.CreateCorpusPath(adapterPath6));
Assert.AreEqual(null, adlsAdapter.CreateCorpusPath(adapterPath7));
}
public async Task ADLSWriteClientIdEmptyContentsNoEmptyFileLeft()
{
AdlsTestHelper.CheckADLSEnvironment();
string filename = "emptycheck_CSharp.txt";
string writeContents = "";
ADLSAdapter adapter = AdlsTestHelper.CreateAdapterWithClientId();
try
{
await adapter.WriteAsync(filename, writeContents);
}
catch (Exception e)
{}
try
{
await adapter.ReadAsync(filename);
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("The specified path does not exist"));
}
}
static void ConfigureADLSAdapter()
{
Console.WriteLine("\nEnter 1 to configure the ADLSAdapter through a shared key authentication. Enter 2 to configure through a token authentication.");
int choice = 1;
while (true)
{
// Get the user's input.
string input = Console.ReadLine().Trim();
if (!string.IsNullOrWhiteSpace(input))
{
if (int.TryParse(input, out choice) && (choice == 1 || choice == 2))
{
break;
}
}
Console.WriteLine("\nEnter 1 or 2.");
}
// Shared key authentication selected.
if (choice == 1)
{
string hostname = GetParameterValueFromUser("hostname", "ADLSAdapter", "test.dfs.core.windows.net" /* this is just to show what the value should look like. */);
string root = GetParameterValueFromUser("root", "ADLSAdapter", "../../../../../../example-public-standards");
// DEV-NOTE: This is just a mock shared key used to demonstrate what a shared key should look like. It is not a real shared key.
string sharedKey = GetParameterValueFromUser("shared key", "ADLSAdapter", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
// Default values for the optional parameters used by the ADLS adapter.
string timeout = "2000";
string maximumTimeout = "10000";
string numberOfRetries = "2";
// Ask the user if optional parameters should be configured, or if defaults should just be used.
if (ConfigureOptionalParameters("ADLSAdapter"))
{
timeout = GetOptionalParameterValueFromUser("timeout", "ADLSAdapter", timeout /* this is just to show what the value should look like. */);
maximumTimeout = GetOptionalParameterValueFromUser("maximum timeout", "ADLSAdapter", maximumTimeout);
numberOfRetries = GetOptionalParameterValueFromUser("number of retries", "ADLSAdapter", numberOfRetries);
}
// Create an ADLS adapter with the parameter values given by the user.
var adapter = new ADLSAdapter(hostname, root, sharedKey)
{
Timeout = TimeSpan.FromMilliseconds(int.Parse(timeout)),
MaximumTimeout = TimeSpan.FromMilliseconds(int.Parse(maximumTimeout)),
NumberOfRetries = int.Parse(numberOfRetries)
// WaitTimeCallback is another optional parameter and can also be configured here.
};
// List the newly configured adapter's properties.
Console.WriteLine("\nADLSAdapter configured. Properties of this ADLSAdapter are:");
Console.WriteLine(" Hostname: " + adapter.Hostname);
Console.WriteLine(" Root: " + adapter.Root);
Console.WriteLine(" SharedKey: " + adapter.SharedKey);
Console.WriteLine(" Timeout: " + adapter.Timeout.Value.TotalMilliseconds);
Console.WriteLine(" MaximumTimeout: " + adapter.MaximumTimeout.Value.TotalMilliseconds);
Console.WriteLine(" NumberOfRetries: " + adapter.NumberOfRetries);
Console.WriteLine();
}
// Token (clientId/secret) authentication selected.
else
{
string hostname = GetParameterValueFromUser("hostname", "ADLSAdapter", "test.dfs.core.windows.net" /* this is just to show what the value should look like. */);
string root = GetParameterValueFromUser("root path", "ADLSAdapter", "../../../../../../example-public-standards");
string tenant = GetParameterValueFromUser("tenant", "ADLSAdapter", "00x000xx-00x0-00xx-00xx-0x0xx000xx00");
string clientId = GetParameterValueFromUser("client ID", "ADLSAdapter", "xxx00x0x-0x00-0000-x0x0-00xxx000xxx0");
// DEV-NOTE: This is just a mock secret used to demonstrate what a secret should look like. It is not a real secret.
string secret = GetParameterValueFromUser("secret", "ADLSAdapter", "xSDfdzI92d:sd832j8jd@ac823sSglJ");
// Default values for the optional parameters used by the ADLS adapter.
string timeout = "2000";
string maximumTimeout = "10000";
string numberOfRetries = "2";
// Ask the user if optional parameters should be configured, or if defaults should just be used.
if (ConfigureOptionalParameters("ADLSAdapter"))
{
timeout = GetOptionalParameterValueFromUser("timeout", "ADLSAdapter", timeout /* this is just to show what the value should look like. */);
maximumTimeout = GetOptionalParameterValueFromUser("maximum timeout", "ADLSAdapter", maximumTimeout);
numberOfRetries = GetOptionalParameterValueFromUser("number of retries", "ADLSAdapter", numberOfRetries);
}
// Create an ADLS adapter with the parameter values given by the user.
var adapter = new ADLSAdapter(hostname, root, tenant, clientId, secret)
{
Timeout = TimeSpan.FromMilliseconds(int.Parse(timeout)),
MaximumTimeout = TimeSpan.FromMilliseconds(int.Parse(maximumTimeout)),
NumberOfRetries = int.Parse(numberOfRetries)
// WaitTimeCallback is another optional parameter and can also be configured here.
};
// List the newly configured adapter's properties.
Console.WriteLine("\nADLSAdapter configured. Properties of this ADLSAdapter are:");
Console.WriteLine(" Hostname: " + adapter.Hostname);
Console.WriteLine(" Root: " + adapter.Root);
Console.WriteLine(" Tenant: " + adapter.Tenant);
Console.WriteLine(" ClientId: " + adapter.ClientId);
Console.WriteLine(" Secret: " + adapter.Secret);
Console.WriteLine(" Timeout: " + adapter.Timeout.Value.TotalMilliseconds);
Console.WriteLine(" MaximumTimeout: " + adapter.MaximumTimeout.Value.TotalMilliseconds);
Console.WriteLine(" NumberOfRetries: " + adapter.NumberOfRetries);
Console.WriteLine();
}
}
