private static void ConfigureADLSClient(string azureTenantId, string clientId, string spKey, string adlsAccountName)
{
try
{
System.Uri ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/");
//
System.Uri ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/");
//
var armCreds = GetCreds_SPI_SecretKey(azureTenantId, ARM_TOKEN_AUDIENCE, clientId, spKey);
//
var adlCreds = GetCreds_SPI_SecretKey(azureTenantId, ADL_TOKEN_AUDIENCE, clientId, spKey);
//
client = AdlsClient.CreateClient(adlsAccountName, adlCreds);
}
catch (Exception ex)
{
//
WriteExceptionToConsole(ex);
//
//telemetryClient.TrackException(ex);
////
//telemetryClient.TrackEvent("ConfigureADLSClient() Failed" + " : Process Id = " + _folderId);
}
finally
{
//
//telemetryClient.Flush();
}
}
internal MakeDirJob(string sourceDir, string dirNm, AdlsClient client, bool isUpload) : base(0)
{
SourceDirectoryNm = sourceDir;
DestDirectoryNm = dirNm;
Client = client;
IsUpload = isUpload;
}
static void Main(string[] args)
{
// Service principal / appplication authentication with client secret / key
// Use the client ID of an existing AAD "Web App" application.
string TENANT = "6c5ccde4-18b5-4936-ae88-995adc1f3c22";
string CLIENTID = "0649e374-afa1-44e6-a061-c44944a156d1";
System.Uri ARM_TOKEN_AUDIENCE = new System.Uri(@"https://management.core.windows.net/");
System.Uri ADL_TOKEN_AUDIENCE = new System.Uri(@"https://datalake.azure.net/");
string secret_key = "oVOdbXKU28RE5oH2symmWXp3sYOoG1geV8x2q4nrgm0=";
var armCreds = GetCreds_SPI_SecretKey(TENANT, ARM_TOKEN_AUDIENCE, CLIENTID, secret_key);
var adlCreds = GetCreds_SPI_SecretKey(TENANT, ADL_TOKEN_AUDIENCE, CLIENTID, secret_key);
client = AdlsClient.CreateClient(_adlsAccountName, adlCreds);
string fileName = "/Github/2018050810/pullrequest";
byte[] textByteArray = Encoding.ASCII.GetBytes("1");
client.ConcurrentAppend(fileName, true, textByteArray, 0, textByteArray.Length);
List <Task> tasks = new List <Task>();
Random random = new Random();
// Execute the task 10 times.
for (int ctr = 1; ctr <= 90000; ctr++)
{
tasks.Add(Task.Factory.StartNew(() => {
textByteArray = Encoding.UTF8.GetBytes(string.Format("{0}:{1}\r\n", ctr, random.Next(1, 10000)));
client.ConcurrentAppend(fileName, true, textByteArray, 0, textByteArray.Length);
}));
}
Task.WaitAll(tasks.ToArray());
}
private static void ModifyAdlsAcl(TraceWriter log, AdlsClient adlsClient)
{
var aclEntry = new List <AclEntry>()
{
new AclEntry(AclType.other, null, AclScope.Access, AclAction.All),
new AclEntry(AclType.other, null, AclScope.Default, AclAction.All)
};
adlsClient.ModifyAclEntries("/", aclEntry);
log.Info($"Set access and default ACL for /");
/* Function App doesn't support recursive ACL modificaitons -- April 5, 2017
* var aclEntry = new List<AclEntry>()
* {
* new AclEntry(AclType.other, null, AclScope.Access, AclAction.All)
* };
* var aclProcessorStats = adlsClient.ChangeAcl("/", aclEntry, RequestedAclType.ModifyAcl);
* log.Info($"Set access ACL for {aclProcessorStats.DirectoryProcessed} directories and {aclProcessorStats.FilesProcessed} files");
* aclEntry = new List<AclEntry>()
* {
* new AclEntry(AclType.other, null, AclScope.Default, AclAction.All)
* };
* aclProcessorStats = adlsClient.ChangeAcl("/", aclEntry, RequestedAclType.ModifyAcl);
* log.Info($"Set default ACL for {aclProcessorStats.DirectoryProcessed} directories and {aclProcessorStats.FilesProcessed} files");*/
}
public static void SetupTest(TestContext context)
{
_adlsClient = SdkUnitTest.SetupSuperClient();
_adlsClient.DeleteRecursive(RemotePath);
_adlsClient.CreateDirectory(RemotePath);
AdlsClient.ConcatenateStreamListThreshold = 10;
}
public void TestRetry()
{
int port = 8080;
AdlsClient adlsClient = AdlsClient.CreateClientWithoutAccntValidation(MockWebServer.Host + ":" + port, TestToken);
MockWebServer server = new MockWebServer(port);
server.StartServer();
server.EnqueMockResponse(new MockResponse(502, "Bad Gateway"));
server.EnqueMockResponse(new MockResponse(503, "Service Unavailable"));
server.EnqueMockResponse(new MockResponse(504, "Gateway Timeout"));
server.EnqueMockResponse(new MockResponse(503, "Service Unavailable"));
server.EnqueMockResponse(new MockResponse(503, "Service Unavailable"));
RequestOptions req = new RequestOptions(new ExponentialRetryPolicy());
OperationResponse resp = new OperationResponse();
adlsClient.SetInsecureHttp();
Stopwatch watch = Stopwatch.StartNew();
Core.AppendAsync("/Test/dir", null, null, SyncFlag.DATA, 0, null, -1, 0, adlsClient, req, resp).GetAwaiter()
.GetResult();
watch.Stop();
long time = watch.ElapsedMilliseconds;
Assert.IsTrue(time >= 83500 && time <= 86500);
Assert.IsTrue(resp.HttpStatus == (HttpStatusCode)503);
Assert.IsTrue(resp.HttpMessage.Equals("Service Unavailable"));
Assert.IsTrue(resp.Retries == 4);
server.StopServer();
}
/// <summary>
/// Serializes the client FQDN, queryparams and token into a request URL
/// </summary>
/// <param name="op">Operation</param>
/// <param name="path">Path of directory or file</param>
/// <param name="client">AdlsClient</param>
/// <param name="resp">OperationResponse</param>
/// <param name="queryParams">Serialized queryparams</param>
/// <returns>URL</returns>
private static string CreateHttpRequestUrl(Operation op, string path, AdlsClient client, OperationResponse resp, string queryParams)
{
StringBuilder urlString = new StringBuilder(UrlLength);
urlString.Append(client.GetHttpPrefix());
urlString.Append("://");
urlString.Append(client.AccountFQDN);
urlString.Append(op.Namespace);
try
{
urlString.Append(Uri.EscapeDataString(path));
}
catch (UriFormatException ex)
{
resp.Error = "UriFormatException: " + ex.Message;
return(null);
}
urlString.Append("?");
urlString.Append(queryParams);
try
{
var uri = new Uri(urlString.ToString());
}
catch (UriFormatException ur)
{
resp.Error = "UriFormatException: " + ur.Message;
return(null);
}
return(urlString.ToString());
}
private void run(object data)
{
RequestState state = data as RequestState;
if (state == null)
{
return;
}
AdlsClient adlsClient = state.AdlsClient;
OperationResponse resp = new OperationResponse();
adlsClient.SetInsecureHttp();
RequestOptions req = null;
if (state.Timeout != -1)
{
req = new RequestOptions(Guid.NewGuid().ToString(), new TimeSpan(0, 0, state.Timeout), state.IsRetry ? new ExponentialRetryPolicy() : (RetryPolicy) new NoRetryPolicy());
}
else
{
req = new RequestOptions(state.IsRetry ? new ExponentialRetryPolicy() : (RetryPolicy) new NoRetryPolicy());
}
byte[] ip = Encoding.UTF8.GetBytes("wait:300");
Core.AppendAsync("/Test/dir", null, null, SyncFlag.DATA, 0, ip, 0, ip.Length, adlsClient, req, resp, state.CancelToken).GetAwaiter().GetResult();
state.Ex = resp.Ex;
state.IsConnectionFailure = resp.ConnectionFailure;
}
private static void PerformConcat(AdlsClient client)
{
// Create two files
string file1 = "/dir1/testConcat1.txt";
string file2 = "/dir1/testConcat2.txt";
string destFile = "/dir1/testConcat.txt";
using (var createStream = new StreamWriter(client.CreateFile(file1, IfExists.Overwrite)))
{
createStream.WriteLine("This is the first file.");
}
using (var createStream = new StreamWriter(client.CreateFile(file2, IfExists.Overwrite)))
{
createStream.WriteLine("This is the second file.");
}
// Concatenate the files
List <string> list = new List <string> {
file1, file2
};
client.ConcatenateFiles(destFile, list);
// Print the resultant combined file
using (var readStream = new StreamReader(client.GetReadStream(destFile)))
{
string line;
while ((line = readStream.ReadLine()) != null)
{
Console.WriteLine(line);
}
}
}
public async Task WriteAsync(string fullPath, Stream dataStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPath(fullPath);
if (dataStream is null)
{
throw new ArgumentNullException(nameof(dataStream));
}
AdlsClient client = await GetAdlsClientAsync().ConfigureAwait(false);
AdlsOutputStream stream;
if (append)
{
stream = await client.GetAppendStreamAsync(fullPath, cancellationToken).ConfigureAwait(false);
}
else
{
stream = await client.CreateFileAsync(fullPath, IfExists.Overwrite,
createParent : true,
cancelToken : cancellationToken).ConfigureAwait(false);
}
using (stream)
{
await dataStream.CopyToAsync(stream).ConfigureAwait(false);
}
}
private void runClientTimeout(object data)
{
RequestState state = data as RequestState;
if (state == null)
{
return;
}
AdlsClient adlsClient = state.AdlsClient;
OperationResponse resp = new OperationResponse();
adlsClient.SetInsecureHttp();
if (state.Timeout != -1)
{
adlsClient.SetPerRequestTimeout(new TimeSpan(0, 0, state.Timeout));
}
byte[] ip = Encoding.UTF8.GetBytes("wait:300");
try
{
adlsClient.ConcurrentAppendAsync("/Test/dir", true, ip, 0, ip.Length, state.CancelToken).GetAwaiter().GetResult();
}
catch (Exception ex)
{
state.Ex = ex;
}
}
public async Task <Stream> ConvertAsync(DataLakeStoreAttribute input, CancellationToken cancellationToken)
{
// Create ADLS client object
var adlsClient = _adlsClient ?? (_adlsClient = await DataLakeAdlsService.CreateAdlsClientAsync(input.TenantID, input.ClientSecret, input.ApplicationId, input.AccountFQDN));
return(await adlsClient.GetReadStreamAsync(input.FileName));
}
internal static AdlsClient GetAdlsClient(string accntNm, IAzureContext context)
{
accntNm = HandleAccntName(accntNm, context);
lock (ClientFactory)
{
if (ClientFactory.ContainsKey(accntNm))
{
return(ClientFactory[accntNm]);
}
ServiceClientCredentials creds;
if (IsTest)
{
if (MockCredentials != null)
{
creds = MockCredentials;
}
else
{
ClientFactory.Add(accntNm, MockAdlsClient.GetMockClient());
return(ClientFactory[accntNm]);
}
}
else
{
creds = AzureSession.Instance.AuthenticationFactory.GetServiceClientCredentials(context,
AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix);
}
var client = AdlsClient.CreateClient(accntNm, creds);
client.AddUserAgentSuffix(AzurePowerShell.UserAgentValue.ToString());
ClientFactory.Add(accntNm, client);
return(client);
}
}
public static async Task <IActionResult> Run([HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log)
{
log.LogInformation("C# HTTP trigger function is processing a request.");
var creds = new ClientCredential(applicationId, clientSecret);
var clientCreds = ApplicationTokenProvider.LoginSilentAsync(tenantId, creds).GetAwaiter().GetResult();
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
requestBody = requestBody.Replace(System.Environment.NewLine, " ");
// Create ADLS client object
AdlsClient client = AdlsClient.CreateClient(adlsAccountName, clientCreds);
try
{
string fileName = "/yvr/YVR-test";
if (!client.CheckExists(fileName))
{
// Create a file - automatically creates any parent directories that don't exist
// The AdlsOutputStream preserves record boundaries - it does not break records while writing to the store
using (var stream = client.CreateFile(fileName, IfExists.Overwrite))
{
byte[] textByteArray = Encoding.UTF8.GetBytes(requestBody + "\r\n");
stream.Write(textByteArray, 0, textByteArray.Length);
Console.WriteLine("File Created, Data Appended. \n");
log.LogInformation("File Created, Data Appended. \n");
}
}
else
{
// Append to existing file
using (var stream = client.GetAppendStream(fileName))
{
byte[] textByteArray = Encoding.UTF8.GetBytes(requestBody + "\r\n");
stream.Write(textByteArray, 0, textByteArray.Length);
Console.WriteLine("Data Appended. \n");
log.LogInformation("Data Appended. \n");
}
}
//Read file contents
using (var readStream = new StreamReader(client.GetReadStream(fileName)))
{
string line;
while ((line = readStream.ReadLine()) != null)
{
log.LogInformation(line);
Console.WriteLine(line);
}
}
}
catch (AdlsException e)
{
PrintAdlsException(e);
}
return(new OkObjectResult("Success"));
}
public TestAzureDataLakeFileSystemE2E()
{
// Service principal / application authentication with client secret / key
// Use the application ID of an existing AAD "Web App" application as the ClientId and
// use its authentication key as the SecretKey
// https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal#get-application-id-and-authentication-key
// Fill in before running test!
const string AdlsAccountName = "#####.azuredatalakestore.net";
const string Tenant = "microsoft.onmicrosoft.com";
const string TokenAudience = @"https://datalake.azure.net/";
const string ClientId = "########-####-####-####-############"; // e.g. "c2897d56-5eef-4030-8b7a-46b5c0acd05c"
const string SecretKey = "##########"; // e.g. "SecretKey1234!"
_defaultFolderName = "reef-test-folder-" + Guid.NewGuid();
IConfiguration conf = AzureDataLakeFileSystemConfiguration.ConfigurationModule
.Set(AzureDataLakeFileSystemConfiguration.DataLakeStorageAccountName, AdlsAccountName)
.Set(AzureDataLakeFileSystemConfiguration.Tenant, Tenant)
.Set(AzureDataLakeFileSystemConfiguration.ClientId, ClientId)
.Set(AzureDataLakeFileSystemConfiguration.SecretKey, SecretKey)
.Build();
_fileSystem = TangFactory.GetTang().NewInjector(conf).GetInstance <AzureDataLakeFileSystem>();
ServiceClientCredentials adlCreds = GetCredsSpiSecretKey(Tenant, new Uri(TokenAudience), ClientId, SecretKey);
_adlsClient = AdlsClient.CreateClient(AdlsAccountName, adlCreds);
}
public void TestValidAccnt()
{
AdlsClient.CreateClient("contoso.cabostore.net", "Test");
AdlsClient.CreateClient("contoso.dogfood.com.net", "Test");
AdlsClient.CreateClient("contoso-test.azure-data.net", "test");
AdlsClient.CreateClient("contoso-test.Azuredatalakestore.eglax.inc.bov", "test");
}
private PropertyManager(AdlsClient client, bool getAclProperty, bool getDiskUsage, string saveFileName, bool saveToLocal, int numThreads, bool displayFiles, bool displayConsistentAcl, long maxDepth, CancellationToken cancelToken = default(CancellationToken))
{
Client = client;
SaveToLocal = saveToLocal;
if (string.IsNullOrWhiteSpace(saveFileName))
{
throw new ArgumentNullException(nameof(saveFileName));
}
DumpFileName = saveFileName;
GetSizeProperty = getDiskUsage;
GetAclProperty = getAclProperty;
_numThreads = numThreads <= 0 ? AdlsClient.DefaultNumThreads : numThreads;
DisplayFiles = displayFiles;
HideConsistentAclTree = GetAclProperty && displayConsistentAcl;
MaxDepth = maxDepth;
ConsumerQueue = new PriorityQueueWrapper <BaseJob>(_numThreads);
PropertyWriterQueue = new PriorityQueueWrapper <BaseJob>();
_cancelToken = cancelToken;
Stream underLyingStream;
if (saveToLocal)
{
Utils.CreateParentDirectory(DumpFileName);
underLyingStream = new FileStream(DumpFileName, FileMode.Create, FileAccess.ReadWrite);
}
else
{
underLyingStream = client.CreateFile(DumpFileName, IfExists.Overwrite);
}
PropertyDumpWriter = new StreamWriter(underLyingStream);
WriteHeader();
}
private AclProcessor(string path, AdlsClient client, List <AclEntry> aclEntries, RequestedAclType type, int threadCount, IProgress <AclProcessorStats> aclStatusTracker, CancellationToken cancelToken, bool verify = false)
{
_inputPath = path;
Client = client;
NumThreads = threadCount <= 0 ? AdlsClient.DefaultNumThreads: threadCount;
Queue = new PriorityQueueWrapper <BaseJob>(NumThreads);
_threadWorker = new Thread[NumThreads];
if (aclEntries == null || aclEntries.Count == 0)
{
throw new ArgumentException("Input acl is null or empty");
}
AclEntries = aclEntries;
FileAclEntries = new List <AclEntry>(AclEntries.Count);
foreach (var entry in AclEntries)
{
if (entry.Scope == AclScope.Access)
{
FileAclEntries.Add(entry);
}
}
if (FileAclEntries.Count == 0 && AclLog.IsDebugEnabled)
{
AclLog.Debug("AclEntries for file are empty so input acl must be containing default acls");
}
Type = type;
_isVerify = verify;
_aclStatusTracker = aclStatusTracker;
_cancelToken = cancelToken;
if (AclLog.IsDebugEnabled)
{
AclLog.Debug($"AclProcessor, Name: {_inputPath}, Threads: {NumThreads}, AclChangeType: {Type}, InputAcl: {string.Join(":",AclEntries)}{(_isVerify?", RunInVerifyMode":string.Empty)}");
}
}
/// <summary>
/// Sets the WebRequest headers
/// </summary>
/// <param name="webReq">HttpWebRequest</param>
/// <param name="client">AdlsClient</param>
/// <param name="req">RequestOptions</param>
/// <param name="token">Auth token</param>
/// <param name="opMethod">Operation method (e.g. POST/GET)</param>
/// <param name="customHeaders">Custom headers</param>
private static void AssignCommonHttpHeaders(HttpWebRequest webReq, AdlsClient client, RequestOptions req, string token, string opMethod, IDictionary <string, string> customHeaders, int postRequestLength)
{
webReq.Headers["Authorization"] = token;
string latencyHeader = LatencyTracker.GetLatency();
if (!string.IsNullOrEmpty(latencyHeader))
{
webReq.Headers["x-ms-adl-client-latency"] = latencyHeader;
}
if (client.ContentEncoding != null && postRequestLength > MinDataSizeForCompression)
{
webReq.Headers["Content-Encoding"] = client.ContentEncoding;
}
if (client.DipIp != null && !req.IgnoreDip)
{
#if NET452
webReq.Host = client.AccountFQDN;
#else
webReq.Headers["Host"] = client.AccountFQDN;
#endif
}
if (!req.KeepAlive)
{
/*
* Connection cant be set directly as a header in net452.
* KeepAlive needs to be set as a property in HttpWebRequest when we want to close connection.
*/
#if NET452
webReq.KeepAlive = false;
#else
webReq.Headers["Connection"] = "Close";
#endif
}
if (customHeaders != null)
{
string contentType;
if (customHeaders.TryGetValue("Content-Type", out contentType))
{
webReq.ContentType = contentType;
customHeaders.Remove("Content-Type");
}
foreach (var key in customHeaders.Keys)
{
webReq.Headers[key] = customHeaders[key];
}
}
#if NET452
webReq.UserAgent = client.GetUserAgent();
webReq.ServicePoint.UseNagleAlgorithm = false;
webReq.ServicePoint.Expect100Continue = false;
#else
webReq.Headers["User-Agent"] = client.GetUserAgent();
#endif
webReq.Headers["x-ms-client-request-id"] = req.RequestId;
webReq.Method = opMethod;
}
public void TestCancellation()
{
int port = 8084;
MockWebServer server = new MockWebServer(port);
server.StartServer();
server.EnqueMockResponse(new MockResponse(200, "OK"));
AdlsClient adlsClient = AdlsClient.CreateClientWithoutAccntValidation(MockWebServer.Host + ":" + port, TestToken);
CancellationTokenSource source = new CancellationTokenSource();
RequestState state = new RequestState()
{
AdlsClient = adlsClient,
CancelToken = source.Token
};
Thread worker = new Thread(run);
worker.Start(state);
Thread.Sleep(10000);
Stopwatch watch = Stopwatch.StartNew();
source.Cancel();
worker.Join();
watch.Stop();
Assert.IsTrue(watch.ElapsedMilliseconds < 1000);
Assert.IsNotNull(state.AdlsClient);
Assert.IsInstanceOfType(state.Ex, typeof(OperationCanceledException));
server.StopAbruptly();
}
public void TestTimeout()
{
int port = 8085;
MockWebServer server = new MockWebServer(port);
server.StartServer();
server.EnqueMockResponse(new MockResponse(200, "OK"));
AdlsClient adlsClient = AdlsClient.CreateClientWithoutAccntValidation(MockWebServer.Host + ":" + port, TestToken);
CancellationTokenSource source = new CancellationTokenSource();
RequestState state = new RequestState()
{
AdlsClient = adlsClient,
CancelToken = source.Token,
Timeout = 5,
IsRetry = false
};
Thread worker = new Thread(runClientTimeout);
worker.Start(state);
Stopwatch watch = Stopwatch.StartNew();
worker.Join();
watch.Stop();
Assert.IsTrue(watch.ElapsedMilliseconds < 7000);
Assert.IsNotNull(state.AdlsClient);
Assert.IsInstanceOfType(state.Ex, typeof(Exception));
Assert.IsTrue(state.Ex.Message.Contains("Operation timed out"));
server.StopAbruptly();
}
// Create a sample hierarchical directory tree using async operations
private static async Task CreateDirRecursiveAsync(AdlsClient client, string path, int recursLevel, int noDirEntries, int noFileEntries)
{
await client.CreateDirectoryAsync(path);
string[] str = path.Split('/');
char nextLevel = str[str.Length - 1][0];
nextLevel++;
for (int i = 0; i < noFileEntries; i++)
{
using (var ostream = new StreamWriter(await client.CreateFileAsync(path + "/" + nextLevel + i + "File.txt", IfExists.Overwrite, "")))
{
await ostream.WriteLineAsync("This is first line.");
}
}
if (recursLevel == 0)
{
return;
}
string newPath = path + "/";
for (int i = 0; i < noDirEntries; i++)
{
await CreateDirRecursiveAsync(client, newPath + nextLevel + i, recursLevel - 1, noDirEntries, noFileEntries);
}
}
public void HandleEvent(OrderPlacedEvent eventMessage)
{
var dataLakeCredentials = GetCredentials(_tenant, _dataLakeTokenAudience, _clientId, _secretKey);
var client = AdlsClient.CreateClient(_adlsg1AccountName, dataLakeCredentials);
try
{
var fileName = "/Test/" + eventMessage.Order.OrderGuid.ToString() + ".txt";
using (var stream = client.CreateFile(fileName, IfExists.Overwrite))
{
Order order = eventMessage.Order;
Customer customer = order.Customer;
var simplifiedOrder = new SimplifiedOrderData(order.OrderGuid.ToString(), "NopOrder", customer.Id.ToString(), customer.SystemName);
var json = JsonConvert.SerializeObject(simplifiedOrder, new JsonSerializerSettings()
{
Formatting = Formatting.Indented,
});
var fileToWrite = Encoding.UTF8.GetBytes(json);
stream.Write(fileToWrite, 0, fileToWrite.Length);
}
}
catch (AdlsException exception)
{
PrintAdlsException(exception);
}
}
public async Task WriteAsync(string id, Stream sourceStream, bool append, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(id);
AdlsClient client = await GetAdlsClient();
if (append && (await ExistsAsync(new[] { id }, cancellationToken)).First())
{
AdlsOutputStream adlsStream = await client.GetAppendStreamAsync(id, cancellationToken);
using (var writeStream = new AdlsWriteableStream(adlsStream))
{
await sourceStream.CopyToAsync(writeStream);
}
}
else
{
AdlsOutputStream adlsStream = await client.CreateFileAsync(id, IfExists.Overwrite,
createParent : true,
cancelToken : cancellationToken);
using (var writeStream = new AdlsWriteableStream(adlsStream))
{
await sourceStream.CopyToAsync(writeStream);
}
}
}
private static Stream GetCompressedStream(Stream inputStream, AdlsClient client, int postRequestLength)
{
if (client.WrapperStream != null && postRequestLength > MinDataSizeForCompression)
{
return(client.WrapperStream(inputStream));
}
return(inputStream);
}
public async Task <IReadOnlyCollection <Blob> > GetBlobsAsync(IEnumerable <string> fullPaths, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobFullPaths(fullPaths);
AdlsClient client = await GetAdlsClientAsync().ConfigureAwait(false);
return(await Task.WhenAll(fullPaths.Select(fullPath => GetBlobWithMetaAsync(fullPath, client, cancellationToken))).ConfigureAwait(false));
}
public async Task DeleteAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(ids);
AdlsClient client = await GetAdlsClient();
await Task.WhenAll(ids.Select(id => client.DeleteAsync(id, cancellationToken)));
}
public static async Task <AdlsClient> CreateAdlsClientAsync(string tenant, string clientsecret, string applicationid, string fqdn)
{
Uri ADL_TOKEN_AUDIENCE = new Uri(@"https://datalake.azure.net/");
var adlCreds = await GetCreds_SPI_SecretKey(tenant, ADL_TOKEN_AUDIENCE, applicationid, clientsecret);
return(AdlsClient.CreateClient(fqdn, adlCreds));
}
internal State(string ph, long fLength, bool writeNewLines, bool local = true, AdlsClient ct = null)
{
Path = ph;
FileLength = fLength;
WriteInNewLines = writeNewLines;
IsLocal = local;
Client = ct;
}
public async Task <IEnumerable <BlobMeta> > GetMetaAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(ids);
AdlsClient client = await GetAdlsClient();
return(await Task.WhenAll(ids.Select(id => GetMetaAsync(id, client, cancellationToken))));
}
