private static async Task PublishFolder(string sourcePath, CloudQueue queue) { //Console.WriteLine("Starting queue insert {0} UTC +0:00", DateTime.UtcNow); Console.WriteLine("Publishing directory {0}", sourcePath); // Get list of files // Check ADLS directory access rights bool adlsAccess = client.CheckAccess(sourcePath, "r-x"); // Console.WriteLine(adlsAccess.ToString()); if (adlsAccess) { // Reset queue if (_flag_WriteMode == "reset") { await queue.ClearAsync(); } var stream = client.EnumerateDirectory(sourcePath); string currentFullFileName; int _BUFFER_LIMIT = 1000; int msgCnt = 0; int buffCnt = 0; //Console.WriteLine("Starting queue insert {0} UTC +0:00", DateTime.UtcNow); foreach (var item in stream) { if (item.Type == DirectoryEntryType.FILE && !item.FullName.ToLower().Contains(".txt")) { currentFullFileName = item.FullName; // Broadcast each path as a message // Create a message and add it to the queue. Set expiration time to 7 days. CloudQueueMessage message = new CloudQueueMessage(currentFullFileName); await queue.AddMessageAsync(message, new TimeSpan(7, 0, 0, 0), null, null, null); msgCnt++; buffCnt++; if (buffCnt % _BUFFER_LIMIT == 0) { buffCnt = 0; Console.WriteLine("{0} file paths published to queue '{1}' @ {2}", msgCnt, queue.Name, DateTime.UtcNow); } } else if (item.Type == DirectoryEntryType.DIRECTORY) { await PublishFolder(item.FullName, queue); } } Console.WriteLine("Directory {0} Published at {1} UTC +0:00", sourcePath, DateTime.UtcNow); Console.WriteLine("{0} Total file paths published", msgCnt); } }
private void VerifyAccessRights(AdlsClient client, string sourceFilePath) { if (client.CheckAccess(sourceFilePath, "r--")) { //no action required } else { var message = $"Insufficient access rights, read access is not given for this file: {sourceFilePath}"; LogService.Instance.Fatal(message); throw new UnauthorizedAccessException(message); } }