private static async Task PublishFolder(string sourcePath, CloudQueue queue)
{
//Console.WriteLine("Starting queue insert {0} UTC +0:00", DateTime.UtcNow);
Console.WriteLine("Publishing directory {0}", sourcePath);
// Get list of files
// Check ADLS directory access rights
bool adlsAccess = client.CheckAccess(sourcePath, "r-x");
// Console.WriteLine(adlsAccess.ToString());
if (adlsAccess)
{
// Reset queue
if (_flag_WriteMode == "reset")
{
await queue.ClearAsync();
}
var stream = client.EnumerateDirectory(sourcePath);
string currentFullFileName;
int _BUFFER_LIMIT = 1000;
int msgCnt = 0;
int buffCnt = 0;
//Console.WriteLine("Starting queue insert {0} UTC +0:00", DateTime.UtcNow);
foreach (var item in stream)
{
if (item.Type == DirectoryEntryType.FILE && !item.FullName.ToLower().Contains(".txt"))
{
currentFullFileName = item.FullName;
// Broadcast each path as a message
// Create a message and add it to the queue. Set expiration time to 7 days.
CloudQueueMessage message = new CloudQueueMessage(currentFullFileName);
await queue.AddMessageAsync(message, new TimeSpan(7, 0, 0, 0), null, null, null);
msgCnt++;
buffCnt++;
if (buffCnt % _BUFFER_LIMIT == 0)
{
buffCnt = 0;
Console.WriteLine("{0} file paths published to queue '{1}' @ {2}", msgCnt, queue.Name, DateTime.UtcNow);
}
}
else if (item.Type == DirectoryEntryType.DIRECTORY)
{
await PublishFolder(item.FullName, queue);
}
}
Console.WriteLine("Directory {0} Published at {1} UTC +0:00", sourcePath, DateTime.UtcNow);
Console.WriteLine("{0} Total file paths published", msgCnt);
}
}
private void VerifyAccessRights(AdlsClient client, string sourceFilePath)
{
if (client.CheckAccess(sourceFilePath, "r--"))
{
//no action required
}
else
{
var message = $"Insufficient access rights, read access is not given for this file: {sourceFilePath}";
LogService.Instance.Fatal(message);
throw new UnauthorizedAccessException(message);
}
}
