// </Snippet_UpdateACLsRecursively>
#endregion
#region Remove ACL entry
// ---------------------------------------------------------
// Remove directory-level ACL entry
//----------------------------------------------------------
// <Snippet_RemoveACLEntry>
public async Task RemoveDirectoryACLEntry
(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("");
PathAccessControl directoryAccessControl =
await directoryClient.GetAccessControlAsync();
List <PathAccessControlItem> accessControlListUpdate
= (List <PathAccessControlItem>)directoryAccessControl.AccessControlList;
PathAccessControlItem entryToRemove = null;
foreach (var item in accessControlListUpdate)
{
if (item.EntityId == "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")
{
entryToRemove = item;
break;
}
}
if (entryToRemove != null)
{
accessControlListUpdate.Remove(entryToRemove);
directoryClient.SetAccessControlList(accessControlListUpdate);
}
}
// </Snippet_FileACL>
#endregion
#region Update ACL
// ---------------------------------------------------------
// Update directory-level ACLs
//----------------------------------------------------------
// <Snippet_UpdateACL>
public async Task UpdateDirectoryACLs(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("");
PathAccessControl directoryAccessControl =
await directoryClient.GetAccessControlAsync();
List <PathAccessControlItem> accessControlListUpdate
= (List <PathAccessControlItem>)directoryAccessControl.AccessControlList;
int index = -1;
foreach (var item in accessControlListUpdate)
{
if (item.AccessControlType == AccessControlType.Other)
{
index = accessControlListUpdate.IndexOf(item);
break;
}
}
if (index > -1)
{
accessControlListUpdate[index] = new PathAccessControlItem(AccessControlType.Other,
RolePermissions.Read |
RolePermissions.Execute);
directoryClient.SetAccessControlList(accessControlListUpdate);
}
}
// If recursive is false, remove the ACL from a directory. None of the sub-directory or sub-path ACLs are updated
// If recursive is true, remove ACLs from the directory and all sub-directories and sub-paths
// When removing ACLs recursively, the ACLs on all sub-directories and sub-paths are replaced with this directory's ACL
static async Task RemoveACLsForDirectory(DataLakeDirectoryClient directoryClient, AppSettings settings, bool recursive = false)
{
PathAccessControl directoryAccessControl =
await directoryClient.GetAccessControlAsync();
List <PathAccessControlItem> accessControlList = RemoveACLs(directoryAccessControl.AccessControlList, settings);
if (recursive)
{
await directoryClient.SetAccessControlRecursiveAsync(accessControlList);
}
else
{
await directoryClient.SetAccessControlListAsync(accessControlList);
}
}
// If recursive is false, apply ACLs to a directory. None of the sub-directory or sub-path ACLs are updated
// If recursive is true, apply ACLs to the directory and all sub-directories and sub-paths
// When applying ACL recursively, the ACLs on all sub-directories and sub-paths are replaced with this directory's ACL
static async Task ApplyACLsForDirectory(DataLakeDirectoryClient directoryClient, RolePermissions newACLs, AppSettings settings, bool recursive = false)
{
PathAccessControl directoryAccessControl =
await directoryClient.GetAccessControlAsync();
List <PathAccessControlItem> accessControlList = UpdateACLs(directoryAccessControl.AccessControlList, newACLs, settings);
if (recursive)
{
await directoryClient.SetAccessControlRecursiveAsync(accessControlList);
}
else
{
await directoryClient.SetAccessControlListAsync(accessControlList);
}
}
// </Snippet_GetFileSystem>
#endregion
#region Get and set directory ACLs
// ---------------------------------------------------------
// Get and set directory-level ACLs
//----------------------------------------------------------
// <Snippet_ACLDirectory>
public async Task ManageDirectoryACLs(DataLakeFileSystemClient fileSystemClient)
{
DataLakeDirectoryClient directoryClient =
fileSystemClient.GetDirectoryClient("");
PathAccessControl directoryAccessControl =
await directoryClient.GetAccessControlAsync();
foreach (var item in directoryAccessControl.AccessControlList)
{
Console.WriteLine(item.ToString());
}
IList <PathAccessControlItem> accessControlList
= PathAccessControlExtensions.ParseAccessControlList
("user::rwx,group::r-x,other::rw-");
directoryClient.SetAccessControlList(accessControlList);
}
public async Task <string> Run([ServiceBusTrigger("%ServiceBusQueue%", Connection = "ServiceBusConnection")] Message message, ILogger log, MessageReceiver messageReceiver)
{
telemetryClient.GetMetric("LakeMessagesReceived").TrackValue(1);
log.LogDebug($"ProcessLakeFolder: Message {message.MessageId} is locked until {message.SystemProperties.LockedUntilUtc}");
log.LogDebug($"");
CrawlerResult cr = new CrawlerResult();
Stopwatch w = new Stopwatch();
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
log.LogInformation($"C# ServiceBus queue trigger function processed message: {payload}");
var x = JObject.Parse(payload);
var name = x["Name"].ToString();
cr.Path = name;
cr.IsDirectory = Boolean.Parse(x["IsDirectory"].ToString());
if (cr.IsDirectory)
{
telemetryClient.TrackEvent($"Directory processing request recieved for directory {cr.Path} was recieved");
}
else
{
Exception e = new Exception("Laker invoked by passing a file. This is not supported!");
telemetryClient.TrackException(e);
await messageReceiver.DeadLetterAsync(message.SystemProperties.LockToken);
throw e;
}
w.Start();
DataLakeDirectoryClient directoryClient = null;
try
{
directoryClient = fileSystemClient.GetDirectoryClient(name);
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt directory client for {name} failed. Exception was {e}");
}
log.LogDebug($"Time to obtain directory client was {w.ElapsedMilliseconds} ms");
w.Reset();
Response <PathAccessControl> aclResult = null;
try
{
aclResult = await directoryClient.GetAccessControlAsync();
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeAclRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt to retrieve acl for directory {name} failed. Exception was {e}");
throw e;
}
log.LogDebug($"Time to GetAccessControlAsync was {w.ElapsedMilliseconds} ms");
foreach (var item in aclResult.Value.AccessControlList)
{
cr.ACLs.Add(item);
}
log.LogDebug($"{cr.ACLs.Count} ACLS are present");
log.LogInformation($"Processing directory {name}");
AsyncPageable <PathItem> pathItems = null;
try
{
pathItems = directoryClient.GetPathsAsync(false);
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt to process directory {name} failed. Exception was {e}");
throw e;
}
await foreach (var pathItem in pathItems)
{
if ((bool)pathItem.IsDirectory)
{
log.LogWarning($"{pathItem.Name} is a directory. Add to processing queue.");
string data = JsonConvert.SerializeObject(pathItem);
Message newMessage = new Message(Encoding.UTF8.GetBytes(data));
await _queueClient.SendAsync(newMessage);
}
else
{
log.LogDebug($"File {pathItem} will be added to output");
var fileClient = fileSystemClient.GetFileClient(pathItem.Name);
CrawlerFile cf = new CrawlerFile();
cf.Name = pathItem.Name;
w.Reset();
try
{
aclResult = await fileClient.GetAccessControlAsync();
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeAclRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt to retrieve acl for file {pathItem} failed. Exception was {e}");
throw e;
}
log.LogDebug($"Time to GetAccessControlAsync was {w.ElapsedMilliseconds} ms");
foreach (var item in aclResult.Value.AccessControlList)
{
cf.ACLs.Add(item);
}
cr.Files.Add(cf);
}
}
log.LogDebug($"ProcessLakeFolder: - completion {message.MessageId} is locked until {message.SystemProperties.LockedUntilUtc} and time now is {DateTime.UtcNow}");
// await messageReceiver.CompleteAsync(message.SystemProperties.LockToken);
telemetryClient.GetMetric("LakeMessagesProcessed").TrackValue(1);
return(JsonConvert.SerializeObject(cr));
}