public async Task PathClient_CanGetParentDirectoryClient_WithContainerSAS()
{
// Arrange
var parentDirName = DataLakeClientBuilder.GetNewDirectoryName();
await using DisposingFileSystem test = await DataLakeClientBuilder.GetNewFileSystem();
var fileName = DataLakeClientBuilder.GetNewFileName();
DataLakeFileClient fileClient = InstrumentClient(
GetServiceClient_DataLakeServiceSas_FileSystem(test.Container.Name)
.GetFileSystemClient(test.FileSystem.Name)
.GetRootDirectoryClient()
.GetSubDirectoryClient(parentDirName)
.GetFileClient(fileName));
await fileClient.CreateAsync();
// Act
DataLakeDirectoryClient parentDirClient = fileClient.GetParentDirectoryClient();
// make sure that client is functional
var pathItems = await parentDirClient.GetPathsAsync().ToListAsync();
// Assert
Assert.AreEqual(fileClient.Path.GetParentPath(), parentDirClient.Path);
Assert.AreEqual(fileClient.AccountName, parentDirClient.AccountName);
Assert.IsNotNull(pathItems);
}
// </Snippet_GetUserDelegationSasDirectory>
#endregion
#region
// <Snippet_ListFilePathsWithDirectorySasAsync>
private static async Task ListFilesPathsWithDirectorySasAsync(Uri sasUri)
{
// Try performing an operation using the directory SAS provided.
// Create a directory client object for listing operations.
DataLakeDirectoryClient dataLakeDirectoryClient = new DataLakeDirectoryClient(sasUri);
// List file paths in the directory.
try
{
// Call the listing operation and return pages of the specified size.
var resultSegment = dataLakeDirectoryClient.GetPathsAsync(false, false).AsPages();
// Enumerate the file paths returned with each page.
await foreach (Page <PathItem> pathPage in resultSegment)
{
foreach (PathItem pathItem in pathPage.Values)
{
Console.WriteLine("File name: {0}", pathItem.Name);
}
Console.WriteLine();
}
Console.WriteLine();
Console.WriteLine("Directory listing operation succeeded for SAS {0}", sasUri);
}
catch (RequestFailedException e)
{
// Check for a 403 (Forbidden) error. If the SAS is invalid,
// Azure Storage returns this error.
if (e.Status == 403)
{
Console.WriteLine("Directory listing operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
}
public async Task <string> Run([ServiceBusTrigger("%ServiceBusQueue%", Connection = "ServiceBusConnection")] Message message, ILogger log, MessageReceiver messageReceiver)
{
telemetryClient.GetMetric("LakeMessagesReceived").TrackValue(1);
log.LogDebug($"ProcessLakeFolder: Message {message.MessageId} is locked until {message.SystemProperties.LockedUntilUtc}");
log.LogDebug($"");
CrawlerResult cr = new CrawlerResult();
Stopwatch w = new Stopwatch();
string payload = System.Text.Encoding.UTF8.GetString(message.Body);
log.LogInformation($"C# ServiceBus queue trigger function processed message: {payload}");
var x = JObject.Parse(payload);
var name = x["Name"].ToString();
cr.Path = name;
cr.IsDirectory = Boolean.Parse(x["IsDirectory"].ToString());
if (cr.IsDirectory)
{
telemetryClient.TrackEvent($"Directory processing request recieved for directory {cr.Path} was recieved");
}
else
{
Exception e = new Exception("Laker invoked by passing a file. This is not supported!");
telemetryClient.TrackException(e);
await messageReceiver.DeadLetterAsync(message.SystemProperties.LockToken);
throw e;
}
w.Start();
DataLakeDirectoryClient directoryClient = null;
try
{
directoryClient = fileSystemClient.GetDirectoryClient(name);
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt directory client for {name} failed. Exception was {e}");
}
log.LogDebug($"Time to obtain directory client was {w.ElapsedMilliseconds} ms");
w.Reset();
Response <PathAccessControl> aclResult = null;
try
{
aclResult = await directoryClient.GetAccessControlAsync();
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeAclRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt to retrieve acl for directory {name} failed. Exception was {e}");
throw e;
}
log.LogDebug($"Time to GetAccessControlAsync was {w.ElapsedMilliseconds} ms");
foreach (var item in aclResult.Value.AccessControlList)
{
cr.ACLs.Add(item);
}
log.LogDebug($"{cr.ACLs.Count} ACLS are present");
log.LogInformation($"Processing directory {name}");
AsyncPageable <PathItem> pathItems = null;
try
{
pathItems = directoryClient.GetPathsAsync(false);
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt to process directory {name} failed. Exception was {e}");
throw e;
}
await foreach (var pathItem in pathItems)
{
if ((bool)pathItem.IsDirectory)
{
log.LogWarning($"{pathItem.Name} is a directory. Add to processing queue.");
string data = JsonConvert.SerializeObject(pathItem);
Message newMessage = new Message(Encoding.UTF8.GetBytes(data));
await _queueClient.SendAsync(newMessage);
}
else
{
log.LogDebug($"File {pathItem} will be added to output");
var fileClient = fileSystemClient.GetFileClient(pathItem.Name);
CrawlerFile cf = new CrawlerFile();
cf.Name = pathItem.Name;
w.Reset();
try
{
aclResult = await fileClient.GetAccessControlAsync();
}
catch (Exception e)
{
telemetryClient.TrackException(e);
telemetryClient.GetMetric("FailedLakeAclRequests").TrackValue(1);
telemetryClient.TrackTrace($"Salamander - Attempt to retrieve acl for file {pathItem} failed. Exception was {e}");
throw e;
}
log.LogDebug($"Time to GetAccessControlAsync was {w.ElapsedMilliseconds} ms");
foreach (var item in aclResult.Value.AccessControlList)
{
cf.ACLs.Add(item);
}
cr.Files.Add(cf);
}
}
log.LogDebug($"ProcessLakeFolder: - completion {message.MessageId} is locked until {message.SystemProperties.LockedUntilUtc} and time now is {DateTime.UtcNow}");
// await messageReceiver.CompleteAsync(message.SystemProperties.LockToken);
telemetryClient.GetMetric("LakeMessagesProcessed").TrackValue(1);
return(JsonConvert.SerializeObject(cr));
}