public static async Task <IActionResult> DataLakeGetItems(
[HttpTrigger(AuthorizationLevel.Function, "get" /*, "post"*/, Route = null)] HttpRequest req,
ILogger log)
{
var userAgentKey = req.Headers.Keys.FirstOrDefault(k => k.ToLower() == "user-agent" || k.ToLower() == "useragent");
log.LogInformation($"C# HTTP trigger function processed a request [User Agent: { (userAgentKey == null ? "Unknown" : req.Headers[userAgentKey].ToString()) }].");
try
{
var settings = DataLakeGetItemsConfig.ParseFromRequestBody(req, log);
if (string.IsNullOrWhiteSpace(settings.AccountUri))
{
throw new ArgumentException($"Account Uri '{settings.AccountUri}' not found. Check the URI is correct.");
}
var client = GetDataLakeClient(settings, log);
if (client == null || !client.Exists())
{
throw new ArgumentException($"Container '{settings.Container}' not found in storage account '{settings.AccountUri}'. Check the names are correct, and that access is granted to the functions application service principal.");
}
return(await GetItemsAsync(client, settings, log));
}
catch (ArgumentException ex)
{
log.LogError(ex.Message);
return(new BadRequestObjectResult(ex.Message));
}
catch (Exception ex)
{
log.LogError(ex.ToString());
return(new BadRequestObjectResult("An error occurred, see the Azure Function logs for more details"));
}
}
private static async Task <IActionResult> GetItemsAsync(DataLakeFileSystemClient client, DataLakeGetItemsConfig settings, ILogger log)
{
var directory = settings.IgnoreDirectoryCase ?
await CheckPathAsync(client, settings.Directory, true, log) :
settings.Directory;
var paramsJsonFragment = GetParamsJsonFragment(settings);
if (!client.GetDirectoryClient(directory).Exists())
{
return(new BadRequestObjectResult(JObject.Parse($"{{ {paramsJsonFragment}, \"error\": \"Directory '{directory} could not be found'\" }}")));
}
var paths = client
.GetPaths(path: directory ?? string.Empty, recursive: settings.Recursive)
.Select(p => new DataLakeFile
{
Name = Path.GetFileName(p.Name),
Directory = p.IsDirectory.GetValueOrDefault(false) ?
p.Name :
Path.GetDirectoryName(p.Name).Replace(Path.DirectorySeparatorChar, '/'),
FullPath = p.Name,
Url = Url.Combine(settings.BaseUrl, p.Name),
IsDirectory = p.IsDirectory.GetValueOrDefault(false),
ContentLength = p.ContentLength.GetValueOrDefault(0),
LastModified = p.LastModified.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ")
})
.ToList();
// 1: Filter the results using dynamic LINQ
foreach (var filter in settings.Filters.Where(f => f.IsValid))
{
var dynamicLinqQuery = filter.GetDynamicLinqString();
string dynamicLinqQueryValue = filter.GetDynamicLinqValue();
log.LogInformation($"Applying filter: paths.AsQueryable().Where(\"{dynamicLinqQuery}\", \"{filter.Value}\").ToList()");
paths = paths.AsQueryable().Where(dynamicLinqQuery, dynamicLinqQueryValue).ToList();
}
// 2: Sort the results
if (!string.IsNullOrWhiteSpace(settings.OrderByColumn))
{
paths = paths.AsQueryable()
.OrderBy(settings.OrderByColumn + (settings.OrderByDescending ? " descending" : string.Empty))
.ToList();
}
// 3: Do a top N if required
if (settings.Limit > 0 && settings.Limit < paths.Count)
{
paths = paths.Take(settings.Limit).ToList();
}
// Output the results
var versionAttribute = Attribute.GetCustomAttribute(Assembly.GetExecutingAssembly(), typeof(AssemblyInformationalVersionAttribute)) as AssemblyInformationalVersionAttribute;
var IsEveryFilterValid = settings.Filters.All(f => f.IsValid);
var filesListJson = IsEveryFilterValid ?
$"\"fileCount\": {paths.Count}," +
$"\"files\": {JsonConvert.SerializeObject(paths, Formatting.Indented)}" :
string.Empty;
var resultJson = $"{{ {paramsJsonFragment}, {(settings.IgnoreDirectoryCase && directory != settings.Directory ? $"\"correctedFilePath\": \"{directory}\"," : string.Empty)} {filesListJson} }}";
return(IsEveryFilterValid ?
(IActionResult) new OkObjectResult(JObject.Parse(resultJson)) :
(IActionResult) new BadRequestObjectResult(JObject.Parse(resultJson)));
}
