protected JObject GetTemplateResponse(IDataLakeConnectionConfig dataLakeConnectionConfig, object parameters, Microsoft.Azure.WebJobs.ExecutionContext context)
{
var assemblyInfo = AssemblyHelpers.GetAssemblyVersionInfoJson();
var responseJson = new JObject();
responseJson.Add("invocationId", context.InvocationId);
if (assemblyInfo.HasValues)
{
responseJson.Add("debugInfo", assemblyInfo);
}
if (dataLakeConnectionConfig.BaseUrl != null)
{
responseJson.Add("storageContainerUrl", dataLakeConnectionConfig.BaseUrl);
}
if (dataLakeConnectionConfig is DataLakeUserServicePrincipalConnectionConfig config)
{
responseJson.Add("clientId", config.ServicePrincipalClientId);
}
responseJson.Add("authType", dataLakeConnectionConfig.AuthType.ToString());
var parametersJson = JObject.FromObject(parameters);
responseJson.Add("parameters", parametersJson);
return(responseJson);
}
public DataLakeFileSystemClient GetDataLakeClient(IDataLakeConnectionConfig dataLakeConnectionConfig)
{
var client = GetClient(dataLakeConnectionConfig);
if (client == null || !client.Exists())
{
throw new ArgumentException($"Container '{dataLakeConnectionConfig.Container}' not found in storage account '{dataLakeConnectionConfig.Account}'. Check the names are correct, and that access is granted using the authentication method used.");
}
return(client);
}
private DataLakeFileSystemClient GetClient(IDataLakeConnectionConfig dataLakeConnectionConfig)
{
switch (dataLakeConnectionConfig)
{
case DataLakeFunctionsServicePrincipalConnectionConfig config:
return(GetClient(config));
case DataLakeUserServicePrincipalConnectionConfig config:
return(GetClient(config));
case DataLakeSasTokenConnectionConfig config:
return(GetClient(config));
case DataLakeAccountKeyConnectionConfig config:
return(GetClient(config));
}
// This should never happen
throw new NotImplementedException("Unknown Authentication Type");
}
public async Task <JObject> GetItemsAsync(IDataLakeConnectionConfig dataLakeConnectionConfig, DataLakeGetItemsConfig getItemsConfig)
{
// Check the directory exists. If multiple directories match (ie different casing), it will throw an error, as we don't know
// which one we wanted the files from.
var directory = getItemsConfig.IgnoreDirectoryCase ?
await CheckPathAsync(getItemsConfig.Path, true) :
getItemsConfig.Path;
// Only check the path if it is not the root. Checking is the root exists throws, and if the container is valid the root will always be valid
if (directory != "/" && !await _client.GetDirectoryClient(directory).ExistsAsync())
{
throw new DirectoryNotFoundException(string.Format(ErrorMessage.PathNotFound, directory));
}
var paths = _client
.GetPaths(path: directory ?? string.Empty, recursive: getItemsConfig.Recursive)
.Select(p => new DataLakeItem
{
Name = Path.GetFileName(p.Name),
Directory = Path.GetDirectoryName(p.Name).Replace(Path.DirectorySeparatorChar, '/'),
Url = Url.Combine(dataLakeConnectionConfig.BaseUrl, p.Name),
IsDirectory = p.IsDirectory.GetValueOrDefault(false),
ContentLength = p.ContentLength.GetValueOrDefault(0),
LastModified = p.LastModified.ToUniversalTime()
})
.ToList();
// 1: Filter the results using dynamic LINQ
foreach (var filter in getItemsConfig.Filters.Where(f => f.IsValid))
{
var dynamicLinqQuery = filter.GetDynamicLinqString();
var dynamicLinqQueryValue = filter.GetDynamicLinqValue();
_logger.LogInformation($"Applying filter: paths.AsQueryable().Where(\"{dynamicLinqQuery}\", \"{filter.Value}\").ToList()");
paths = paths.AsQueryable().Where(dynamicLinqQuery, dynamicLinqQueryValue).ToList();
}
// 2: Sort the results
if (!string.IsNullOrWhiteSpace(getItemsConfig.OrderByColumn))
{
paths = paths.AsQueryable()
.OrderBy(getItemsConfig.OrderByColumn + (getItemsConfig.OrderByDescending ? " descending" : string.Empty))
.ToList();
}
// 3: Do a top N if required
if (getItemsConfig.Limit > 0 && getItemsConfig.Limit < paths.Count)
{
paths = paths.Take(getItemsConfig.Limit).ToList();
}
// Output the results
var isEveryFilterValid = getItemsConfig.Filters.All(f => f.IsValid);
if (!isEveryFilterValid)
{
throw new InvalidFilterCriteriaException("Some filters are not valid");
}
var formatter = new IsoDateTimeConverter()
{
DateTimeFormat = "yyyy-MM-ddTHH:mm:ss.fffZ"
};
var filesListJson = isEveryFilterValid ?
$"\"fileCount\": {paths.Count}," +
$"\"files\": {JsonConvert.SerializeObject(paths, Formatting.Indented, formatter)}" :
string.Empty;
var resultJson = $"{{ {(getItemsConfig.IgnoreDirectoryCase && directory != getItemsConfig.Path ? $"\"correctedFilePath\": \"{directory}\"," : string.Empty)} {filesListJson} }}";
return(JObject.Parse(resultJson));
}
