protected JObject GetTemplateResponse(IDataLakeConnectionConfig dataLakeConnectionConfig, object parameters, Microsoft.Azure.WebJobs.ExecutionContext context) { var assemblyInfo = AssemblyHelpers.GetAssemblyVersionInfoJson(); var responseJson = new JObject(); responseJson.Add("invocationId", context.InvocationId); if (assemblyInfo.HasValues) { responseJson.Add("debugInfo", assemblyInfo); } if (dataLakeConnectionConfig.BaseUrl != null) { responseJson.Add("storageContainerUrl", dataLakeConnectionConfig.BaseUrl); } if (dataLakeConnectionConfig is DataLakeUserServicePrincipalConnectionConfig config) { responseJson.Add("clientId", config.ServicePrincipalClientId); } responseJson.Add("authType", dataLakeConnectionConfig.AuthType.ToString()); var parametersJson = JObject.FromObject(parameters); responseJson.Add("parameters", parametersJson); return(responseJson); }
public DataLakeFileSystemClient GetDataLakeClient(IDataLakeConnectionConfig dataLakeConnectionConfig) { var client = GetClient(dataLakeConnectionConfig); if (client == null || !client.Exists()) { throw new ArgumentException($"Container '{dataLakeConnectionConfig.Container}' not found in storage account '{dataLakeConnectionConfig.Account}'. Check the names are correct, and that access is granted using the authentication method used."); } return(client); }
private DataLakeFileSystemClient GetClient(IDataLakeConnectionConfig dataLakeConnectionConfig) { switch (dataLakeConnectionConfig) { case DataLakeFunctionsServicePrincipalConnectionConfig config: return(GetClient(config)); case DataLakeUserServicePrincipalConnectionConfig config: return(GetClient(config)); case DataLakeSasTokenConnectionConfig config: return(GetClient(config)); case DataLakeAccountKeyConnectionConfig config: return(GetClient(config)); } // This should never happen throw new NotImplementedException("Unknown Authentication Type"); }
public async Task <JObject> GetItemsAsync(IDataLakeConnectionConfig dataLakeConnectionConfig, DataLakeGetItemsConfig getItemsConfig) { // Check the directory exists. If multiple directories match (ie different casing), it will throw an error, as we don't know // which one we wanted the files from. var directory = getItemsConfig.IgnoreDirectoryCase ? await CheckPathAsync(getItemsConfig.Path, true) : getItemsConfig.Path; // Only check the path if it is not the root. Checking is the root exists throws, and if the container is valid the root will always be valid if (directory != "/" && !await _client.GetDirectoryClient(directory).ExistsAsync()) { throw new DirectoryNotFoundException(string.Format(ErrorMessage.PathNotFound, directory)); } var paths = _client .GetPaths(path: directory ?? string.Empty, recursive: getItemsConfig.Recursive) .Select(p => new DataLakeItem { Name = Path.GetFileName(p.Name), Directory = Path.GetDirectoryName(p.Name).Replace(Path.DirectorySeparatorChar, '/'), Url = Url.Combine(dataLakeConnectionConfig.BaseUrl, p.Name), IsDirectory = p.IsDirectory.GetValueOrDefault(false), ContentLength = p.ContentLength.GetValueOrDefault(0), LastModified = p.LastModified.ToUniversalTime() }) .ToList(); // 1: Filter the results using dynamic LINQ foreach (var filter in getItemsConfig.Filters.Where(f => f.IsValid)) { var dynamicLinqQuery = filter.GetDynamicLinqString(); var dynamicLinqQueryValue = filter.GetDynamicLinqValue(); _logger.LogInformation($"Applying filter: paths.AsQueryable().Where(\"{dynamicLinqQuery}\", \"{filter.Value}\").ToList()"); paths = paths.AsQueryable().Where(dynamicLinqQuery, dynamicLinqQueryValue).ToList(); } // 2: Sort the results if (!string.IsNullOrWhiteSpace(getItemsConfig.OrderByColumn)) { paths = paths.AsQueryable() .OrderBy(getItemsConfig.OrderByColumn + (getItemsConfig.OrderByDescending ? " descending" : string.Empty)) .ToList(); } // 3: Do a top N if required if (getItemsConfig.Limit > 0 && getItemsConfig.Limit < paths.Count) { paths = paths.Take(getItemsConfig.Limit).ToList(); } // Output the results var isEveryFilterValid = getItemsConfig.Filters.All(f => f.IsValid); if (!isEveryFilterValid) { throw new InvalidFilterCriteriaException("Some filters are not valid"); } var formatter = new IsoDateTimeConverter() { DateTimeFormat = "yyyy-MM-ddTHH:mm:ss.fffZ" }; var filesListJson = isEveryFilterValid ? $"\"fileCount\": {paths.Count}," + $"\"files\": {JsonConvert.SerializeObject(paths, Formatting.Indented, formatter)}" : string.Empty; var resultJson = $"{{ {(getItemsConfig.IgnoreDirectoryCase && directory != getItemsConfig.Path ? $"\"correctedFilePath\": \"{directory}\"," : string.Empty)} {filesListJson} }}"; return(JObject.Parse(resultJson)); }