/// <inheritdoc /> public async Task <List <string> > FetchAllFilesAsync(string folderCorpusPath) { this.CreateFetchAllFilesUrl(this.FormatCorpusPath(folderCorpusPath), out string url, out string directory); var request = await this.BuildRequest($"{url}?directory={directory}&recursive=True&resource=filesystem", HttpMethod.Get); CdmHttpResponse cdmResponse = await base.ExecuteRequest(request); if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK)) { string json = await cdmResponse.Content.ReadAsStringAsync(); JObject jObject1 = JObject.Parse(json); JArray jArray = JArray.FromObject(jObject1.GetValue("paths")); List <string> result = new List <string>(); foreach (JObject jObject in jArray.Children <JObject>()) { jObject.TryGetValue("isDirectory", StringComparison.OrdinalIgnoreCase, out JToken isDirectory); if (isDirectory == null || !isDirectory.ToObject <bool>()) { jObject.TryGetValue("name", StringComparison.OrdinalIgnoreCase, out JToken name); string nameWithoutSubPath = this.subPath.Length > 0 && name.ToString().StartsWith(this.subPath) ? name.ToString().Substring(this.subPath.Length + 1) : name.ToString(); result.Add(this.FormatCorpusPath(nameWithoutSubPath)); } } return(result); } return(null); }
/// <inheritdoc /> public override async Task WriteAsync(string corpusPath, string data) { if (EnsurePath($"{this.Root}{corpusPath}") == false) { throw new Exception($"Could not create folder for document '{corpusPath}'"); } string url = this.CreateFormattedAdapterPath(corpusPath); CdmHttpResponse response = await this.CreateFileAtPath(corpusPath, url); try { CdmHttpRequest request = await this.BuildRequest($"{url}?action=append&position=0", new HttpMethod("PATCH"), data, "application/json"); response = await this.ExecuteRequest(request); if (response.StatusCode.Equals(HttpStatusCode.Accepted)) // The uploaded data was accepted. { var stringContent = new StringContent(request.Content, Encoding.UTF8, request.ContentType); // Building a request and setting a URL with a position argument to be the length of the byte array of the string content (or length of UTF-8 string content). request = await this.BuildRequest($"{url}?action=flush&position={(await stringContent.ReadAsByteArrayAsync()).Length}", new HttpMethod("PATCH")); response = await this.ExecuteRequest(request); if (!response.StatusCode.Equals(HttpStatusCode.OK)) // Data was not flushed correctly. Delete empty file. { await this.DeleteContentAtPath(corpusPath, url, null); throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'"); } } else { await this.DeleteContentAtPath(corpusPath, url, null); throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'"); } } catch (StorageAdapterException exc) { throw exc; } catch (Exception e) { await this.DeleteContentAtPath(corpusPath, url, e); throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'", e); } }
/// <summary> /// A callback function for http request retries /// </summary> /// <param name="response">Http response</param> /// <param name="hasFailed">Indicates whether the request has failed</param> /// <param name="retryNumber">The number of retries happened</param> /// <returns>The wait time before starting the next retry</returns> private TimeSpan?GetRetryWaitTime(CdmHttpResponse response, bool hasFailed, int retryNumber) { if (response != null && (response.IsSuccessful && !hasFailed)) { return(null); } else { // Default wait time is calculated using exponential backoff with with random jitter value to avoid 'waves'. Random random = new Random(); double waitTime = random.Next(1 << retryNumber) * backoffForThrottling; return(TimeSpan.FromMilliseconds(waitTime)); } }
private TimeSpan?DefaultGetWaitTime(CdmHttpResponse response, bool hasFailed, int retryNumber) { if (response != null && response.IsSuccessful && !hasFailed) { return(null); } else { Random random = new Random(); // Default wait time will be using exponential backoff with default shortest wait time. double waitTime = random.Next(1 << retryNumber) * 500; return(TimeSpan.FromMilliseconds(waitTime)); } }
/// <summary> /// Callback function for a CDM Http client, it does exponential backoff. /// </summary> /// <param name="response">The response received by system's Http client.</param> /// <param name="hasFailed">Denotes whether the request has failed (usually an exception or 500 error).</param> /// <param name="retryNumber">The current retry number (starts from 1) up to the number of retries specified by CDM request.</param> /// <returns>The <see cref="TimeSpan"/>, specifying the waiting time, or null if no wait time is necessary.</returns> private TimeSpan?DefaultGetWaitTime(CdmHttpResponse response, bool hasFailed, int retryNumber) { if (response != null && ((response.IsSuccessful && !hasFailed) || this.AvoidRetryCodes.Contains(response.StatusCode))) { return(null); } else { Random random = new Random(); // Default wait time is calculated using exponential backoff with with random jitter value to avoid 'waves'. double waitTime = random.Next(1 << retryNumber) * DefaultShortestTimeWait; return(TimeSpan.FromMilliseconds(waitTime)); } }
/// <summary> /// Get an authorization token and send the query to Kusto /// </summary> /// <param name="query">The Kusto query command to be posted to the cluster</param> public async Task PostKustoQuery(string query) { string authToken = await this.config.GetAuthenticationToken(); string queryEndpoint = $"https://{this.config.KustoClusterName}.kusto.windows.net/v1/rest/mgmt"; string kustoHost = $"{this.config.KustoClusterName}.kusto.windows.net"; string queryBody = $"{{\"db\":\"{this.config.KustoDatabaseName}\",\"csl\":\"{query}\"}}"; Dictionary <string, string> headers = new Dictionary <string, string>() { { "Accept", "application/json" }, { "Authorization", authToken }, { "Host", kustoHost } }; var httpRequest = new CdmHttpRequest(queryEndpoint) { Method = HttpMethod.Post, Headers = headers, Content = queryBody, ContentType = "application/json", NumberOfRetries = MaxNumRetries, Timeout = TimeSpan.FromMilliseconds(TimeoutMilliseconds), MaximumTimeout = TimeSpan.FromMilliseconds(MaxTimeoutMilliseconds) }; CdmHttpResponse response = await httpClient.SendAsync(httpRequest, GetRetryWaitTime, this.ctx); if (response == null) { throw new HttpRequestException ($"Kusto query post failed. The result of a request is undefined."); } if (!response.IsSuccessful) { throw new HttpRequestException ($"Kusto query post failed. HTTP {response.StatusCode} - {response.Reason}."); } }
/// <inheritdoc /> public async Task <DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath) { var adapterPath = this.CreateAdapterPath(corpusPath); var request = await this.BuildRequest(adapterPath, HttpMethod.Head); try { CdmHttpResponse cdmResponse = await base.ExecuteRequest(request); if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK)) { return(cdmResponse.Content.Headers.LastModified); } } catch (HttpRequestException ex) { // We don't have standard logger here, so use one from system diagnostics Debug.WriteLine($"ADLS file not found, skipping last modified time calculation for it. Exception: {ex}"); } return(null); }