/// <inheritdoc />
public async Task <List <string> > FetchAllFilesAsync(string folderCorpusPath)
{
this.CreateFetchAllFilesUrl(this.FormatCorpusPath(folderCorpusPath), out string url, out string directory);
var request = await this.BuildRequest($"{url}?directory={directory}&recursive=True&resource=filesystem", HttpMethod.Get);
CdmHttpResponse cdmResponse = await base.ExecuteRequest(request);
if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
{
string json = await cdmResponse.Content.ReadAsStringAsync();
JObject jObject1 = JObject.Parse(json);
JArray jArray = JArray.FromObject(jObject1.GetValue("paths"));
List <string> result = new List <string>();
foreach (JObject jObject in jArray.Children <JObject>())
{
jObject.TryGetValue("isDirectory", StringComparison.OrdinalIgnoreCase, out JToken isDirectory);
if (isDirectory == null || !isDirectory.ToObject <bool>())
{
jObject.TryGetValue("name", StringComparison.OrdinalIgnoreCase, out JToken name);
string nameWithoutSubPath = this.subPath.Length > 0 && name.ToString().StartsWith(this.subPath) ?
name.ToString().Substring(this.subPath.Length + 1) : name.ToString();
result.Add(this.FormatCorpusPath(nameWithoutSubPath));
}
}
return(result);
}
return(null);
}
/// <inheritdoc />
public override async Task WriteAsync(string corpusPath, string data)
{
if (EnsurePath($"{this.Root}{corpusPath}") == false)
{
throw new Exception($"Could not create folder for document '{corpusPath}'");
}
string url = this.CreateFormattedAdapterPath(corpusPath);
CdmHttpResponse response = await this.CreateFileAtPath(corpusPath, url);
try
{
CdmHttpRequest request = await this.BuildRequest($"{url}?action=append&position=0", new HttpMethod("PATCH"), data, "application/json");
response = await this.ExecuteRequest(request);
if (response.StatusCode.Equals(HttpStatusCode.Accepted)) // The uploaded data was accepted.
{
var stringContent = new StringContent(request.Content, Encoding.UTF8, request.ContentType);
// Building a request and setting a URL with a position argument to be the length of the byte array of the string content (or length of UTF-8 string content).
request = await this.BuildRequest($"{url}?action=flush&position={(await stringContent.ReadAsByteArrayAsync()).Length}", new HttpMethod("PATCH"));
response = await this.ExecuteRequest(request);
if (!response.StatusCode.Equals(HttpStatusCode.OK)) // Data was not flushed correctly. Delete empty file.
{
await this.DeleteContentAtPath(corpusPath, url, null);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'");
}
}
else
{
await this.DeleteContentAtPath(corpusPath, url, null);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'");
}
}
catch (StorageAdapterException exc)
{
throw exc;
}
catch (Exception e)
{
await this.DeleteContentAtPath(corpusPath, url, e);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'", e);
}
}
/// <summary>
/// A callback function for http request retries
/// </summary>
/// <param name="response">Http response</param>
/// <param name="hasFailed">Indicates whether the request has failed</param>
/// <param name="retryNumber">The number of retries happened</param>
/// <returns>The wait time before starting the next retry</returns>
private TimeSpan?GetRetryWaitTime(CdmHttpResponse response, bool hasFailed, int retryNumber)
{
if (response != null && (response.IsSuccessful && !hasFailed))
{
return(null);
}
else
{
// Default wait time is calculated using exponential backoff with with random jitter value to avoid 'waves'.
Random random = new Random();
double waitTime = random.Next(1 << retryNumber) * backoffForThrottling;
return(TimeSpan.FromMilliseconds(waitTime));
}
}
private TimeSpan?DefaultGetWaitTime(CdmHttpResponse response, bool hasFailed, int retryNumber)
{
if (response != null && response.IsSuccessful && !hasFailed)
{
return(null);
}
else
{
Random random = new Random();
// Default wait time will be using exponential backoff with default shortest wait time.
double waitTime = random.Next(1 << retryNumber) * 500;
return(TimeSpan.FromMilliseconds(waitTime));
}
}
/// <summary>
/// Callback function for a CDM Http client, it does exponential backoff.
/// </summary>
/// <param name="response">The response received by system's Http client.</param>
/// <param name="hasFailed">Denotes whether the request has failed (usually an exception or 500 error).</param>
/// <param name="retryNumber">The current retry number (starts from 1) up to the number of retries specified by CDM request.</param>
/// <returns>The <see cref="TimeSpan"/>, specifying the waiting time, or null if no wait time is necessary.</returns>
private TimeSpan?DefaultGetWaitTime(CdmHttpResponse response, bool hasFailed, int retryNumber)
{
if (response != null && ((response.IsSuccessful && !hasFailed) || this.AvoidRetryCodes.Contains(response.StatusCode)))
{
return(null);
}
else
{
Random random = new Random();
// Default wait time is calculated using exponential backoff with with random jitter value to avoid 'waves'.
double waitTime = random.Next(1 << retryNumber) * DefaultShortestTimeWait;
return(TimeSpan.FromMilliseconds(waitTime));
}
}
/// <summary>
/// Get an authorization token and send the query to Kusto
/// </summary>
/// <param name="query">The Kusto query command to be posted to the cluster</param>
public async Task PostKustoQuery(string query)
{
string authToken = await this.config.GetAuthenticationToken();
string queryEndpoint = $"https://{this.config.KustoClusterName}.kusto.windows.net/v1/rest/mgmt";
string kustoHost = $"{this.config.KustoClusterName}.kusto.windows.net";
string queryBody = $"{{\"db\":\"{this.config.KustoDatabaseName}\",\"csl\":\"{query}\"}}";
Dictionary <string, string> headers = new Dictionary <string, string>()
{
{ "Accept", "application/json" },
{ "Authorization", authToken },
{ "Host", kustoHost }
};
var httpRequest = new CdmHttpRequest(queryEndpoint)
{
Method = HttpMethod.Post,
Headers = headers,
Content = queryBody,
ContentType = "application/json",
NumberOfRetries = MaxNumRetries,
Timeout = TimeSpan.FromMilliseconds(TimeoutMilliseconds),
MaximumTimeout = TimeSpan.FromMilliseconds(MaxTimeoutMilliseconds)
};
CdmHttpResponse response = await httpClient.SendAsync(httpRequest, GetRetryWaitTime, this.ctx);
if (response == null)
{
throw new HttpRequestException
($"Kusto query post failed. The result of a request is undefined.");
}
if (!response.IsSuccessful)
{
throw new HttpRequestException
($"Kusto query post failed. HTTP {response.StatusCode} - {response.Reason}.");
}
}
/// <inheritdoc />
public async Task <DateTimeOffset?> ComputeLastModifiedTimeAsync(string corpusPath)
{
var adapterPath = this.CreateAdapterPath(corpusPath);
var request = await this.BuildRequest(adapterPath, HttpMethod.Head);
try
{
CdmHttpResponse cdmResponse = await base.ExecuteRequest(request);
if (cdmResponse.StatusCode.Equals(HttpStatusCode.OK))
{
return(cdmResponse.Content.Headers.LastModified);
}
}
catch (HttpRequestException ex)
{
// We don't have standard logger here, so use one from system diagnostics
Debug.WriteLine($"ADLS file not found, skipping last modified time calculation for it. Exception: {ex}");
}
return(null);
}
