/// <summary>
/// Sets up the CDM request that can be used by CDM Http Client.
/// </summary>
/// <param name="path">Partial or full path to a network location.</param>
/// <param name="headers">The headers.</param>
/// <param name="method">The method.</param>
/// <returns>The <see cref="CdmHttpRequest"/>, representing CDM Http request.</returns>
protected CdmHttpRequest SetUpCdmRequest(string path, Dictionary <string, string> headers, HttpMethod method)
{
var httpRequest = new CdmHttpRequest(path, this.NumberOfRetries);
httpRequest.Headers = headers ?? new Dictionary <string, string>();
httpRequest.Timeout = this.Timeout;
httpRequest.MaximumTimeout = this.MaximumTimeout;
httpRequest.Method = method;
return(httpRequest);
}
/// <inheritdoc />
public override async Task WriteAsync(string corpusPath, string data)
{
if (EnsurePath($"{this.Root}{corpusPath}") == false)
{
throw new Exception($"Could not create folder for document '{corpusPath}'");
}
string url = this.CreateFormattedAdapterPath(corpusPath);
CdmHttpResponse response = await this.CreateFileAtPath(corpusPath, url);
try
{
CdmHttpRequest request = await this.BuildRequest($"{url}?action=append&position=0", new HttpMethod("PATCH"), data, "application/json");
response = await this.ExecuteRequest(request);
if (response.StatusCode.Equals(HttpStatusCode.Accepted)) // The uploaded data was accepted.
{
var stringContent = new StringContent(request.Content, Encoding.UTF8, request.ContentType);
// Building a request and setting a URL with a position argument to be the length of the byte array of the string content (or length of UTF-8 string content).
request = await this.BuildRequest($"{url}?action=flush&position={(await stringContent.ReadAsByteArrayAsync()).Length}", new HttpMethod("PATCH"));
response = await this.ExecuteRequest(request);
if (!response.StatusCode.Equals(HttpStatusCode.OK)) // Data was not flushed correctly. Delete empty file.
{
await this.DeleteContentAtPath(corpusPath, url, null);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'");
}
}
else
{
await this.DeleteContentAtPath(corpusPath, url, null);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'");
}
}
catch (StorageAdapterException exc)
{
throw exc;
}
catch (Exception e)
{
await this.DeleteContentAtPath(corpusPath, url, e);
throw new StorageAdapterException($"Could not write ADLS content at path, there was an issue at: '{corpusPath}'", e);
}
}
public void TestResultHttpExceptionZeroRetry()
{
using (var cdmHttpClient = new CdmHttpClient("https://www.example1.com", new CdmHttpMessageHandlerStub(method1)))
{
var cdmHttpRequest = new CdmHttpRequest("/folder1")
{
Timeout = TimeSpan.FromSeconds(5),
MaximumTimeout = TimeSpan.FromSeconds(9)
};
Func <Task> func = async() => await cdmHttpClient.SendAsync(cdmHttpRequest, null);
func.Should().Throw <HttpRequestException>();
}
}
public void TestMaximumTimeout()
{
using (var cdmHttpClient = new CdmHttpClient("https://www.example.com", new CdmHttpMessageHandlerStub(method2)))
{
var cdmHttpRequest = new CdmHttpRequest("/folder2")
{
Timeout = TimeSpan.FromSeconds(1),
MaximumTimeout = TimeSpan.FromSeconds(6),
NumberOfRetries = 3
};
Func <Task> func = async() => await cdmHttpClient.SendAsync(cdmHttpRequest, DefaultGetWaitTime);
func.Should().Throw <CdmTimedOutException>();
}
}
public async Task <CdmHttpResponse> ExecuteRequest(CdmHttpRequest httpRequest)
{
var response = await this.httpClient.SendAsync(httpRequest, this.WaitTimeCallback, this.Ctx);
if (response == null)
{
throw new Exception("The result of a request is undefined.");
}
if (!response.IsSuccessful)
{
throw new HttpRequestException(
$"HTTP {response.StatusCode} - {response.Reason}. Response headers: {string.Join(", ", response.ResponseHeaders.Select(m => m.Key + ":" + m.Value).ToArray())}. URL: {httpRequest.StripSasSig()}");
}
return(response);
}
public async Task TestResultReturnedFirstTime()
{
using (var cdmHttpClient = new CdmHttpClient("https://www.example.com", new CdmHttpMessageHandlerStub(method1)))
{
var cdmHttpRequest = new CdmHttpRequest("/folder1")
{
Timeout = TimeSpan.FromSeconds(5),
MaximumTimeout = TimeSpan.FromSeconds(9)
};
var result = await cdmHttpClient.SendAsync(cdmHttpRequest, null);
var content = await result.Content.ReadAsStringAsync();
Assert.AreEqual("REPLY1", content);
}
}
public async Task TestTimeoutThenSuccessMultipleRetries()
{
using (var cdmHttpClient = new CdmHttpClient("https://www.example.com", new CdmHttpMessageHandlerStub(method2)))
{
var cdmHttpRequest = new CdmHttpRequest("/folder2")
{
Timeout = TimeSpan.FromSeconds(3),
MaximumTimeout = TimeSpan.FromSeconds(150),
NumberOfRetries = 3
};
var result = await cdmHttpClient.SendAsync(cdmHttpRequest, this.DefaultGetWaitTime);
var content = await result.Content.ReadAsStringAsync();
Assert.AreEqual("REPLY2", content);
}
}
/// <summary>
/// Get an authorization token and send the query to Kusto
/// </summary>
/// <param name="query">The Kusto query command to be posted to the cluster</param>
public async Task PostKustoQuery(string query)
{
string authToken = await this.config.GetAuthenticationToken();
string queryEndpoint = $"https://{this.config.KustoClusterName}.kusto.windows.net/v1/rest/mgmt";
string kustoHost = $"{this.config.KustoClusterName}.kusto.windows.net";
string queryBody = $"{{\"db\":\"{this.config.KustoDatabaseName}\",\"csl\":\"{query}\"}}";
Dictionary <string, string> headers = new Dictionary <string, string>()
{
{ "Accept", "application/json" },
{ "Authorization", authToken },
{ "Host", kustoHost }
};
var httpRequest = new CdmHttpRequest(queryEndpoint)
{
Method = HttpMethod.Post,
Headers = headers,
Content = queryBody,
ContentType = "application/json",
NumberOfRetries = MaxNumRetries,
Timeout = TimeSpan.FromMilliseconds(TimeoutMilliseconds),
MaximumTimeout = TimeSpan.FromMilliseconds(MaxTimeoutMilliseconds)
};
CdmHttpResponse response = await httpClient.SendAsync(httpRequest, GetRetryWaitTime, this.ctx);
if (response == null)
{
throw new HttpRequestException
($"Kusto query post failed. The result of a request is undefined.");
}
if (!response.IsSuccessful)
{
throw new HttpRequestException
($"Kusto query post failed. HTTP {response.StatusCode} - {response.Reason}.");
}
}
public async Task <CdmHttpResponse> ExecuteRequest(CdmHttpRequest httpRequest)
{
try
{
var res = await this.httpClient.SendAsync(httpRequest, this.WaitTimeCallback);
if (res == null)
{
throw new Exception("The result of a request is undefined.");
}
if (!res.IsSuccessful)
{
throw new HttpRequestException(res.ToString());
}
return(res);
}
catch (Exception err)
{
throw (err);
}
}
