private async Task <IResultChunk> DownloadChunkAsync(DownloadContextV3 downloadContext)
{
//logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFReusableChunk chunk = downloadContext.chunk;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
Url = new UriBuilder(chunk.Url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = Timeout.InfiniteTimeSpan, // Disable timeout for each request
chunkHeaders = downloadContext.chunkHeaders
};
using (var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false))
using (Stream stream = await httpResponse.Content.ReadAsStreamAsync()
.ConfigureAwait(continueOnCapturedContext: false))
{
ParseStreamIntoChunk(stream, chunk);
}
logger.Info($"Succeed downloading chunk #{chunk.chunkIndexToDownload}");
return(chunk);
}
internal async Task <SFBaseResultSet> ExecuteAsync(int timeout, string sql, Dictionary <string, BindingDTO> bindings, bool describeOnly,
CancellationToken cancellationToken)
{
registerQueryCancellationCallback(timeout, cancellationToken);
var queryRequest = BuildQueryRequest(sql, bindings, describeOnly);
try
{
QueryExecResponse response = null;
bool receivedFirstQueryResponse = false;
while (!receivedFirstQueryResponse)
{
response = await _restRequester.PostAsync <QueryExecResponse>(queryRequest, cancellationToken).ConfigureAwait(false);
if (SessionExpired(response))
{
SfSession.renewSession();
queryRequest.authorizationToken = string.Format(SF_AUTHORIZATION_SNOWFLAKE_FMT, SfSession.sessionToken);
}
else
{
receivedFirstQueryResponse = true;
}
}
var lastResultUrl = response.data?.getResultUrl;
while (RequestInProgress(response) || SessionExpired(response))
{
var req = BuildResultRequest(lastResultUrl);
response = await _restRequester.GetAsync <QueryExecResponse>(req, cancellationToken).ConfigureAwait(false);
if (SessionExpired(response))
{
logger.Info("Ping pong request failed with session expired, trying to renew the session.");
SfSession.renewSession();
}
else
{
lastResultUrl = response.data?.getResultUrl;
}
}
return(BuildResultSet(response, cancellationToken));
}
catch
{
logger.Error("Query execution failed.");
throw;
}
finally
{
ClearQueryRequestId();
}
}
internal async Task <SFBaseResultSet> ExecuteAsync(int timeout, string sql, Dictionary <string, BindingDTO> bindings, bool describeOnly,
CancellationToken cancellationToken)
{
registerQueryCancellationCallback(timeout, cancellationToken);
var queryRequest = BuildQueryRequest(sql, bindings, describeOnly);
try
{
var response = await _restRequester.PostAsync <QueryExecResponse>(queryRequest, cancellationToken);
if (SessionExpired(response))
{
SfSession.renewSession();
ClearQueryRequestId();
return(await ExecuteAsync(timeout, sql, bindings, describeOnly, cancellationToken));
}
var lastResultUrl = response.data?.getResultUrl;
while (RequestInProgress(response) || SessionExpired(response))
{
var req = BuildResultRequest(lastResultUrl);
response = await _restRequester.GetAsync <QueryExecResponse>(req, cancellationToken);
if (SessionExpired(response))
{
logger.Info("Ping pong request failed with session expired, trying to renew the session.");
SfSession.renewSession();
}
else
{
lastResultUrl = response.data?.getResultUrl;
}
}
return(BuildResultSet(response, cancellationToken));
}
catch (Exception ex)
{
logger.Error("Query execution failed.", ex);
throw;
}
finally
{
ClearQueryRequestId();
}
}
private async Task <IResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest =
new S3DownloadRequest(ResultSet.sfStatement.SfSession.InsecureMode)
{
Url = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = TimeSpan.FromSeconds(32),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = Task.Run(async() => await httpResponse.Content.ReadAsStreamAsync()).Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
ParseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex}");
return(chunk);
}
private async Task <IResultChunk> DownloadChunkAsync(DownloadContextV2 downloadContext)
{
logger.Info($"Start downloading chunk #{downloadContext.chunkIndex+1}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest = new S3DownloadRequest(InsecureMode)
{
Url = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = TimeSpan.FromSeconds(16),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out var encoding))
{
if (string.Equals(encoding.First(), "gzip", StringComparison.OrdinalIgnoreCase))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex+1}");
return(chunk);
}