private async Task <IResultChunk> DownloadChunkAsync(DownloadContextV3 downloadContext)
{
//logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFReusableChunk chunk = downloadContext.chunk;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
Url = new UriBuilder(chunk.Url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = Timeout.InfiniteTimeSpan, // Disable timeout for each request
chunkHeaders = downloadContext.chunkHeaders
};
using (var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false))
using (Stream stream = await httpResponse.Content.ReadAsStreamAsync()
.ConfigureAwait(continueOnCapturedContext: false))
{
ParseStreamIntoChunk(stream, chunk);
}
logger.Info($"Succeed downloading chunk #{chunk.chunkIndexToDownload}");
return(chunk);
}
static void downloadChunkCallBack(Object context)
{
DownloadContext downloadContext = (DownloadContext)context;
SFResultChunk chunk = downloadContext.chunk;
// change download status
Monitor.Enter(chunk.syncPrimitive);
try
{
chunk.downloadState = DownloadState.IN_PROGRESS;
}
finally
{
Monitor.Exit(chunk.syncPrimitive);
}
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = 60 * 60,
httpRequestTimeout = 16000,
chunkHeaders = downloadContext.chunkHeaders
};
HttpResponseMessage httpResponse = restRequest.get(downloadRequest);
Stream stream = httpResponse.Content.ReadAsStreamAsync().Result;
IEnumerable <string> encoding;
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
/*StreamReader r = new StreamReader(stream);
* string l = r.ReadLine();
* Console.WriteLine(l);*/
chunk.downloadState = DownloadState.SUCCESS;
// signal main thread to start consuming
lock (chunk.syncPrimitive)
{
Monitor.Pulse(chunk.syncPrimitive);
}
}
public Task <HttpResponseMessage> GetAsync(S3DownloadRequest getRequest, CancellationToken cancellationToken)
{
HttpRequestMessage message = new HttpRequestMessage(HttpMethod.Get, getRequest.uri);
if (getRequest.chunkHeaders != null)
{
foreach (var item in getRequest.chunkHeaders)
{
message.Headers.Add(item.Key, item.Value);
}
}
else
{
message.Headers.Add(SSE_C_ALGORITHM, SSE_C_AES);
message.Headers.Add(SSE_C_KEY, getRequest.qrmk);
}
message.Properties["TIMEOUT_PER_HTTP_REQUEST"] = getRequest.httpRequestTimeout;
return(SendAsync(message, getRequest.timeout, cancellationToken));
}
private async Task <IResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest =
new S3DownloadRequest(ResultSet.sfStatement.SfSession.InsecureMode)
{
Url = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = TimeSpan.FromSeconds(32),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = Task.Run(async() => await httpResponse.Content.ReadAsStreamAsync()).Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
ParseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex}");
return(chunk);
}
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContextV2 downloadContext)
{
logger.Info($"Start downloading chunk #{downloadContext.chunkIndex+1}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = TimeSpan.FromHours(1),
httpRequestTimeout = TimeSpan.FromSeconds(16),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out var encoding))
{
if (string.Equals(encoding.First(), "gzip", StringComparison.OrdinalIgnoreCase))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex+1}");
return(chunk);
}
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = TimeSpan.FromHours(1),
httpRequestTimeout = TimeSpan.FromSeconds(16),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken);
Stream stream = httpResponse.Content.ReadAsStreamAsync().Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
return(chunk);
}
