static void downloadChunkCallBack(Object context)
{
DownloadContext downloadContext = (DownloadContext)context;
SFResultChunk chunk = downloadContext.chunk;
// change download status
Monitor.Enter(chunk.syncPrimitive);
try
{
chunk.downloadState = DownloadState.IN_PROGRESS;
}
finally
{
Monitor.Exit(chunk.syncPrimitive);
}
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = 60 * 60,
httpRequestTimeout = 16000,
chunkHeaders = downloadContext.chunkHeaders
};
HttpResponseMessage httpResponse = restRequest.get(downloadRequest);
Stream stream = httpResponse.Content.ReadAsStreamAsync().Result;
IEnumerable <string> encoding;
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
/*StreamReader r = new StreamReader(stream);
* string l = r.ReadLine();
* Console.WriteLine(l);*/
chunk.downloadState = DownloadState.SUCCESS;
// signal main thread to start consuming
lock (chunk.syncPrimitive)
{
Monitor.Pulse(chunk.syncPrimitive);
}
}
private void startNextDownload()
{
while (nextChunkToDownloadIndex - nextChunkToConsumeIndex < prefetchSlot && nextChunkToDownloadIndex < chunks.Count)
{
DownloadContext downloadContext = new DownloadContext()
{
chunk = chunks[nextChunkToDownloadIndex],
chunkIndex = nextChunkToDownloadIndex,
qrmk = this.qrmk,
chunkHeaders = this.chunkHeaders
};
ThreadPool.QueueUserWorkItem(new WaitCallback(downloadChunkCallBack), downloadContext);
nextChunkToDownloadIndex++;
}
}
private async Task <IResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest =
new S3DownloadRequest(ResultSet.sfStatement.SfSession.InsecureMode)
{
Url = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = TimeSpan.FromSeconds(32),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = Task.Run(async() => await httpResponse.Content.ReadAsStreamAsync()).Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
ParseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex}");
return(chunk);
}
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = TimeSpan.FromHours(1),
httpRequestTimeout = TimeSpan.FromSeconds(16),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken);
Stream stream = httpResponse.Content.ReadAsStreamAsync().Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
return(chunk);
}