static void downloadChunkCallBack(Object context) { DownloadContext downloadContext = (DownloadContext)context; SFResultChunk chunk = downloadContext.chunk; // change download status Monitor.Enter(chunk.syncPrimitive); try { chunk.downloadState = DownloadState.IN_PROGRESS; } finally { Monitor.Exit(chunk.syncPrimitive); } S3DownloadRequest downloadRequest = new S3DownloadRequest() { uri = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour timeout = 60 * 60, httpRequestTimeout = 16000, chunkHeaders = downloadContext.chunkHeaders }; HttpResponseMessage httpResponse = restRequest.get(downloadRequest); Stream stream = httpResponse.Content.ReadAsStreamAsync().Result; IEnumerable <string> encoding; if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding)) { if (String.Compare(encoding.First(), "gzip", true) == 0) { stream = new GZipStream(stream, CompressionMode.Decompress); } } parseStreamIntoChunk(stream, chunk); /*StreamReader r = new StreamReader(stream); * string l = r.ReadLine(); * Console.WriteLine(l);*/ chunk.downloadState = DownloadState.SUCCESS; // signal main thread to start consuming lock (chunk.syncPrimitive) { Monitor.Pulse(chunk.syncPrimitive); } }
private void startNextDownload() { while (nextChunkToDownloadIndex - nextChunkToConsumeIndex < prefetchSlot && nextChunkToDownloadIndex < chunks.Count) { DownloadContext downloadContext = new DownloadContext() { chunk = chunks[nextChunkToDownloadIndex], chunkIndex = nextChunkToDownloadIndex, qrmk = this.qrmk, chunkHeaders = this.chunkHeaders }; ThreadPool.QueueUserWorkItem(new WaitCallback(downloadChunkCallBack), downloadContext); nextChunkToDownloadIndex++; } }
private async Task <IResultChunk> DownloadChunkAsync(DownloadContext downloadContext) { logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}"); SFResultChunk chunk = downloadContext.chunk; chunk.downloadState = DownloadState.IN_PROGRESS; S3DownloadRequest downloadRequest = new S3DownloadRequest(ResultSet.sfStatement.SfSession.InsecureMode) { Url = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour RestTimeout = TimeSpan.FromHours(1), HttpTimeout = TimeSpan.FromSeconds(32), chunkHeaders = downloadContext.chunkHeaders }; var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false); Stream stream = Task.Run(async() => await httpResponse.Content.ReadAsStreamAsync()).Result; IEnumerable <string> encoding; //TODO this shouldn't be required. if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding)) { if (String.Compare(encoding.First(), "gzip", true) == 0) { stream = new GZipStream(stream, CompressionMode.Decompress); } } ParseStreamIntoChunk(stream, chunk); chunk.downloadState = DownloadState.SUCCESS; logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex}"); return(chunk); }
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContext downloadContext) { SFResultChunk chunk = downloadContext.chunk; chunk.downloadState = DownloadState.IN_PROGRESS; S3DownloadRequest downloadRequest = new S3DownloadRequest() { uri = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour timeout = TimeSpan.FromHours(1), httpRequestTimeout = TimeSpan.FromSeconds(16), chunkHeaders = downloadContext.chunkHeaders }; var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken); Stream stream = httpResponse.Content.ReadAsStreamAsync().Result; IEnumerable <string> encoding; //TODO this shouldn't be required. if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding)) { if (String.Compare(encoding.First(), "gzip", true) == 0) { stream = new GZipStream(stream, CompressionMode.Decompress); } } parseStreamIntoChunk(stream, chunk); chunk.downloadState = DownloadState.SUCCESS; return(chunk); }