public SFResultChunk getNextChunkToConsume() { // prefetch startNextDownload(); SFResultChunk currentChunk = this.chunks[nextChunkToConsumeIndex]; if (currentChunk.downloadState == DownloadState.SUCCESS) { nextChunkToConsumeIndex++; return(currentChunk); } else { // wait until donwload finish lock (currentChunk.syncPrimitive) { while (currentChunk.downloadState == DownloadState.IN_PROGRESS || currentChunk.downloadState == DownloadState.NOT_STARTED) { Monitor.Wait(currentChunk.syncPrimitive); } } nextChunkToConsumeIndex++; return(currentChunk); } }
internal override async Task <bool> NextAsync() { if (isClosed) { throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED); } _currentChunkRowIdx++; if (_currentChunkRowIdx < _currentChunkRowCount) { return(true); } if (_chunkDownloader != null) { // GetNextChunk could be blocked if download result is not done yet. // So put this piece of code in a seperate task Logger.Info("Get next chunk from chunk downloader"); SFResultChunk nextChunk = await _chunkDownloader.GetNextChunkAsync(); if (nextChunk != null) { resetChunkInfo(nextChunk); return(true); } else { return(false); } } return(false); }
public SFResultSet(QueryExecResponseData responseData, SFStatement sfStatement, CancellationToken cancellationToken) { columnCount = responseData.rowType.Count; _currentChunkRowIdx = -1; _currentChunkRowCount = responseData.rowSet.GetLength(0); this.sfStatement = sfStatement; if (responseData.chunks != null) { // counting the first chunk _totalChunkCount = responseData.chunks.Count + 1; _chunkDownloader = new SFChunkDownloader(columnCount, responseData.chunks, responseData.qrmk, responseData.chunkHeaders, cancellationToken); } _currentChunk = new SFResultChunk(responseData.rowSet); responseData.rowSet = null; sfResultSetMetaData = new SFResultSetMetaData(responseData); updateSessionStatus(responseData); isClosed = false; }
internal override bool next() { if (isClosed) { throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED); } currentChunkRowIdx++; if (currentChunkRowIdx < currentChunkRowCount) { return(true); } else if (++currentChunkIndex < totalChunkCount) { execFirstChunkData.rowSet = null; logger.DebugFormat("Get chunk #{0} to consume", currentChunkIndex); SFResultChunk chunk = chunkDownloader.getNextChunkToConsume(); if (chunk == null) { throw new SnowflakeDbException(SFError.INTERNAL_ERROR, "Failed to get chunk to consume"); } currentChunk = chunk; currentChunkRowIdx = 0; currentChunkRowCount = currentChunk.rowCount; return(true); } else { return(false); } }
public SFResultSet(QueryExecResponseData responseData, SFStatement sfStatement) { execFirstChunkData = responseData; columnCount = responseData.rowType.Count; currentChunkRowIdx = -1; currentChunkRowCount = responseData.rowSet.GetLength(0); currentChunkIndex = 0; this.sfStatement = sfStatement; if (responseData.chunks != null) { // counting the first chunk totalChunkCount = responseData.chunks.Count + 1; chunkDownloader = new SFChunkDownloader(responseData.rowType.Count, responseData.chunks, responseData.qrmk, responseData.chunkHeaders); } currentChunk = new SFResultChunk(responseData.rowSet); sfResultSetMetaData = new SFResultSetMetaData(responseData); updateSessionStatus(responseData); isClosed = false; }
internal void resetChunkInfo(SFResultChunk nextChunk) { Logger.Debug($"Recieved chunk #{nextChunk.ChunkIndex + 1} of {_totalChunkCount}"); _currentChunk.rowSet = null; _currentChunk = nextChunk; _currentChunkRowIdx = 0; _currentChunkRowCount = _currentChunk.rowCount; }
public void ParseChunk(SFResultChunk chunk) { // parse results row by row using (StreamReader sr = new StreamReader(stream)) using (JsonTextReader jr = new JsonTextReader(sr)) { chunk.rowSet = JsonSerializer.Deserialize <string[, ]>(jr); } }
static void downloadChunkCallBack(Object context) { DownloadContext downloadContext = (DownloadContext)context; SFResultChunk chunk = downloadContext.chunk; // change download status Monitor.Enter(chunk.syncPrimitive); try { chunk.downloadState = DownloadState.IN_PROGRESS; } finally { Monitor.Exit(chunk.syncPrimitive); } S3DownloadRequest downloadRequest = new S3DownloadRequest() { uri = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour timeout = 60 * 60, httpRequestTimeout = 16000, chunkHeaders = downloadContext.chunkHeaders }; HttpResponseMessage httpResponse = restRequest.get(downloadRequest); Stream stream = httpResponse.Content.ReadAsStreamAsync().Result; IEnumerable <string> encoding; if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding)) { if (String.Compare(encoding.First(), "gzip", true) == 0) { stream = new GZipStream(stream, CompressionMode.Decompress); } } parseStreamIntoChunk(stream, chunk); /*StreamReader r = new StreamReader(stream); * string l = r.ReadLine(); * Console.WriteLine(l);*/ chunk.downloadState = DownloadState.SUCCESS; // signal main thread to start consuming lock (chunk.syncPrimitive) { Monitor.Pulse(chunk.syncPrimitive); } }
/// <summary> /// Content from s3 in format of /// ["val1", "val2", null, ...], /// ["val3", "val4", null, ...], /// ... /// To parse it as a json, we need to preappend '[' and append ']' to the stream /// </summary> /// <param name="content"></param> /// <param name="resultChunk"></param> private static void parseStreamIntoChunk(Stream content, SFResultChunk resultChunk) { Stream openBracket = new MemoryStream(Encoding.UTF8.GetBytes("[")); Stream closeBracket = new MemoryStream(Encoding.UTF8.GetBytes("]")); Stream concatStream = new ConcatenatedStream(new Stream[3] { openBracket, content, closeBracket }); IChunkParser parser = ChunkParserFactory.GetParser(concatStream); parser.ParseChunk(resultChunk); }
/// <summary> /// Content from s3 in format of /// ["val1", "val2", null, ...], /// ["val3", "val4", null, ...], /// ... /// To parse it as a json, we need to preappend '[' and append ']' to the stream /// </summary> /// <param name="content"></param> /// <param name="resultChunk"></param> private static void parseStreamIntoChunk(Stream content, SFResultChunk resultChunk) { Stream openBracket = new MemoryStream(Encoding.UTF8.GetBytes("[")); Stream closeBracket = new MemoryStream(Encoding.UTF8.GetBytes("]")); Stream concatStream = new ConcatenatedStream(new Stream[3] { openBracket, content, closeBracket }); // parse results row by row using (StreamReader sr = new StreamReader(concatStream)) using (JsonTextReader jr = new JsonTextReader(sr)) { resultChunk.rowSet = jsonSerializer.Deserialize <string[, ]>(jr); } }
public void ParseChunk(SFResultChunk chunk) { // parse results row by row using (StreamReader sr = new StreamReader(stream)) using (JsonTextReader jr = new JsonTextReader(sr)) { int row = 0; int col = 0; var outputMatrix = new string[chunk.rowCount, chunk.colCount]; while (jr.Read()) { switch (jr.TokenType) { case JsonToken.StartArray: case JsonToken.None: break; case JsonToken.EndArray: if (col > 0) { col = 0; row++; } break; case JsonToken.Null: outputMatrix[row, col++] = null; break; case JsonToken.String: outputMatrix[row, col++] = (string)jr.Value; break; default: throw new SnowflakeDbException(SFError.INTERNAL_ERROR, $"Unexpected token type: {jr.TokenType}"); } } chunk.rowSet = outputMatrix; } }
private async Task <IResultChunk> DownloadChunkAsync(DownloadContext downloadContext) { logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}"); SFResultChunk chunk = downloadContext.chunk; chunk.downloadState = DownloadState.IN_PROGRESS; S3DownloadRequest downloadRequest = new S3DownloadRequest(ResultSet.sfStatement.SfSession.InsecureMode) { Url = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour RestTimeout = TimeSpan.FromHours(1), HttpTimeout = TimeSpan.FromSeconds(32), chunkHeaders = downloadContext.chunkHeaders }; var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false); Stream stream = Task.Run(async() => await httpResponse.Content.ReadAsStreamAsync()).Result; IEnumerable <string> encoding; //TODO this shouldn't be required. if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding)) { if (String.Compare(encoding.First(), "gzip", true) == 0) { stream = new GZipStream(stream, CompressionMode.Decompress); } } ParseStreamIntoChunk(stream, chunk); chunk.downloadState = DownloadState.SUCCESS; logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex}"); return(chunk); }
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContextV2 downloadContext) { logger.Info($"Start downloading chunk #{downloadContext.chunkIndex+1}"); SFResultChunk chunk = downloadContext.chunk; chunk.downloadState = DownloadState.IN_PROGRESS; S3DownloadRequest downloadRequest = new S3DownloadRequest() { uri = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour timeout = TimeSpan.FromHours(1), httpRequestTimeout = TimeSpan.FromSeconds(16), chunkHeaders = downloadContext.chunkHeaders }; var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false); Stream stream = await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false); if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out var encoding)) { if (string.Equals(encoding.First(), "gzip", StringComparison.OrdinalIgnoreCase)) { stream = new GZipStream(stream, CompressionMode.Decompress); } } parseStreamIntoChunk(stream, chunk); chunk.downloadState = DownloadState.SUCCESS; logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex+1}"); return(chunk); }
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContext downloadContext) { SFResultChunk chunk = downloadContext.chunk; chunk.downloadState = DownloadState.IN_PROGRESS; S3DownloadRequest downloadRequest = new S3DownloadRequest() { uri = new UriBuilder(chunk.url).Uri, qrmk = downloadContext.qrmk, // s3 download request timeout to one hour timeout = TimeSpan.FromHours(1), httpRequestTimeout = TimeSpan.FromSeconds(16), chunkHeaders = downloadContext.chunkHeaders }; var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken); Stream stream = httpResponse.Content.ReadAsStreamAsync().Result; IEnumerable <string> encoding; //TODO this shouldn't be required. if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding)) { if (String.Compare(encoding.First(), "gzip", true) == 0) { stream = new GZipStream(stream, CompressionMode.Decompress); } } parseStreamIntoChunk(stream, chunk); chunk.downloadState = DownloadState.SUCCESS; return(chunk); }