public SFResultChunk getNextChunkToConsume()
{
// prefetch
startNextDownload();
SFResultChunk currentChunk = this.chunks[nextChunkToConsumeIndex];
if (currentChunk.downloadState == DownloadState.SUCCESS)
{
nextChunkToConsumeIndex++;
return(currentChunk);
}
else
{
// wait until donwload finish
lock (currentChunk.syncPrimitive)
{
while (currentChunk.downloadState == DownloadState.IN_PROGRESS ||
currentChunk.downloadState == DownloadState.NOT_STARTED)
{
Monitor.Wait(currentChunk.syncPrimitive);
}
}
nextChunkToConsumeIndex++;
return(currentChunk);
}
}
internal override async Task <bool> NextAsync()
{
if (isClosed)
{
throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED);
}
_currentChunkRowIdx++;
if (_currentChunkRowIdx < _currentChunkRowCount)
{
return(true);
}
if (_chunkDownloader != null)
{
// GetNextChunk could be blocked if download result is not done yet.
// So put this piece of code in a seperate task
Logger.Info("Get next chunk from chunk downloader");
SFResultChunk nextChunk = await _chunkDownloader.GetNextChunkAsync();
if (nextChunk != null)
{
resetChunkInfo(nextChunk);
return(true);
}
else
{
return(false);
}
}
return(false);
}
public SFResultSet(QueryExecResponseData responseData, SFStatement sfStatement, CancellationToken cancellationToken)
{
columnCount = responseData.rowType.Count;
_currentChunkRowIdx = -1;
_currentChunkRowCount = responseData.rowSet.GetLength(0);
this.sfStatement = sfStatement;
if (responseData.chunks != null)
{
// counting the first chunk
_totalChunkCount = responseData.chunks.Count + 1;
_chunkDownloader = new SFChunkDownloader(columnCount,
responseData.chunks,
responseData.qrmk,
responseData.chunkHeaders,
cancellationToken);
}
_currentChunk = new SFResultChunk(responseData.rowSet);
responseData.rowSet = null;
sfResultSetMetaData = new SFResultSetMetaData(responseData);
updateSessionStatus(responseData);
isClosed = false;
}
internal override bool next()
{
if (isClosed)
{
throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED);
}
currentChunkRowIdx++;
if (currentChunkRowIdx < currentChunkRowCount)
{
return(true);
}
else if (++currentChunkIndex < totalChunkCount)
{
execFirstChunkData.rowSet = null;
logger.DebugFormat("Get chunk #{0} to consume", currentChunkIndex);
SFResultChunk chunk = chunkDownloader.getNextChunkToConsume();
if (chunk == null)
{
throw new SnowflakeDbException(SFError.INTERNAL_ERROR, "Failed to get chunk to consume");
}
currentChunk = chunk;
currentChunkRowIdx = 0;
currentChunkRowCount = currentChunk.rowCount;
return(true);
}
else
{
return(false);
}
}
public SFResultSet(QueryExecResponseData responseData, SFStatement sfStatement)
{
execFirstChunkData = responseData;
columnCount = responseData.rowType.Count;
currentChunkRowIdx = -1;
currentChunkRowCount = responseData.rowSet.GetLength(0);
currentChunkIndex = 0;
this.sfStatement = sfStatement;
if (responseData.chunks != null)
{
// counting the first chunk
totalChunkCount = responseData.chunks.Count + 1;
chunkDownloader = new SFChunkDownloader(responseData.rowType.Count,
responseData.chunks,
responseData.qrmk,
responseData.chunkHeaders);
}
currentChunk = new SFResultChunk(responseData.rowSet);
sfResultSetMetaData = new SFResultSetMetaData(responseData);
updateSessionStatus(responseData);
isClosed = false;
}
internal void resetChunkInfo(SFResultChunk nextChunk)
{
Logger.Debug($"Recieved chunk #{nextChunk.ChunkIndex + 1} of {_totalChunkCount}");
_currentChunk.rowSet = null;
_currentChunk = nextChunk;
_currentChunkRowIdx = 0;
_currentChunkRowCount = _currentChunk.rowCount;
}
public void ParseChunk(SFResultChunk chunk)
{
// parse results row by row
using (StreamReader sr = new StreamReader(stream))
using (JsonTextReader jr = new JsonTextReader(sr))
{
chunk.rowSet = JsonSerializer.Deserialize <string[, ]>(jr);
}
}
static void downloadChunkCallBack(Object context)
{
DownloadContext downloadContext = (DownloadContext)context;
SFResultChunk chunk = downloadContext.chunk;
// change download status
Monitor.Enter(chunk.syncPrimitive);
try
{
chunk.downloadState = DownloadState.IN_PROGRESS;
}
finally
{
Monitor.Exit(chunk.syncPrimitive);
}
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = 60 * 60,
httpRequestTimeout = 16000,
chunkHeaders = downloadContext.chunkHeaders
};
HttpResponseMessage httpResponse = restRequest.get(downloadRequest);
Stream stream = httpResponse.Content.ReadAsStreamAsync().Result;
IEnumerable <string> encoding;
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
/*StreamReader r = new StreamReader(stream);
* string l = r.ReadLine();
* Console.WriteLine(l);*/
chunk.downloadState = DownloadState.SUCCESS;
// signal main thread to start consuming
lock (chunk.syncPrimitive)
{
Monitor.Pulse(chunk.syncPrimitive);
}
}
/// <summary>
/// Content from s3 in format of
/// ["val1", "val2", null, ...],
/// ["val3", "val4", null, ...],
/// ...
/// To parse it as a json, we need to preappend '[' and append ']' to the stream
/// </summary>
/// <param name="content"></param>
/// <param name="resultChunk"></param>
private static void parseStreamIntoChunk(Stream content, SFResultChunk resultChunk)
{
Stream openBracket = new MemoryStream(Encoding.UTF8.GetBytes("["));
Stream closeBracket = new MemoryStream(Encoding.UTF8.GetBytes("]"));
Stream concatStream = new ConcatenatedStream(new Stream[3] {
openBracket, content, closeBracket
});
IChunkParser parser = ChunkParserFactory.GetParser(concatStream);
parser.ParseChunk(resultChunk);
}
/// <summary>
/// Content from s3 in format of
/// ["val1", "val2", null, ...],
/// ["val3", "val4", null, ...],
/// ...
/// To parse it as a json, we need to preappend '[' and append ']' to the stream
/// </summary>
/// <param name="content"></param>
/// <param name="resultChunk"></param>
private static void parseStreamIntoChunk(Stream content, SFResultChunk resultChunk)
{
Stream openBracket = new MemoryStream(Encoding.UTF8.GetBytes("["));
Stream closeBracket = new MemoryStream(Encoding.UTF8.GetBytes("]"));
Stream concatStream = new ConcatenatedStream(new Stream[3] {
openBracket, content, closeBracket
});
// parse results row by row
using (StreamReader sr = new StreamReader(concatStream))
using (JsonTextReader jr = new JsonTextReader(sr))
{
resultChunk.rowSet = jsonSerializer.Deserialize <string[, ]>(jr);
}
}
public void ParseChunk(SFResultChunk chunk)
{
// parse results row by row
using (StreamReader sr = new StreamReader(stream))
using (JsonTextReader jr = new JsonTextReader(sr))
{
int row = 0;
int col = 0;
var outputMatrix = new string[chunk.rowCount, chunk.colCount];
while (jr.Read())
{
switch (jr.TokenType)
{
case JsonToken.StartArray:
case JsonToken.None:
break;
case JsonToken.EndArray:
if (col > 0)
{
col = 0;
row++;
}
break;
case JsonToken.Null:
outputMatrix[row, col++] = null;
break;
case JsonToken.String:
outputMatrix[row, col++] = (string)jr.Value;
break;
default:
throw new SnowflakeDbException(SFError.INTERNAL_ERROR, $"Unexpected token type: {jr.TokenType}");
}
}
chunk.rowSet = outputMatrix;
}
}
private async Task <IResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest =
new S3DownloadRequest(ResultSet.sfStatement.SfSession.InsecureMode)
{
Url = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = TimeSpan.FromSeconds(32),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = Task.Run(async() => await httpResponse.Content.ReadAsStreamAsync()).Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
ParseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex}");
return(chunk);
}
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContextV2 downloadContext)
{
logger.Info($"Start downloading chunk #{downloadContext.chunkIndex+1}");
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = TimeSpan.FromHours(1),
httpRequestTimeout = TimeSpan.FromSeconds(16),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken).ConfigureAwait(false);
Stream stream = await httpResponse.Content.ReadAsStreamAsync().ConfigureAwait(false);
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out var encoding))
{
if (string.Equals(encoding.First(), "gzip", StringComparison.OrdinalIgnoreCase))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
logger.Info($"Succeed downloading chunk #{downloadContext.chunkIndex+1}");
return(chunk);
}
private async Task <SFResultChunk> DownloadChunkAsync(DownloadContext downloadContext)
{
SFResultChunk chunk = downloadContext.chunk;
chunk.downloadState = DownloadState.IN_PROGRESS;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
uri = new UriBuilder(chunk.url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
timeout = TimeSpan.FromHours(1),
httpRequestTimeout = TimeSpan.FromSeconds(16),
chunkHeaders = downloadContext.chunkHeaders
};
var httpResponse = await restRequest.GetAsync(downloadRequest, downloadContext.cancellationToken);
Stream stream = httpResponse.Content.ReadAsStreamAsync().Result;
IEnumerable <string> encoding;
//TODO this shouldn't be required.
if (httpResponse.Content.Headers.TryGetValues("Content-Encoding", out encoding))
{
if (String.Compare(encoding.First(), "gzip", true) == 0)
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
}
parseStreamIntoChunk(stream, chunk);
chunk.downloadState = DownloadState.SUCCESS;
return(chunk);
}
