public void ParseChunk(IResultChunk chunk)
{
SFReusableChunk rc = (SFReusableChunk)chunk;
// parse results row by row
using (StreamReader sr = new StreamReader(stream))
using (JsonTextReader jr = new JsonTextReader(sr))
{
while (jr.Read())
{
switch (jr.TokenType)
{
case JsonToken.StartArray:
case JsonToken.None:
case JsonToken.EndArray:
break;
case JsonToken.Null:
rc.AddCell(null);
break;
case JsonToken.String:
rc.AddCell((string)jr.Value);
break;
default:
throw new SnowflakeDbException(SFError.INTERNAL_ERROR, $"Unexpected token type: {jr.TokenType}");
}
}
}
}
public SFBlockingChunkDownloaderV3(int colCount,
List <ExecResponseChunk> chunkInfos, string qrmk,
Dictionary <string, string> chunkHeaders,
CancellationToken cancellationToken,
SFBaseResultSet ResultSet)
{
this.qrmk = qrmk;
this.chunkHeaders = chunkHeaders;
this.nextChunkToDownloadIndex = 0;
this.ResultSet = ResultSet;
this.prefetchSlot = Math.Min(chunkInfos.Count, GetPrefetchThreads(ResultSet));
this.chunkInfos = chunkInfos;
this.nextChunkToConsumeIndex = 0;
this.taskQueues = new List <Task <IResultChunk> >();
externalCancellationToken = cancellationToken;
for (int i = 0; i < prefetchSlot; i++)
{
SFReusableChunk reusableChunk = new SFReusableChunk(colCount);
reusableChunk.Reset(chunkInfos[nextChunkToDownloadIndex], nextChunkToDownloadIndex);
chunkDatas.Add(reusableChunk);
taskQueues.Add(DownloadChunkAsync(new DownloadContextV3()
{
chunk = reusableChunk,
qrmk = this.qrmk,
chunkHeaders = this.chunkHeaders,
cancellationToken = this.externalCancellationToken
}));
nextChunkToDownloadIndex++;
}
}
/*public Task<IResultChunk> GetNextChunkAsync()
* {
* return _downloadTasks.IsCompleted ? Task.FromResult<SFResultChunk>(null) : _downloadTasks.Take();
* }*/
public Task <IResultChunk> GetNextChunkAsync()
{
logger.Info($"NextChunkToConsume: {nextChunkToConsumeIndex}, NextChunkToDownload: {nextChunkToDownloadIndex}");
if (nextChunkToConsumeIndex < chunkInfos.Count)
{
Task <IResultChunk> chunk = taskQueues[nextChunkToConsumeIndex % prefetchSlot];
if (nextChunkToDownloadIndex < chunkInfos.Count && nextChunkToConsumeIndex > 0)
{
SFReusableChunk reusableChunk = chunkDatas[nextChunkToDownloadIndex % prefetchSlot];
reusableChunk.Reset(chunkInfos[nextChunkToDownloadIndex], nextChunkToDownloadIndex);
taskQueues[nextChunkToDownloadIndex % prefetchSlot] = DownloadChunkAsync(new DownloadContextV3()
{
chunk = reusableChunk,
qrmk = this.qrmk,
chunkHeaders = this.chunkHeaders,
cancellationToken = externalCancellationToken
});
nextChunkToDownloadIndex++;
}
nextChunkToConsumeIndex++;
return(chunk);
}
else
{
return(Task.FromResult <IResultChunk>(null));
}
}
private async Task <IResultChunk> DownloadChunkAsync(DownloadContextV3 downloadContext)
{
//logger.Info($"Start donwloading chunk #{downloadContext.chunkIndex}");
SFReusableChunk chunk = downloadContext.chunk;
S3DownloadRequest downloadRequest = new S3DownloadRequest()
{
Url = new UriBuilder(chunk.Url).Uri,
qrmk = downloadContext.qrmk,
// s3 download request timeout to one hour
RestTimeout = TimeSpan.FromHours(1),
HttpTimeout = Timeout.InfiniteTimeSpan, // Disable timeout for each request
chunkHeaders = downloadContext.chunkHeaders
};
using (var httpResponse = await restRequester.GetAsync(downloadRequest, downloadContext.cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false))
using (Stream stream = await httpResponse.Content.ReadAsStreamAsync()
.ConfigureAwait(continueOnCapturedContext: false))
{
ParseStreamIntoChunk(stream, chunk);
}
logger.Info($"Succeed downloading chunk #{chunk.chunkIndexToDownload}");
return(chunk);
}
public void ParseChunk(IResultChunk chunk)
{
SFReusableChunk rc = (SFReusableChunk)chunk;
bool inString = false;
int c;
var input = new FastStreamWrapper(stream);
var ms = new FastMemoryStream();
while ((c = input.ReadByte()) >= 0)
{
if (!inString)
{
// n means null
// " quote means begin string
// all else are ignored
if (c == '"')
{
inString = true;
}
else if (c == 'n')
{
rc.AddCell(null, 0);
}
// ignore anything else
}
else
{
// Inside a string, look for end string
// Anything else is saved in the buffer
if (c == '"')
{
rc.AddCell(ms.GetBuffer(), ms.Length);
ms.Clear();
inString = false;
}
else if (c == '\\')
{
// Process next character
c = input.ReadByte();
switch (c)
{
case 'n':
c = '\n';
break;
case 'r':
c = '\r';
break;
case 'b':
c = '\b';
break;
case 't':
c = '\t';
break;
case -1:
throw new SnowflakeDbException(SFError.INTERNAL_ERROR, $"Unexpected end of stream in escape sequence");
}
ms.WriteByte((byte)c);
}
else
{
ms.WriteByte((byte)c);
}
}
}
if (inString)
{
throw new SnowflakeDbException(SFError.INTERNAL_ERROR, $"Unexpected end of stream in string");
}
}