public ArrayEncoder(IReadOnlyList <Column> children, ColumnMetadata metadata)
{
_children = children;
var child = this._children.FirstOrDefault();
encoder = EncoderHelper.GetEncoder(child.Type, metadata.SubColumns.FirstOrDefault(), child.Children);
}
public ObjectEncoder(IReadOnlyList <Column> children, Grpc.ColumnMetadata metadata)
{
this.columnId = metadata.ColumnId;
encoders = new IEncoder[children.Count];
propertyGetters = new Func <object, object> [children.Count];
for (int i = 0; i < children.Count; i++)
{
encoders[i] = EncoderHelper.GetEncoder(children[i].Type, metadata.SubColumns[i], children[i].Children);
propertyGetters[i] = children[i].GetFunction;
}
}
public async Task Query(QueryRequest request, ChannelWriter <Page> channelWriter, ServerCallContext context)
{
var parameters = ParameterDecoder.DecodeParameters(request.Parameters);
QueryResult queryResult;
try
{
queryResult = await _koraliumExecutor.Execute(request.Query, parameters, context.GetHttpContext());
}
catch (AuthorizationFailedException authFailed)
{
throw new RpcException(new Status(StatusCode.Unauthenticated, authFailed.Message));
}
var enumerator = queryResult.Result.GetEnumerator();
Page page = new Page()
{
Metadata = new QueryMetadata()
};
foreach (var metadata in queryResult.Metadata)
{
page.Metadata.CustomMetadata.Add(new KeyValue()
{
Name = metadata.Key, Value = ScalarEncoder.EncodeScalarResult(metadata.Value)
});
}
IEncoder[] encoders = new IEncoder[queryResult.Columns.Count];
Func <object, object>[] propertyGetters = new Func <object, object> [encoders.Length];
int indexCounter = 0;
for (int i = 0; i < queryResult.Columns.Count; i++)
{
var column = queryResult.Columns[i];
var columnMetadata = ToMetadata(ref indexCounter, column);
encoders[i] = EncoderHelper.GetEncoder(column.Type, columnMetadata, column.Children);
propertyGetters[i] = column.GetFunction;
page.Metadata.Columns.Add(columnMetadata);
}
System.Diagnostics.Stopwatch encodeWatch = new System.Diagnostics.Stopwatch();
encodeWatch.Start();
await EncoderHelper.ReadData(_logger, page, encoders, propertyGetters, enumerator, request.MaxBatchSize, channelWriter);
encodeWatch.Stop();
_logger.LogTrace($"Encode time: {encodeWatch.ElapsedMilliseconds} ms");
}
public ListEncoder(Column column)
{
Debug.Assert(column.Children.Count == 1);
_getFunc = column.GetFunction;
_nullable = column.IsNullable;
var child = column.Children.First();
offsetBuilder = new ArrowBuffer.Builder <int>();
_childEncoder = EncoderHelper.GetEncoder(child);
_valueType = TypeConverter.Convert(column);
nullBitmap = new ArrowBuffer.BitmapBuilder();
}
public ObjectEncoder(Column column)
{
_getFunc = column.GetFunction;
_nullable = column.IsNullable;
_type = TypeConverter.Convert(column);
_childEncoders = new IArrowEncoder[column.Children.Count];
for (int i = 0; i < _childEncoders.Length; i++)
{
_childEncoders[i] = EncoderHelper.GetEncoder(column.Children[i]);
}
nullBitmap = new ArrowBuffer.BitmapBuilder();
}
private static async Task Execute(string sql, HttpContext context)
{
context.Response.Headers.Add("Content-Type", "application/json");
var koraliumService = context.RequestServices.GetService <IKoraliumTransportService>();
var logger = context.RequestServices.GetService <ILogger <IKoraliumTransportService> >();
QueryResult result = null;
try
{
result = await koraliumService.Execute(sql, new Shared.SqlParameters(), context);
}
catch (SqlErrorException error)
{
logger.LogWarning(error.Message);
await WriteError(context, 400, error.Message);
return;
}
catch (AuthorizationFailedException authFailed)
{
logger.LogWarning(authFailed.Message, authFailed);
await WriteError(context, 401, authFailed.Message);
return;
}
catch (Exception e)
{
logger.LogError(e, "Unexpected exception thrown");
await WriteError(context, 500, "Internal error");
return;
}
var responseStream = new System.Text.Json.Utf8JsonWriter(context.Response.Body);
IJsonEncoder[] encoders = new IJsonEncoder[result.Columns.Count];
JsonEncodedText[] names = new JsonEncodedText[result.Columns.Count];
for (int i = 0; i < encoders.Length; i++)
{
encoders[i] = EncoderHelper.GetEncoder(result.Columns[i]);
names[i] = JsonEncodedText.Encode(result.Columns[i].Name);
}
System.Diagnostics.Stopwatch encodingWatch = new System.Diagnostics.Stopwatch();
encodingWatch.Start();
responseStream.WriteStartObject();
responseStream.WriteStartArray(_valuesText);
foreach (var row in result.Result)
{
responseStream.WriteStartObject();
for (int i = 0; i < encoders.Length; i++)
{
responseStream.WritePropertyName(names[i]);
encoders[i].Encode(in responseStream, in row);
}
responseStream.WriteEndObject();
}
responseStream.WriteEndArray();
responseStream.WriteEndObject();
encodingWatch.Stop();
await responseStream.FlushAsync();
}
public override async Task DoGet(FlightTicket ticket, FlightServerRecordBatchStreamWriter responseStream, ServerCallContext context)
{
try
{
int maxBatchSize = DefaultMaxBatchSize;
var maxBatchSizeMetadata = context.RequestHeaders.Get("max-batch-size");
if (maxBatchSizeMetadata != null && int.TryParse(maxBatchSizeMetadata.Value, out var parsedMaxBatchSize))
{
maxBatchSize = parsedMaxBatchSize;
}
var queryResult = await _koraliumTransportService.Execute(ticket.Ticket.ToStringUtf8(), new Shared.SqlParameters(), context.GetHttpContext());
//Get the resulting schema
var schema = GetSchema(queryResult.Columns);
var encoders = queryResult.Columns.Select(x => EncoderHelper.GetEncoder(x)).ToArray();
foreach (var encoder in encoders)
{
encoder.NewBatch();
}
List <object> list = new List <object>(20000);
int count = 0;
System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch();
stopwatch.Start();
foreach (var obj in queryResult.Result)
{
count++;
for (int i = 0; i < encoders.Length; i++)
{
encoders[i].Encode(obj);
}
if (encoders.Select(x => x.Size()).Sum() > maxBatchSize)
{
await responseStream.WriteAsync(new RecordBatch(schema, encoders.Select(x => x.BuildArray()), count));
foreach (var encoder in encoders)
{
encoder.NewBatch();
}
count = 0;
}
}
stopwatch.Stop();
var batch = new RecordBatch(schema, encoders.Select(x => x.BuildArray()), count);
await responseStream.WriteAsync(batch);
}
catch (SqlErrorException error)
{
throw new RpcException(new Status(StatusCode.InvalidArgument, error.Message));
}
catch (AuthorizationFailedException authFailed)
{
throw new RpcException(new Status(StatusCode.Unauthenticated, authFailed.Message));
}
catch (Exception e)
{
throw new RpcException(new Status(StatusCode.Internal, "Internal error"));
}
}
