public ArrayEncoder(IReadOnlyList <Column> children, ColumnMetadata metadata) { _children = children; var child = this._children.FirstOrDefault(); encoder = EncoderHelper.GetEncoder(child.Type, metadata.SubColumns.FirstOrDefault(), child.Children); }
public ObjectEncoder(IReadOnlyList <Column> children, Grpc.ColumnMetadata metadata) { this.columnId = metadata.ColumnId; encoders = new IEncoder[children.Count]; propertyGetters = new Func <object, object> [children.Count]; for (int i = 0; i < children.Count; i++) { encoders[i] = EncoderHelper.GetEncoder(children[i].Type, metadata.SubColumns[i], children[i].Children); propertyGetters[i] = children[i].GetFunction; } }
public async Task Query(QueryRequest request, ChannelWriter <Page> channelWriter, ServerCallContext context) { var parameters = ParameterDecoder.DecodeParameters(request.Parameters); QueryResult queryResult; try { queryResult = await _koraliumExecutor.Execute(request.Query, parameters, context.GetHttpContext()); } catch (AuthorizationFailedException authFailed) { throw new RpcException(new Status(StatusCode.Unauthenticated, authFailed.Message)); } var enumerator = queryResult.Result.GetEnumerator(); Page page = new Page() { Metadata = new QueryMetadata() }; foreach (var metadata in queryResult.Metadata) { page.Metadata.CustomMetadata.Add(new KeyValue() { Name = metadata.Key, Value = ScalarEncoder.EncodeScalarResult(metadata.Value) }); } IEncoder[] encoders = new IEncoder[queryResult.Columns.Count]; Func <object, object>[] propertyGetters = new Func <object, object> [encoders.Length]; int indexCounter = 0; for (int i = 0; i < queryResult.Columns.Count; i++) { var column = queryResult.Columns[i]; var columnMetadata = ToMetadata(ref indexCounter, column); encoders[i] = EncoderHelper.GetEncoder(column.Type, columnMetadata, column.Children); propertyGetters[i] = column.GetFunction; page.Metadata.Columns.Add(columnMetadata); } System.Diagnostics.Stopwatch encodeWatch = new System.Diagnostics.Stopwatch(); encodeWatch.Start(); await EncoderHelper.ReadData(_logger, page, encoders, propertyGetters, enumerator, request.MaxBatchSize, channelWriter); encodeWatch.Stop(); _logger.LogTrace($"Encode time: {encodeWatch.ElapsedMilliseconds} ms"); }
public ListEncoder(Column column) { Debug.Assert(column.Children.Count == 1); _getFunc = column.GetFunction; _nullable = column.IsNullable; var child = column.Children.First(); offsetBuilder = new ArrowBuffer.Builder <int>(); _childEncoder = EncoderHelper.GetEncoder(child); _valueType = TypeConverter.Convert(column); nullBitmap = new ArrowBuffer.BitmapBuilder(); }
public ObjectEncoder(Column column) { _getFunc = column.GetFunction; _nullable = column.IsNullable; _type = TypeConverter.Convert(column); _childEncoders = new IArrowEncoder[column.Children.Count]; for (int i = 0; i < _childEncoders.Length; i++) { _childEncoders[i] = EncoderHelper.GetEncoder(column.Children[i]); } nullBitmap = new ArrowBuffer.BitmapBuilder(); }
private static async Task Execute(string sql, HttpContext context) { context.Response.Headers.Add("Content-Type", "application/json"); var koraliumService = context.RequestServices.GetService <IKoraliumTransportService>(); var logger = context.RequestServices.GetService <ILogger <IKoraliumTransportService> >(); QueryResult result = null; try { result = await koraliumService.Execute(sql, new Shared.SqlParameters(), context); } catch (SqlErrorException error) { logger.LogWarning(error.Message); await WriteError(context, 400, error.Message); return; } catch (AuthorizationFailedException authFailed) { logger.LogWarning(authFailed.Message, authFailed); await WriteError(context, 401, authFailed.Message); return; } catch (Exception e) { logger.LogError(e, "Unexpected exception thrown"); await WriteError(context, 500, "Internal error"); return; } var responseStream = new System.Text.Json.Utf8JsonWriter(context.Response.Body); IJsonEncoder[] encoders = new IJsonEncoder[result.Columns.Count]; JsonEncodedText[] names = new JsonEncodedText[result.Columns.Count]; for (int i = 0; i < encoders.Length; i++) { encoders[i] = EncoderHelper.GetEncoder(result.Columns[i]); names[i] = JsonEncodedText.Encode(result.Columns[i].Name); } System.Diagnostics.Stopwatch encodingWatch = new System.Diagnostics.Stopwatch(); encodingWatch.Start(); responseStream.WriteStartObject(); responseStream.WriteStartArray(_valuesText); foreach (var row in result.Result) { responseStream.WriteStartObject(); for (int i = 0; i < encoders.Length; i++) { responseStream.WritePropertyName(names[i]); encoders[i].Encode(in responseStream, in row); } responseStream.WriteEndObject(); } responseStream.WriteEndArray(); responseStream.WriteEndObject(); encodingWatch.Stop(); await responseStream.FlushAsync(); }
public override async Task DoGet(FlightTicket ticket, FlightServerRecordBatchStreamWriter responseStream, ServerCallContext context) { try { int maxBatchSize = DefaultMaxBatchSize; var maxBatchSizeMetadata = context.RequestHeaders.Get("max-batch-size"); if (maxBatchSizeMetadata != null && int.TryParse(maxBatchSizeMetadata.Value, out var parsedMaxBatchSize)) { maxBatchSize = parsedMaxBatchSize; } var queryResult = await _koraliumTransportService.Execute(ticket.Ticket.ToStringUtf8(), new Shared.SqlParameters(), context.GetHttpContext()); //Get the resulting schema var schema = GetSchema(queryResult.Columns); var encoders = queryResult.Columns.Select(x => EncoderHelper.GetEncoder(x)).ToArray(); foreach (var encoder in encoders) { encoder.NewBatch(); } List <object> list = new List <object>(20000); int count = 0; System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); stopwatch.Start(); foreach (var obj in queryResult.Result) { count++; for (int i = 0; i < encoders.Length; i++) { encoders[i].Encode(obj); } if (encoders.Select(x => x.Size()).Sum() > maxBatchSize) { await responseStream.WriteAsync(new RecordBatch(schema, encoders.Select(x => x.BuildArray()), count)); foreach (var encoder in encoders) { encoder.NewBatch(); } count = 0; } } stopwatch.Stop(); var batch = new RecordBatch(schema, encoders.Select(x => x.BuildArray()), count); await responseStream.WriteAsync(batch); } catch (SqlErrorException error) { throw new RpcException(new Status(StatusCode.InvalidArgument, error.Message)); } catch (AuthorizationFailedException authFailed) { throw new RpcException(new Status(StatusCode.Unauthenticated, authFailed.Message)); } catch (Exception e) { throw new RpcException(new Status(StatusCode.Internal, "Internal error")); } }