public static RecordBatch CreateSampleRecordBatch(int length, int columnSetCount)
{
Schema.Builder builder = new Schema.Builder();
for (int i = 0; i < columnSetCount; i++)
{
builder.Field(CreateField(BooleanType.Default, i));
builder.Field(CreateField(UInt8Type.Default, i));
builder.Field(CreateField(Int8Type.Default, i));
builder.Field(CreateField(UInt16Type.Default, i));
builder.Field(CreateField(Int16Type.Default, i));
builder.Field(CreateField(UInt32Type.Default, i));
builder.Field(CreateField(Int32Type.Default, i));
builder.Field(CreateField(UInt64Type.Default, i));
builder.Field(CreateField(Int64Type.Default, i));
builder.Field(CreateField(FloatType.Default, i));
builder.Field(CreateField(DoubleType.Default, i));
builder.Field(CreateField(Date32Type.Default, i));
builder.Field(CreateField(Date64Type.Default, i));
builder.Field(CreateField(TimestampType.Default, i));
builder.Field(CreateField(StringType.Default, i));
//builder.Field(CreateField(new FixedSizeBinaryType(16), i));
//builder.Field(CreateField(new DecimalType(19, 2)));
//builder.Field(CreateField(HalfFloatType.Default));
//builder.Field(CreateField(StringType.Default));
//builder.Field(CreateField(Time32Type.Default));
//builder.Field(CreateField(Time64Type.Default));
}
Schema schema = builder.Build();
IEnumerable <IArrowArray> arrays = CreateArrays(schema, length);
return(new RecordBatch(schema, arrays, length));
}
public void WritesMetadataCorrectly()
{
Schema.Builder schemaBuilder = new Schema.Builder()
.Metadata("index", "1, 2, 3, 4, 5")
.Metadata("reverseIndex", "5, 4, 3, 2, 1")
.Field(f => f
.Name("IntCol")
.DataType(UInt32Type.Default)
.Metadata("custom1", "false")
.Metadata("custom2", "true"))
.Field(f => f
.Name("StringCol")
.DataType(StringType.Default)
.Metadata("custom2", "false")
.Metadata("custom3", "4"))
.Field(f => f
.Name("StructCol")
.DataType(new StructType(new[] {
new Field("Inner1", FloatType.Default, nullable: false),
new Field("Inner2", DoubleType.Default, nullable: true, new Dictionary <string, string>()
{
{ "customInner", "1" }, { "customInner2", "3" }
})
}))
.Metadata("custom4", "6.4")
.Metadata("custom1", "true"));
var schema = schemaBuilder.Build();
RecordBatch originalBatch = TestData.CreateSampleRecordBatch(schema, length: 10);
TestRoundTripRecordBatch(originalBatch);
}
public void getOutputSchemaInner(string sql)
{
reader = new ArrowStreamReader(this.outputBuffer, leaveOpen: false);
// reader one batch to get the arrow schema first
reader.ReadNextRecordBatch();
this.schema = ArrowSchemaToASARecordSchema(reader.Schema);
var result =
SqlCompiler.Compile(
sql,
new QueryBindings(
new Dictionary <string, InputDescription> {
{ "input", new InputDescription(this.schema, InputType.Stream) }
}));
var step = result.Steps.First();
Schema.Builder builder = new Schema.Builder();
foreach (KeyValuePair <string, int> kv in step.Output.PayloadSchema.Ordinals.OrderBy(kv => kv.Value))
{
builder = builder.Field(f => f.Name(kv.Key).DataType(ASATypeToArrowType(step.Output.PayloadSchema[kv.Value].Schema)).Nullable(false));
}
this.outputArrowSchema = builder.Build();
this.writer = new ArrowStreamWriter(this.inputBuffer, this.outputArrowSchema);
//Write empty batch to send the schema to Java side
var emptyRecordBatch = createOutputRecordBatch(new List <IRecord>());
WriteRecordBatch(emptyRecordBatch);
}
public static RecordBatch CreateSampleRecordBatch(int length)
{
Schema.Builder builder = new Schema.Builder();
builder.Field(CreateField(BooleanType.Default));
builder.Field(CreateField(UInt8Type.Default));
builder.Field(CreateField(Int8Type.Default));
builder.Field(CreateField(UInt16Type.Default));
builder.Field(CreateField(Int16Type.Default));
builder.Field(CreateField(UInt32Type.Default));
builder.Field(CreateField(Int32Type.Default));
builder.Field(CreateField(UInt64Type.Default));
builder.Field(CreateField(Int64Type.Default));
builder.Field(CreateField(FloatType.Default));
builder.Field(CreateField(DoubleType.Default));
//builder.Field(CreateField(new DecimalType(19, 2)));
//builder.Field(CreateField(HalfFloatType.Default));
//builder.Field(CreateField(StringType.Default));
//builder.Field(CreateField(Date32Type.Default));
//builder.Field(CreateField(Date64Type.Default));
//builder.Field(CreateField(Time32Type.Default));
//builder.Field(CreateField(Time64Type.Default));
//builder.Field(CreateField(TimestampType.Default));
Schema schema = builder.Build();
IEnumerable <IArrowArray> arrays = CreateArrays(schema, length);
return(new RecordBatch(schema, arrays, length));
}
private static Schema CreateSchema(JsonSchema jsonSchema)
{
Schema.Builder builder = new Schema.Builder();
for (int i = 0; i < jsonSchema.Fields.Count; i++)
{
builder.Field(f => CreateField(f, jsonSchema.Fields[i]));
}
return(builder.Build());
}
public void TestBuildFromRecordBatch()
{
Schema.Builder builder = new Schema.Builder();
builder.Field(new Field("A", Int64Type.Default, nullable: false));
Schema schema = builder.Build();
RecordBatch batch = TestData.CreateSampleRecordBatch(schema, 10);
Table table = Table.TableFromRecordBatches(schema, new[] { batch });
Assert.NotNull(table.Column(0).Data.Array(0) as Int64Array);
}
private Schema GetSchema(IImmutableList <Column> columns)
{
var schemaBuilder = new Schema.Builder();
foreach (var column in columns)
{
schemaBuilder.Field(new Field(column.Name, TypeConverter.Convert(column), column.IsNullable));
}
return(schemaBuilder.Build());
}
public RecordBatch Build()
{
var schema = _schemaBuilder.Build();
var length = _arrays.Max(x => x.Length);
// each array has its own memoryOwner, so the RecordBatch itself doesn't
// have a memoryOwner
IMemoryOwner <byte> memoryOwner = null;
var batch = new RecordBatch(schema, memoryOwner, _arrays, length);
return(batch);
}
internal static Schema GetSchema(Flatbuf.Schema schema)
{
var schemaBuilder = new Schema.Builder();
for (int i = 0; i < schema.FieldsLength; i++)
{
Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
schemaBuilder.Field(FieldFromFlatbuffer(field));
}
return(schemaBuilder.Build());
}
internal static Schema GetSchema(Flatbuf.Schema schema)
{
var schemaBuilder = new Schema.Builder();
for (var i = 0; i < schema.FieldsLength; i++)
{
var field = schema.Fields(i).GetValueOrDefault();
schemaBuilder.Field(
new Field(field.Name, GetFieldArrowType(field), field.Nullable));
}
return(schemaBuilder.Build());
}
public static RecordBatch CreateSampleRecordBatch(int length, int columnSetCount, bool createDictionaryArray)
{
Schema.Builder builder = new Schema.Builder();
for (int i = 0; i < columnSetCount; i++)
{
builder.Field(CreateField(new ListType(Int64Type.Default), i));
builder.Field(CreateField(BooleanType.Default, i));
builder.Field(CreateField(UInt8Type.Default, i));
builder.Field(CreateField(Int8Type.Default, i));
builder.Field(CreateField(UInt16Type.Default, i));
builder.Field(CreateField(Int16Type.Default, i));
builder.Field(CreateField(UInt32Type.Default, i));
builder.Field(CreateField(Int32Type.Default, i));
builder.Field(CreateField(UInt64Type.Default, i));
builder.Field(CreateField(Int64Type.Default, i));
builder.Field(CreateField(FloatType.Default, i));
builder.Field(CreateField(DoubleType.Default, i));
builder.Field(CreateField(Date32Type.Default, i));
builder.Field(CreateField(Date64Type.Default, i));
builder.Field(CreateField(TimestampType.Default, i));
builder.Field(CreateField(StringType.Default, i));
builder.Field(CreateField(new StructType(new List <Field> {
CreateField(StringType.Default, i), CreateField(Int32Type.Default, i)
}), i));
builder.Field(CreateField(new Decimal128Type(10, 6), i));
builder.Field(CreateField(new Decimal256Type(16, 8), i));
if (createDictionaryArray)
{
builder.Field(CreateField(new DictionaryType(Int32Type.Default, StringType.Default, false), i));
}
//builder.Field(CreateField(new FixedSizeBinaryType(16), i));
//builder.Field(CreateField(HalfFloatType.Default));
//builder.Field(CreateField(StringType.Default));
//builder.Field(CreateField(Time32Type.Default));
//builder.Field(CreateField(Time64Type.Default));
}
Schema schema = builder.Build();
return(CreateSampleRecordBatch(schema, length));
}
private static Schema BuildSchema(IArrowArray[] resultColumns)
{
var schemaBuilder = new Schema.Builder();
if (resultColumns.Length == 1)
{
schemaBuilder = schemaBuilder
.Field(f => f.Name("Result")
.DataType(resultColumns[0].Data.DataType)
.Nullable(false));
}
else
{
for (int i = 0; i < resultColumns.Length; ++i)
{
schemaBuilder = schemaBuilder
.Field(f => f.Name("Result" + i)
.DataType(resultColumns[i].Data.DataType)
.Nullable(false));
}
}
return(schemaBuilder.Build());
}
private FlightInfo GetFlightInfo(string sql, ServerCallContext context)
{
var partitionsResult = _koraliumTransportService.GetPartitions(IsPartitionsEnabled(context), sql, new Shared.SqlParameters(), context.GetHttpContext()).Result;
var schemaBuilder = new Schema.Builder();
foreach (var column in partitionsResult.Columns)
{
schemaBuilder.Field(new Field(column.Name, TypeConverter.Convert(column), column.IsNullable));
}
var descriptor = FlightDescriptor.CreateCommandDescriptor(sql);
List <FlightEndpoint> endpoints = new List <FlightEndpoint>();
foreach (var partition in partitionsResult.Partitions)
{
List <FlightLocation> locations = new List <FlightLocation>();
foreach (var location in partition.Locations)
{
string uri = null;
if (location.Tls)
{
uri = $"grpc+tls://{location.Host}";
}
else
{
uri = $"grpc+tcp://{location.Host}";
}
locations.Add(new FlightLocation(uri));
}
endpoints.Add(new FlightEndpoint(new FlightTicket(partition.Sql), locations));
}
return(new FlightInfo(schemaBuilder.Build(), descriptor, endpoints));
}
public void TestListOfStructArray()
{
Schema.Builder builder = new Schema.Builder();
Field structField = new Field(
"struct",
new StructType(
new[]
{
new Field("name", StringType.Default, nullable: false),
new Field("age", Int64Type.Default, nullable: false),
}),
nullable: false);
Field listField = new Field("listOfStructs", new ListType(structField), nullable: false);
builder.Field(listField);
Schema schema = builder.Build();
StringArray stringArray = new StringArray.Builder()
.Append("joe").AppendNull().AppendNull().Append("mark").Append("abe").Append("phil").Build();
Int64Array intArray = new Int64Array.Builder()
.Append(1).Append(2).AppendNull().Append(4).Append(10).Append(55).Build();
ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder()
.Append(true).Append(true).Append(false).Append(true).Append(true).Append(true).Build();
StructArray structs = new StructArray(structField.DataType, 6, new IArrowArray[] { stringArray, intArray }, nullBitmapBuffer, nullCount: 1);
ArrowBuffer offsetsBuffer = new ArrowBuffer.Builder <int>()
.Append(0).Append(2).Append(5).Append(6).Build();
ListArray listArray = new ListArray(listField.DataType, 3, offsetsBuffer, structs, ArrowBuffer.Empty);
RecordBatch batch = new RecordBatch(schema, new[] { listArray }, 3);
TestRoundTripRecordBatch(batch);
}
public Builder SetSchema(Schema.Builder builderForValue)
{
_subscribe.Schema = builderForValue.Build();
return(this);
}
public Builder SetSchema(Schema.Builder builderForValue)
{
_producer.Schema = builderForValue.Build();
return(this);
}