private RecordBatch CreateTestBatch(int startValue, int length)
{
var batchBuilder = new RecordBatch.Builder();
Int32Array.Builder builder = new Int32Array.Builder();
for (int i = 0; i < length; i++)
{
builder.Append(startValue + i);
}
batchBuilder.Append("test", true, builder.Build());
return(batchBuilder.Build());
}
private RecordBatch createOutputRecordBatch(List <IRecord> rows)
{
var recordBatchBuilder = new RecordBatch.Builder(memoryAllocator);
for (int i = 0; i < this.outputArrowSchema.Fields.Count; i++)
{
var field = this.outputArrowSchema.GetFieldByIndex(i);
switch (field.DataType.TypeId)
{
case ArrowTypeId.Int64:
recordBatchBuilder.Append(field.Name, field.IsNullable, col => col.Int64(
array => array.AppendRange(rows.Select(row => Convert.ToInt64(row[i])))));
break;
case ArrowTypeId.Double:
recordBatchBuilder.Append(field.Name, field.IsNullable, col => col.Double(
array => array.AppendRange(rows.Select(row => Convert.ToDouble(row[i])))));
break;
case ArrowTypeId.String:
recordBatchBuilder.Append(field.Name, field.IsNullable, col => col.String(
array => array.AppendRange(rows.Select(row => Convert.ToString(row[i])))));
break;
case ArrowTypeId.Timestamp:
recordBatchBuilder.Append(field.Name, field.IsNullable, col => col.Int64(
array => array.AppendRange(rows.Select(row => (((DateTime)row[i]).Ticks - epoch.Ticks) / MicrosecToMillisecRatio))));
break;
case ArrowTypeId.Binary:
recordBatchBuilder.Append(field.Name, field.IsNullable, col => col.Binary(
array => array.AppendRange(rows.Select(row => (byte[])(row[i])))));
break;
case ArrowTypeId.Boolean:
recordBatchBuilder.Append(field.Name, field.IsNullable, col => col.Boolean(
array => array.AppendRange(rows.Select(row => Convert.ToBoolean(row[i])))));
break;
default: throw new Exception("Unsupported Arrow type of output arrow schema: " + field.DataType.TypeId);
}
}
return(recordBatchBuilder.Build());
}
public ByteString ToGprcArrowFrame()
{
MemoryStream stream = new MemoryStream();
var recordBatchBuilder = new RecordBatch.Builder();
foreach (Field field in fields)
{
recordBatchBuilder.Append(field.Name, true, field.ToArrowArray());
}
var recordBatch = recordBatchBuilder.Build();
var writer = new ArrowFileWriter(stream, recordBatch.Schema);
writer.WriteRecordBatch(recordBatch);
writer.WriteEnd();
stream.Position = 0;
return(ByteString.FromStream(stream));
}
private static void ExecuteActions(MemoryAllocator allocator, RecordBatch batch, IReadOnlyList <IAction> actions,
int iterations)
{
var builder = new RecordBatch.Builder(allocator);
for (var i = 0; i < iterations; i++)
{
foreach (var action in actions)
{
action.Execute(batch, builder);
}
try
{
batch = builder.Build();
builder = new RecordBatch.Builder(allocator);
}
catch (InvalidOperationException)
{
}
}
}