public void Visit(StructType type)
{
StringArray.Builder stringBuilder = new StringArray.Builder();
for (int i = 0; i < Length; i++)
{
stringBuilder.Append(i.ToString());
}
StringArray stringArray = stringBuilder.Build();
Int32Array.Builder intBuilder = new Int32Array.Builder();
for (int i = 0; i < Length; i++)
{
intBuilder.Append(i);
}
Int32Array intArray = intBuilder.Build();
List <Array> arrays = new List <Array>();
arrays.Add(stringArray);
arrays.Add(intArray);
ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
for (int i = 0; i < Length; i++)
{
nullBitmap.Append(true);
}
Array = new StructArray(type, Length, arrays, nullBitmap.Build());
}
public void Visit(StructType type)
{
// TODO: Make data from type fields.
// The following can be improved with a Builder class for StructArray.
StringArray.Builder resultStringBuilder = new StringArray.Builder();
Int32Array.Builder resultInt32Builder = new Int32Array.Builder();
ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder().Append(true).Append(true).Append(false).Build();
for (int i = 0; i < 3; i++)
{
resultStringBuilder.Append("joe").AppendNull().AppendNull().Append("mark");
resultInt32Builder.Append(1).Append(2).AppendNull().Append(4);
StringArray stringArray = new StringArray.Builder().Append("joe").AppendNull().AppendNull().Append("mark").Build();
Int32Array intArray = new Int32Array.Builder().Append(1).Append(2).AppendNull().Append(4).Build();
List <Array> arrays = new List <Array>
{
stringArray,
intArray
};
TestTargetArrayList.Add(new StructArray(type, 3, arrays, nullBitmapBuffer, 1));
}
StringArray resultStringArray = resultStringBuilder.Build();
Int32Array resultInt32Array = resultInt32Builder.Build();
ExpectedArray = new StructArray(type, 3, new List <Array> {
resultStringArray, resultInt32Array
}, nullBitmapBuffer, 1);
}
private Tuple <StringArray, Int32Array, DictionaryArray> CreateSimpleTestData()
{
StringArray originalDictionary = new StringArray.Builder().AppendRange(new[] { "a", "b", "c" }).Build();
Int32Array originalIndicesArray = new Int32Array.Builder().AppendRange(new[] { 0, 0, 1, 1, 2, 2 }).Build();
var dictionaryArray = new DictionaryArray(new DictionaryType(Int32Type.Default, StringType.Default, false), originalIndicesArray, originalDictionary);
return(Tuple.Create(originalDictionary, originalIndicesArray, dictionaryArray));
}
public void Visit(StringType type)
{
var str = "helloworld";
var builder = new StringArray.Builder();
for (var i = 0; i < Length; i++)
{
builder.Append(str);
}
Array = builder.Build();
}
public void TestBasicArrowStringColumn()
{
StringArray strArray = new StringArray.Builder().Append("foo").Append("bar").Build();
Memory <byte> dataMemory = new byte[] { 102, 111, 111, 98, 97, 114 };
Memory <byte> nullMemory = new byte[] { 0, 0, 0, 0 };
Memory <byte> offsetMemory = new byte[] { 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0 };
ArrowStringDataFrameColumn stringColumn = new ArrowStringDataFrameColumn("String", dataMemory, offsetMemory, nullMemory, strArray.Length, strArray.NullCount);
Assert.Equal(2, stringColumn.Length);
Assert.Equal("foo", stringColumn[0]);
Assert.Equal("bar", stringColumn[1]);
}
public void Visit(DictionaryType type)
{
Int32Array.Builder indicesBuilder = new Int32Array.Builder().Reserve(Length);
StringArray.Builder valueBuilder = new StringArray.Builder().Reserve(Length);
for (int i = 0; i < Length; i++)
{
indicesBuilder.Append(i);
valueBuilder.Append($"{i}");
}
Array = new DictionaryArray(type, indicesBuilder.Build(), valueBuilder.Build());
}
private List <RecordBatch> CreateMultipleDictionaryArraysTestData()
{
var dictionaryData = new List <string> {
"a", "b", "c"
};
int length = dictionaryData.Count;
var schemaForSimpleCase = new Schema(new List <Field> {
new Field("int8", Int8Type.Default, true),
new Field("uint8", UInt8Type.Default, true),
new Field("int16", Int16Type.Default, true),
new Field("uint16", UInt16Type.Default, true),
new Field("int32", Int32Type.Default, true),
new Field("uint32", UInt32Type.Default, true),
new Field("int64", Int64Type.Default, true),
new Field("uint64", UInt64Type.Default, true)
}, null);
StringArray dictionary = new StringArray.Builder().AppendRange(dictionaryData).Build();
IEnumerable <IArrowArray> indicesArraysForSimpleCase = TestData.CreateArrays(schemaForSimpleCase, length);
var fields = new List <Field>(capacity: length + 1);
var testTargetArrays = new List <IArrowArray>(capacity: length + 1);
foreach (IArrowArray indices in indicesArraysForSimpleCase)
{
var dictionaryArray = new DictionaryArray(
new DictionaryType(indices.Data.DataType, StringType.Default, false),
indices, dictionary);
testTargetArrays.Add(dictionaryArray);
fields.Add(new Field($"dictionaryField_{indices.Data.DataType.Name}", dictionaryArray.Data.DataType, false));
}
(Field dictionaryTypeListArrayField, ListArray dictionaryTypeListArray) = CreateDictionaryTypeListArrayTestData(dictionary);
fields.Add(dictionaryTypeListArrayField);
testTargetArrays.Add(dictionaryTypeListArray);
(Field listTypeDictionaryArrayField, DictionaryArray listTypeDictionaryArray) = CreateListTypeDictionaryArrayTestData(dictionaryData);
fields.Add(listTypeDictionaryArrayField);
testTargetArrays.Add(listTypeDictionaryArray);
var schema = new Schema(fields, null);
return(new List <RecordBatch> {
new RecordBatch(schema, testTargetArrays, length),
new RecordBatch(schema, testTargetArrays, length),
});
}
public void TestArrowStringColumnClone()
{
StringArray strArray = new StringArray.Builder().Append("foo").Append("bar").Build();
Memory <byte> dataMemory = new byte[] { 102, 111, 111, 98, 97, 114 };
Memory <byte> nullMemory = new byte[] { 0, 0, 0, 0 };
Memory <byte> offsetMemory = new byte[] { 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0 };
ArrowStringColumn stringColumn = new ArrowStringColumn("String", dataMemory, offsetMemory, nullMemory, strArray.Length, strArray.NullCount);
BaseColumn clone = stringColumn.Clone(numberOfNullsToAppend: 5);
Assert.Equal(7, clone.Length);
Assert.Equal(stringColumn[0], clone[0]);
Assert.Equal(stringColumn[1], clone[1]);
}
public void Visit(StringType type)
{
StringArray.Builder resultBuilder = new StringArray.Builder().Reserve(_baseDataTotalElementCount);
for (int i = 0; i < _baseDataListCount; i++)
{
List <int?> dataList = _baseData[i];
StringArray.Builder builder = new StringArray.Builder().Reserve(dataList.Count);
foreach (string value in dataList.Select(_ => _.ToString() ?? null))
{
builder.Append(value);
resultBuilder.Append(value);
}
TestTargetArrayList.Add(builder.Build());
}
ExpectedArray = resultBuilder.Build();
}
public void TestArrowStringColumnGetReadOnlyBuffers()
{
// Test ArrowStringDataFrameColumn.
StringArray strArray = new StringArray.Builder().Append("foo").Append("bar").Build();
Memory <byte> dataMemory = new byte[] { 102, 111, 111, 98, 97, 114 };
Memory <byte> nullMemory = new byte[] { 1 };
Memory <byte> offsetMemory = new byte[] { 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0 };
ArrowStringDataFrameColumn column = new ArrowStringDataFrameColumn("String", dataMemory, offsetMemory, nullMemory, strArray.Length, strArray.NullCount);
IEnumerable <ReadOnlyMemory <byte> > dataBuffers = column.GetReadOnlyDataBuffers();
IEnumerable <ReadOnlyMemory <byte> > nullBitMaps = column.GetReadOnlyNullBitMapBuffers();
IEnumerable <ReadOnlyMemory <int> > offsetsBuffers = column.GetReadOnlyOffsetsBuffers();
using (IEnumerator <ReadOnlyMemory <byte> > bufferEnumerator = dataBuffers.GetEnumerator())
using (IEnumerator <ReadOnlyMemory <int> > offsetsEnumerator = offsetsBuffers.GetEnumerator())
using (IEnumerator <ReadOnlyMemory <byte> > nullBitMapsEnumerator = nullBitMaps.GetEnumerator())
{
while (bufferEnumerator.MoveNext() && nullBitMapsEnumerator.MoveNext() && offsetsEnumerator.MoveNext())
{
ReadOnlyMemory <byte> dataBuffer = bufferEnumerator.Current;
ReadOnlyMemory <byte> nullBitMap = nullBitMapsEnumerator.Current;
ReadOnlyMemory <int> offsets = offsetsEnumerator.Current;
ReadOnlySpan <byte> dataSpan = dataBuffer.Span;
ReadOnlySpan <int> offsetsSpan = offsets.Span;
int dataStart = 0;
for (int j = 1; j < offsetsSpan.Length; j++)
{
int length = offsetsSpan[j] - offsetsSpan[j - 1];
ReadOnlySpan <byte> str = dataSpan.Slice(dataStart, length);
ReadOnlySpan <byte> columnStr = dataMemory.Span.Slice(dataStart, length);
Assert.Equal(str.Length, columnStr.Length);
for (int s = 0; s < str.Length; s++)
{
Assert.Equal(str[s], columnStr[s]);
}
dataStart = length;
}
}
}
}
public void TestListOfStructArray()
{
Schema.Builder builder = new Schema.Builder();
Field structField = new Field(
"struct",
new StructType(
new[]
{
new Field("name", StringType.Default, nullable: false),
new Field("age", Int64Type.Default, nullable: false),
}),
nullable: false);
Field listField = new Field("listOfStructs", new ListType(structField), nullable: false);
builder.Field(listField);
Schema schema = builder.Build();
StringArray stringArray = new StringArray.Builder()
.Append("joe").AppendNull().AppendNull().Append("mark").Append("abe").Append("phil").Build();
Int64Array intArray = new Int64Array.Builder()
.Append(1).Append(2).AppendNull().Append(4).Append(10).Append(55).Build();
ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder()
.Append(true).Append(true).Append(false).Append(true).Append(true).Append(true).Build();
StructArray structs = new StructArray(structField.DataType, 6, new IArrowArray[] { stringArray, intArray }, nullBitmapBuffer, nullCount: 1);
ArrowBuffer offsetsBuffer = new ArrowBuffer.Builder <int>()
.Append(0).Append(2).Append(5).Append(6).Build();
ListArray listArray = new ListArray(listField.DataType, 3, offsetsBuffer, structs, ArrowBuffer.Empty);
RecordBatch batch = new RecordBatch(schema, new[] { listArray }, 3);
TestRoundTripRecordBatch(batch);
}
public void NewBatch()
{
_size = 0;
_builder = new StringArray.Builder();
}
public void TestStructArray()
{
// The following can be improved with a Builder class for StructArray.
List <Field> fields = new List <Field>();
Field.Builder fieldBuilder = new Field.Builder();
fields.Add(fieldBuilder.Name("Strings").DataType(StringType.Default).Nullable(true).Build());
fieldBuilder = new Field.Builder();
fields.Add(fieldBuilder.Name("Ints").DataType(Int32Type.Default).Nullable(true).Build());
StructType structType = new StructType(fields);
StringArray.Builder stringBuilder = new StringArray.Builder();
StringArray stringArray = stringBuilder.Append("joe").AppendNull().AppendNull().Append("mark").Build();
Int32Array.Builder intBuilder = new Int32Array.Builder();
Int32Array intArray = intBuilder.Append(1).Append(2).AppendNull().Append(4).Build();
List <Array> arrays = new List <Array>();
arrays.Add(stringArray);
arrays.Add(intArray);
ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
var nullBitmapBuffer = nullBitmap.Append(true).Append(true).Append(false).Append(true).Build();
StructArray structs = new StructArray(structType, 4, arrays, nullBitmapBuffer, 1);
Assert.Equal(4, structs.Length);
Assert.Equal(1, structs.NullCount);
ArrayData[] childArrays = structs.Data.Children; // Data for StringArray and Int32Array
Assert.Equal(2, childArrays.Length);
for (int i = 0; i < childArrays.Length; i++)
{
ArrayData arrayData = childArrays[i];
Assert.Null(arrayData.Children);
if (i == 0)
{
Assert.Equal(ArrowTypeId.String, arrayData.DataType.TypeId);
Array array = new StringArray(arrayData);
StringArray structStringArray = array as StringArray;
Assert.NotNull(structStringArray);
Assert.Equal(structs.Length, structStringArray.Length);
Assert.Equal(stringArray.Length, structStringArray.Length);
Assert.Equal(stringArray.NullCount, structStringArray.NullCount);
for (int j = 0; j < stringArray.Length; j++)
{
Assert.Equal(stringArray.GetString(j), structStringArray.GetString(j));
}
}
if (i == 1)
{
Assert.Equal(ArrowTypeId.Int32, arrayData.DataType.TypeId);
Array array = new Int32Array(arrayData);
Int32Array structIntArray = array as Int32Array;
Assert.NotNull(structIntArray);
Assert.Equal(structs.Length, structIntArray.Length);
Assert.Equal(intArray.Length, structIntArray.Length);
Assert.Equal(intArray.NullCount, structIntArray.NullCount);
for (int j = 0; j < intArray.Length; j++)
{
Assert.Equal(intArray.GetValue(j), structIntArray.GetValue(j));
}
}
}
}
