public void TestSingleElement()
{
Int32Array array = new Int32Array.Builder().Append(1).Append(2).Build();
IArrowArray actualArray = ArrowArrayConcatenatorReflector.InvokeConcatenate(new[] { array });
ArrowReaderVerifier.CompareArrays(array, actualArray);
}
public void Visit(StructType type)
{
StringArray.Builder stringBuilder = new StringArray.Builder();
for (int i = 0; i < Length; i++)
{
stringBuilder.Append(i.ToString());
}
StringArray stringArray = stringBuilder.Build();
Int32Array.Builder intBuilder = new Int32Array.Builder();
for (int i = 0; i < Length; i++)
{
intBuilder.Append(i);
}
Int32Array intArray = intBuilder.Build();
List <Array> arrays = new List <Array>();
arrays.Add(stringArray);
arrays.Add(intArray);
ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
for (int i = 0; i < Length; i++)
{
nullBitmap.Append(true);
}
Array = new StructArray(type, Length, arrays, nullBitmap.Build());
}
public void Visit(StructType type)
{
// TODO: Make data from type fields.
// The following can be improved with a Builder class for StructArray.
StringArray.Builder resultStringBuilder = new StringArray.Builder();
Int32Array.Builder resultInt32Builder = new Int32Array.Builder();
ArrowBuffer nullBitmapBuffer = new ArrowBuffer.BitmapBuilder().Append(true).Append(true).Append(false).Build();
for (int i = 0; i < 3; i++)
{
resultStringBuilder.Append("joe").AppendNull().AppendNull().Append("mark");
resultInt32Builder.Append(1).Append(2).AppendNull().Append(4);
StringArray stringArray = new StringArray.Builder().Append("joe").AppendNull().AppendNull().Append("mark").Build();
Int32Array intArray = new Int32Array.Builder().Append(1).Append(2).AppendNull().Append(4).Build();
List <Array> arrays = new List <Array>
{
stringArray,
intArray
};
TestTargetArrayList.Add(new StructArray(type, 3, arrays, nullBitmapBuffer, 1));
}
StringArray resultStringArray = resultStringBuilder.Build();
Int32Array resultInt32Array = resultInt32Builder.Build();
ExpectedArray = new StructArray(type, 3, new List <Array> {
resultStringArray, resultInt32Array
}, nullBitmapBuffer, 1);
}
private Tuple <StringArray, Int32Array, DictionaryArray> CreateSimpleTestData()
{
StringArray originalDictionary = new StringArray.Builder().AppendRange(new[] { "a", "b", "c" }).Build();
Int32Array originalIndicesArray = new Int32Array.Builder().AppendRange(new[] { 0, 0, 1, 1, 2, 2 }).Build();
var dictionaryArray = new DictionaryArray(new DictionaryType(Int32Type.Default, StringType.Default, false), originalIndicesArray, originalDictionary);
return(Tuple.Create(originalDictionary, originalIndicesArray, dictionaryArray));
}
private RecordBatch CreateTestBatch(int startValue, int length)
{
var batchBuilder = new RecordBatch.Builder();
Int32Array.Builder builder = new Int32Array.Builder();
for (int i = 0; i < length; i++)
{
builder.Append(startValue + i);
}
batchBuilder.Append("test", true, builder.Build());
return(batchBuilder.Build());
}
public void Visit(DictionaryType type)
{
Int32Array.Builder indicesBuilder = new Int32Array.Builder().Reserve(Length);
StringArray.Builder valueBuilder = new StringArray.Builder().Reserve(Length);
for (int i = 0; i < Length; i++)
{
indicesBuilder.Append(i);
valueBuilder.Append($"{i}");
}
Array = new DictionaryArray(type, indicesBuilder.Build(), valueBuilder.Build());
}
public void TestInconsistentNullBitMapLength()
{
// Arrow allocates buffers of length 64 by default. 64 * 8 = 512 bits in the NullBitMapBuffer. Anything lesser than 512 will not trigger a throw
Int32Array int32 = new Int32Array.Builder().AppendRange(Enumerable.Range(0, 520)).Build();
RecordBatch originalBatch = new RecordBatch.Builder()
.Append("EmptyDataColumn", true, new Int32Array(
valueBuffer: int32.ValueBuffer,
nullBitmapBuffer: new ArrowBuffer.Builder <byte>().Append(0x00).Build(),
length: 520,
nullCount: 520,
offset: 0)).Build();
Assert.ThrowsAny <ArgumentException>(() => DataFrame.FromArrowRecordBatch(originalBatch));
}
private Tuple <Field, DictionaryArray> CreateListTypeDictionaryArrayTestData(List <string> dictionaryDataBase)
{
var listBuilder = new ListArray.Builder(StringType.Default);
var valueBuilder = listBuilder.ValueBuilder as StringArray.Builder;
foreach (string data in dictionaryDataBase)
{
listBuilder.Append();
valueBuilder.Append(data);
}
ListArray dictionary = listBuilder.Build();
Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
var dictionaryArrayType = new DictionaryType(Int32Type.Default, dictionary.Data.DataType, false);
var dictionaryArray = new DictionaryArray(dictionaryArrayType, indiceArray, dictionary);
return(Tuple.Create(new Field($"dictionaryField_{dictionaryArray.Data.DataType.Name}", dictionaryArrayType, false), dictionaryArray));
}
private Tuple <Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary)
{
Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
//DictionaryArray has no Builder for now, so creating ListArray directly.
var dictionaryType = new DictionaryType(Int32Type.Default, StringType.Default, false);
var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary);
var valueOffsetsBufferBuilder = new ArrowBuffer.Builder <int>();
var validityBufferBuilder = new ArrowBuffer.BitmapBuilder();
foreach (int i in Enumerable.Range(0, dictionary.Length + 1))
{
valueOffsetsBufferBuilder.Append(i);
validityBufferBuilder.Append(true);
}
var dictionaryField = new Field("dictionaryField_list", dictionaryType, false);
var listType = new ListType(dictionaryField);
var listArray = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build());
return(Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray));
}
public void NewBatch()
{
_builder = new Int32Array.Builder();
}
public void TestStructArray()
{
// The following can be improved with a Builder class for StructArray.
List <Field> fields = new List <Field>();
Field.Builder fieldBuilder = new Field.Builder();
fields.Add(fieldBuilder.Name("Strings").DataType(StringType.Default).Nullable(true).Build());
fieldBuilder = new Field.Builder();
fields.Add(fieldBuilder.Name("Ints").DataType(Int32Type.Default).Nullable(true).Build());
StructType structType = new StructType(fields);
StringArray.Builder stringBuilder = new StringArray.Builder();
StringArray stringArray = stringBuilder.Append("joe").AppendNull().AppendNull().Append("mark").Build();
Int32Array.Builder intBuilder = new Int32Array.Builder();
Int32Array intArray = intBuilder.Append(1).Append(2).AppendNull().Append(4).Build();
List <Array> arrays = new List <Array>();
arrays.Add(stringArray);
arrays.Add(intArray);
ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
var nullBitmapBuffer = nullBitmap.Append(true).Append(true).Append(false).Append(true).Build();
StructArray structs = new StructArray(structType, 4, arrays, nullBitmapBuffer, 1);
Assert.Equal(4, structs.Length);
Assert.Equal(1, structs.NullCount);
ArrayData[] childArrays = structs.Data.Children; // Data for StringArray and Int32Array
Assert.Equal(2, childArrays.Length);
for (int i = 0; i < childArrays.Length; i++)
{
ArrayData arrayData = childArrays[i];
Assert.Null(arrayData.Children);
if (i == 0)
{
Assert.Equal(ArrowTypeId.String, arrayData.DataType.TypeId);
Array array = new StringArray(arrayData);
StringArray structStringArray = array as StringArray;
Assert.NotNull(structStringArray);
Assert.Equal(structs.Length, structStringArray.Length);
Assert.Equal(stringArray.Length, structStringArray.Length);
Assert.Equal(stringArray.NullCount, structStringArray.NullCount);
for (int j = 0; j < stringArray.Length; j++)
{
Assert.Equal(stringArray.GetString(j), structStringArray.GetString(j));
}
}
if (i == 1)
{
Assert.Equal(ArrowTypeId.Int32, arrayData.DataType.TypeId);
Array array = new Int32Array(arrayData);
Int32Array structIntArray = array as Int32Array;
Assert.NotNull(structIntArray);
Assert.Equal(structs.Length, structIntArray.Length);
Assert.Equal(intArray.Length, structIntArray.Length);
Assert.Equal(intArray.NullCount, structIntArray.NullCount);
for (int j = 0; j < intArray.Length; j++)
{
Assert.Equal(intArray.GetValue(j), structIntArray.GetValue(j));
}
}
}
}