public void TestSingleElement()
        {
            Int32Array  array       = new Int32Array.Builder().Append(1).Append(2).Build();
            IArrowArray actualArray = ArrowArrayConcatenatorReflector.InvokeConcatenate(new[] { array });

            ArrowReaderVerifier.CompareArrays(array, actualArray);
        }
예제 #2
0
            public void Visit(StructType type)
            {
                StringArray.Builder stringBuilder = new StringArray.Builder();
                for (int i = 0; i < Length; i++)
                {
                    stringBuilder.Append(i.ToString());
                }
                StringArray stringArray = stringBuilder.Build();

                Int32Array.Builder intBuilder = new Int32Array.Builder();
                for (int i = 0; i < Length; i++)
                {
                    intBuilder.Append(i);
                }
                Int32Array intArray = intBuilder.Build();

                List <Array> arrays = new List <Array>();

                arrays.Add(stringArray);
                arrays.Add(intArray);

                ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
                for (int i = 0; i < Length; i++)
                {
                    nullBitmap.Append(true);
                }

                Array = new StructArray(type, Length, arrays, nullBitmap.Build());
            }
            public void Visit(StructType type)
            {
                // TODO: Make data from type fields.

                // The following can be improved with a Builder class for StructArray.
                StringArray.Builder resultStringBuilder = new StringArray.Builder();
                Int32Array.Builder  resultInt32Builder  = new Int32Array.Builder();
                ArrowBuffer         nullBitmapBuffer    = new ArrowBuffer.BitmapBuilder().Append(true).Append(true).Append(false).Build();

                for (int i = 0; i < 3; i++)
                {
                    resultStringBuilder.Append("joe").AppendNull().AppendNull().Append("mark");
                    resultInt32Builder.Append(1).Append(2).AppendNull().Append(4);
                    StringArray  stringArray = new StringArray.Builder().Append("joe").AppendNull().AppendNull().Append("mark").Build();
                    Int32Array   intArray    = new Int32Array.Builder().Append(1).Append(2).AppendNull().Append(4).Build();
                    List <Array> arrays      = new List <Array>
                    {
                        stringArray,
                        intArray
                    };

                    TestTargetArrayList.Add(new StructArray(type, 3, arrays, nullBitmapBuffer, 1));
                }

                StringArray resultStringArray = resultStringBuilder.Build();
                Int32Array  resultInt32Array  = resultInt32Builder.Build();

                ExpectedArray = new StructArray(type, 3, new List <Array> {
                    resultStringArray, resultInt32Array
                }, nullBitmapBuffer, 1);
            }
예제 #4
0
        private Tuple <StringArray, Int32Array, DictionaryArray> CreateSimpleTestData()
        {
            StringArray originalDictionary   = new StringArray.Builder().AppendRange(new[] { "a", "b", "c" }).Build();
            Int32Array  originalIndicesArray = new Int32Array.Builder().AppendRange(new[] { 0, 0, 1, 1, 2, 2 }).Build();
            var         dictionaryArray      = new DictionaryArray(new DictionaryType(Int32Type.Default, StringType.Default, false), originalIndicesArray, originalDictionary);

            return(Tuple.Create(originalDictionary, originalIndicesArray, dictionaryArray));
        }
예제 #5
0
        private RecordBatch CreateTestBatch(int startValue, int length)
        {
            var batchBuilder = new RecordBatch.Builder();

            Int32Array.Builder builder = new Int32Array.Builder();
            for (int i = 0; i < length; i++)
            {
                builder.Append(startValue + i);
            }
            batchBuilder.Append("test", true, builder.Build());
            return(batchBuilder.Build());
        }
예제 #6
0
            public void Visit(DictionaryType type)
            {
                Int32Array.Builder  indicesBuilder = new Int32Array.Builder().Reserve(Length);
                StringArray.Builder valueBuilder   = new StringArray.Builder().Reserve(Length);

                for (int i = 0; i < Length; i++)
                {
                    indicesBuilder.Append(i);
                    valueBuilder.Append($"{i}");
                }

                Array = new DictionaryArray(type, indicesBuilder.Build(), valueBuilder.Build());
            }
        public void TestInconsistentNullBitMapLength()
        {
            // Arrow allocates buffers of length 64 by default. 64 * 8 = 512 bits in the NullBitMapBuffer. Anything lesser than 512 will not trigger a throw
            Int32Array  int32         = new Int32Array.Builder().AppendRange(Enumerable.Range(0, 520)).Build();
            RecordBatch originalBatch = new RecordBatch.Builder()
                                        .Append("EmptyDataColumn", true, new Int32Array(
                                                    valueBuffer: int32.ValueBuffer,
                                                    nullBitmapBuffer: new ArrowBuffer.Builder <byte>().Append(0x00).Build(),
                                                    length: 520,
                                                    nullCount: 520,
                                                    offset: 0)).Build();

            Assert.ThrowsAny <ArgumentException>(() => DataFrame.FromArrowRecordBatch(originalBatch));
        }
        private Tuple <Field, DictionaryArray> CreateListTypeDictionaryArrayTestData(List <string> dictionaryDataBase)
        {
            var listBuilder  = new ListArray.Builder(StringType.Default);
            var valueBuilder = listBuilder.ValueBuilder as StringArray.Builder;

            foreach (string data in dictionaryDataBase)
            {
                listBuilder.Append();
                valueBuilder.Append(data);
            }

            ListArray  dictionary          = listBuilder.Build();
            Int32Array indiceArray         = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();
            var        dictionaryArrayType = new DictionaryType(Int32Type.Default, dictionary.Data.DataType, false);
            var        dictionaryArray     = new DictionaryArray(dictionaryArrayType, indiceArray, dictionary);

            return(Tuple.Create(new Field($"dictionaryField_{dictionaryArray.Data.DataType.Name}", dictionaryArrayType, false), dictionaryArray));
        }
        private Tuple <Field, ListArray> CreateDictionaryTypeListArrayTestData(StringArray dictionary)
        {
            Int32Array indiceArray = new Int32Array.Builder().AppendRange(Enumerable.Range(0, dictionary.Length)).Build();

            //DictionaryArray has no Builder for now, so creating ListArray directly.
            var dictionaryType  = new DictionaryType(Int32Type.Default, StringType.Default, false);
            var dictionaryArray = new DictionaryArray(dictionaryType, indiceArray, dictionary);

            var valueOffsetsBufferBuilder = new ArrowBuffer.Builder <int>();
            var validityBufferBuilder     = new ArrowBuffer.BitmapBuilder();

            foreach (int i in Enumerable.Range(0, dictionary.Length + 1))
            {
                valueOffsetsBufferBuilder.Append(i);
                validityBufferBuilder.Append(true);
            }

            var dictionaryField = new Field("dictionaryField_list", dictionaryType, false);
            var listType        = new ListType(dictionaryField);
            var listArray       = new ListArray(listType, valueOffsetsBufferBuilder.Length - 1, valueOffsetsBufferBuilder.Build(), dictionaryArray, valueOffsetsBufferBuilder.Build());

            return(Tuple.Create(new Field($"listField_{listType.ValueDataType.Name}", listType, false), listArray));
        }
예제 #10
0
 public void NewBatch()
 {
     _builder = new Int32Array.Builder();
 }
예제 #11
0
        public void TestStructArray()
        {
            // The following can be improved with a Builder class for StructArray.
            List <Field> fields = new List <Field>();

            Field.Builder fieldBuilder = new Field.Builder();
            fields.Add(fieldBuilder.Name("Strings").DataType(StringType.Default).Nullable(true).Build());
            fieldBuilder = new Field.Builder();
            fields.Add(fieldBuilder.Name("Ints").DataType(Int32Type.Default).Nullable(true).Build());
            StructType structType = new StructType(fields);

            StringArray.Builder stringBuilder = new StringArray.Builder();
            StringArray         stringArray   = stringBuilder.Append("joe").AppendNull().AppendNull().Append("mark").Build();

            Int32Array.Builder intBuilder = new Int32Array.Builder();
            Int32Array         intArray   = intBuilder.Append(1).Append(2).AppendNull().Append(4).Build();
            List <Array>       arrays     = new List <Array>();

            arrays.Add(stringArray);
            arrays.Add(intArray);

            ArrowBuffer.BitmapBuilder nullBitmap = new ArrowBuffer.BitmapBuilder();
            var         nullBitmapBuffer         = nullBitmap.Append(true).Append(true).Append(false).Append(true).Build();
            StructArray structs = new StructArray(structType, 4, arrays, nullBitmapBuffer, 1);

            Assert.Equal(4, structs.Length);
            Assert.Equal(1, structs.NullCount);
            ArrayData[] childArrays = structs.Data.Children; // Data for StringArray and Int32Array
            Assert.Equal(2, childArrays.Length);
            for (int i = 0; i < childArrays.Length; i++)
            {
                ArrayData arrayData = childArrays[i];
                Assert.Null(arrayData.Children);
                if (i == 0)
                {
                    Assert.Equal(ArrowTypeId.String, arrayData.DataType.TypeId);
                    Array       array             = new StringArray(arrayData);
                    StringArray structStringArray = array as StringArray;
                    Assert.NotNull(structStringArray);
                    Assert.Equal(structs.Length, structStringArray.Length);
                    Assert.Equal(stringArray.Length, structStringArray.Length);
                    Assert.Equal(stringArray.NullCount, structStringArray.NullCount);
                    for (int j = 0; j < stringArray.Length; j++)
                    {
                        Assert.Equal(stringArray.GetString(j), structStringArray.GetString(j));
                    }
                }
                if (i == 1)
                {
                    Assert.Equal(ArrowTypeId.Int32, arrayData.DataType.TypeId);
                    Array      array          = new Int32Array(arrayData);
                    Int32Array structIntArray = array as Int32Array;
                    Assert.NotNull(structIntArray);
                    Assert.Equal(structs.Length, structIntArray.Length);
                    Assert.Equal(intArray.Length, structIntArray.Length);
                    Assert.Equal(intArray.NullCount, structIntArray.NullCount);
                    for (int j = 0; j < intArray.Length; j++)
                    {
                        Assert.Equal(intArray.GetValue(j), structIntArray.GetValue(j));
                    }
                }
            }
        }