コード例 #1
0
        private void ReadSchema()
        {
            if (HasReadSchema)
            {
                return;
            }

            // Figure out length of schema
            int schemaMessageLength = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Span.Slice(_bufferPosition));

            _bufferPosition += sizeof(int);

            ByteBuffer schemaBuffer = CreateByteBuffer(_buffer.Slice(_bufferPosition));

            Schema           = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemaBuffer));
            _bufferPosition += schemaMessageLength;
        }
コード例 #2
0
        protected virtual void ReadSchema()
        {
            if (HasReadSchema)
            {
                return;
            }

            // Figure out length of schema
            int schemaMessageLength = ReadMessageLength(throwOnFullRead: true);

            ArrayPool <byte> .Shared.RentReturn(schemaMessageLength, buff =>
            {
                int bytesRead = BaseStream.ReadFullBuffer(buff);
                EnsureFullRead(buff, bytesRead);

                FlatBuffers.ByteBuffer schemabb = CreateByteBuffer(buff);
                Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemabb), ref _dictionaryMemo);
            });
        }
コード例 #3
0
        protected virtual async ValueTask ReadSchemaAsync()
        {
            if (HasReadSchema)
            {
                return;
            }

            // Figure out length of schema
            int schemaMessageLength = await ReadMessageLengthAsync(throwOnFullRead : true)
                                      .ConfigureAwait(false);

            await ArrayPool <byte> .Shared.RentReturnAsync(schemaMessageLength, async (buff) =>
            {
                // Read in schema
                int bytesRead = await BaseStream.ReadFullBufferAsync(buff).ConfigureAwait(false);
                EnsureFullRead(buff, bytesRead);

                FlatBuffers.ByteBuffer schemabb = CreateByteBuffer(buff);
                Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemabb), ref _dictionaryMemo);
            }).ConfigureAwait(false);
        }
コード例 #4
0
        protected virtual async Task <Schema> ReadSchemaAsync()
        {
            if (HasReadSchema)
            {
                return(Schema);
            }

            byte[] buff = null;

            try
            {
                // Figure out length of schema

                buff = Buffers.Rent(4);
                await BaseStream.ReadAsync(buff, 0, 4);

                var schemaMessageLength = BitConverter.ToInt32(buff, 0);
                Buffers.Return(buff);

                // Allocate byte array for schema flat buffer

                buff = Buffers.Rent(schemaMessageLength);
                var schemabb = new FlatBuffers.ByteBuffer(buff);

                // Read in schema

                await BaseStream.ReadAsync(buff, 0, schemaMessageLength);

                Schema = MessageSerializer.GetSchema(ReadMessage <Flatbuf.Schema>(schemabb));

                return(Schema);
            }
            finally
            {
                if (buff != null)
                {
                    Buffers.Return(buff);
                }
            }
        }
コード例 #5
0
        private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field)
        {
            switch (field.TypeType)
            {
            case Flatbuf.Type.Int:
                var intMetaData = field.Type <Flatbuf.Int>().Value;
                return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned));

            case Flatbuf.Type.FloatingPoint:
                var floatingPointTypeMetadta = field.Type <Flatbuf.FloatingPoint>().Value;
                switch (floatingPointTypeMetadta.Precision)
                {
                case Flatbuf.Precision.SINGLE:
                    return(Types.FloatType.Default);

                case Flatbuf.Precision.DOUBLE:
                    return(Types.DoubleType.Default);

                case Flatbuf.Precision.HALF:
                    return(Types.HalfFloatType.Default);

                default:
                    throw new InvalidDataException("Unsupported floating point precision");
                }

            case Flatbuf.Type.Bool:
                return(new Types.BooleanType());

            case Flatbuf.Type.Decimal:
                var decMeta = field.Type <Flatbuf.Decimal>().Value;
                return(new Types.DecimalType(decMeta.Precision, decMeta.Scale));

            case Flatbuf.Type.Date:
                var dateMeta = field.Type <Flatbuf.Date>().Value;
                switch (dateMeta.Unit)
                {
                case Flatbuf.DateUnit.DAY:
                    return(Types.Date32Type.Default);

                case Flatbuf.DateUnit.MILLISECOND:
                    return(Types.Date64Type.Default);

                default:
                    throw new InvalidDataException("Unsupported date unit");
                }

            case Flatbuf.Type.Time:
                var timeMeta = field.Type <Flatbuf.Time>().Value;
                switch (timeMeta.BitWidth)
                {
                case 32:
                    return(new Types.Time32Type(timeMeta.Unit.ToArrow()));

                case 64:
                    return(new Types.Time64Type(timeMeta.Unit.ToArrow()));

                default:
                    throw new InvalidDataException("Unsupported time bit width");
                }

            case Flatbuf.Type.Timestamp:
                var timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value;
                var unit     = timestampTypeMetadata.Unit.ToArrow();
                var timezone = timestampTypeMetadata.Timezone;
                return(new Types.TimestampType(unit, timezone));

            case Flatbuf.Type.Interval:
                var intervalMetadata = field.Type <Flatbuf.Interval>().Value;
                return(new Types.IntervalType(intervalMetadata.Unit.ToArrow()));

            case Flatbuf.Type.Utf8:
                return(new Types.StringType());

            case Flatbuf.Type.Binary:
                return(Types.BinaryType.Default);

            default:
                throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
            }
        }
コード例 #6
0
        private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, Field[] childFields = null)
        {
            switch (field.TypeType)
            {
            case Flatbuf.Type.Int:
                Flatbuf.Int intMetaData = field.Type <Flatbuf.Int>().Value;
                return(MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned));

            case Flatbuf.Type.FloatingPoint:
                Flatbuf.FloatingPoint floatingPointTypeMetadata = field.Type <Flatbuf.FloatingPoint>().Value;
                switch (floatingPointTypeMetadata.Precision)
                {
                case Flatbuf.Precision.SINGLE:
                    return(Types.FloatType.Default);

                case Flatbuf.Precision.DOUBLE:
                    return(Types.DoubleType.Default);

                case Flatbuf.Precision.HALF:
                    return(Types.HalfFloatType.Default);

                default:
                    throw new InvalidDataException("Unsupported floating point precision");
                }

            case Flatbuf.Type.Bool:
                return(new Types.BooleanType());

            case Flatbuf.Type.Decimal:
                Flatbuf.Decimal decMeta = field.Type <Flatbuf.Decimal>().Value;
                return(new Types.DecimalType(decMeta.Precision, decMeta.Scale));

            case Flatbuf.Type.Date:
                Flatbuf.Date dateMeta = field.Type <Flatbuf.Date>().Value;
                switch (dateMeta.Unit)
                {
                case Flatbuf.DateUnit.DAY:
                    return(Types.Date32Type.Default);

                case Flatbuf.DateUnit.MILLISECOND:
                    return(Types.Date64Type.Default);

                default:
                    throw new InvalidDataException("Unsupported date unit");
                }

            case Flatbuf.Type.Time:
                Flatbuf.Time timeMeta = field.Type <Flatbuf.Time>().Value;
                switch (timeMeta.BitWidth)
                {
                case 32:
                    return(new Types.Time32Type(timeMeta.Unit.ToArrow()));

                case 64:
                    return(new Types.Time64Type(timeMeta.Unit.ToArrow()));

                default:
                    throw new InvalidDataException("Unsupported time bit width");
                }

            case Flatbuf.Type.Timestamp:
                Flatbuf.Timestamp timestampTypeMetadata = field.Type <Flatbuf.Timestamp>().Value;
                Types.TimeUnit    unit     = timestampTypeMetadata.Unit.ToArrow();
                string            timezone = timestampTypeMetadata.Timezone;
                return(new Types.TimestampType(unit, timezone));

            case Flatbuf.Type.Interval:
                Flatbuf.Interval intervalMetadata = field.Type <Flatbuf.Interval>().Value;
                return(new Types.IntervalType(intervalMetadata.Unit.ToArrow()));

            case Flatbuf.Type.Utf8:
                return(new Types.StringType());

            case Flatbuf.Type.Binary:
                return(Types.BinaryType.Default);

            case Flatbuf.Type.List:
                if (childFields == null || childFields.Length != 1)
                {
                    throw new InvalidDataException($"List type must have exactly one child.");
                }
                return(new Types.ListType(childFields[0]));

            case Flatbuf.Type.Struct_:
                Debug.Assert(childFields != null);
                return(new Types.StructType(childFields));

            default:
                throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
            }
        }