Пример #1
0
        internal object Read(Reader reader, Header header, AbstractCodec codec, Resolver resolver)
        {
            if (reader.IsReadToEnd())
            {
                return(string.Empty);
            }


            var result = new List <object>();

            do
            {
                long itemsCount = reader.ReadLong();
                var  data       = reader.ReadDataBlock(header.SyncData, codec);

                reader = new Reader(new MemoryStream(data));

                if (itemsCount > 1)
                {
                    for (int i = 0; i < itemsCount; i++)
                    {
                        result.Add(resolver.Resolve(reader));
                    }
                }
                else
                {
                    return(resolver.Resolve(reader));
                }
            } while (!reader.IsReadToEnd());


            return(result);
        }
Пример #2
0
        internal object Read(IReader reader, Header header, AbstractCodec codec, Resolver resolver)
        {
            var remainingBlocks = reader.ReadLong();
            var blockSize       = reader.ReadLong();
            var syncBuffer      = new byte[DataFileConstants.SyncSize];

            var dataBlock = new byte[blockSize];

            reader.ReadFixed(dataBlock, 0, (int)blockSize);
            reader.ReadFixed(syncBuffer);

            if (!syncBuffer.SequenceEqual(header.SyncData))
            {
                throw new AvroRuntimeException("Invalid sync!");
            }

            dataBlock = codec.Decompress(dataBlock);
            reader    = new Reader(new MemoryStream(dataBlock));

            if (remainingBlocks > 1)
            {
                var result = new List <object>();

                for (int i = 0; i < remainingBlocks; i++)
                {
                    result.Add(resolver.Resolve(reader));
                }

                return(result);
            }
            else
            {
                return(resolver.Resolve(reader));
            }
        }
Пример #3
0
        internal void WriteHeader(string schema, CodecType codecType)
        {
            GenerateSyncData();
            _header.AddMetadata(DataFileConstants.CodecMetadataKey, AbstractCodec.CreateCodec(codecType).Name);
            _header.AddMetadata(DataFileConstants.SchemaMetadataKey, schema);

            _writer.WriteHeader(_header);
        }
        internal byte[] ReadDataBlock(byte[] syncData, AbstractCodec codec)
        {
            var dataBlock = ReadRawBlock();

            ReadAndValidateSync(syncData);
            dataBlock = codec.Decompress(dataBlock);

            return(dataBlock);
        }
Пример #5
0
        internal object Decode(Stream stream, Schema.Schema schema)
        {
            var reader = new Reader(stream);
            var header = new Header();

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

            try
            {
                reader.ReadFixed(firstBytes);
            }
            catch (EndOfStreamException)
            {
                //stream shorter than AvroHeader
            }

            //does not contain header
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                if (schema == null)
                {
                    throw new MissingSchemaException("Provide valid schema for the Avro data");
                }
                var resolver = new Resolver(schema);
                stream.Seek(0, SeekOrigin.Begin);
                return(resolver.Resolve(reader));
            }
            else
            {
                // read meta data
                long len = reader.ReadMapStart();
                if (len > 0)
                {
                    do
                    {
                        for (long i = 0; i < len; i++)
                        {
                            string key = reader.ReadString();
                            byte[] val = reader.ReadBytes();
                            header.AddMetadata(key, val);
                        }
                    } while ((len = reader.ReadMapNext()) != 0);
                }

                schema = schema ?? Schema.Schema.Parse(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                var resolver = new Resolver(schema);

                // read in sync data
                reader.ReadFixed(header.SyncData);
                var codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));

                return(Read(reader, header, codec, resolver));
            }
        }
Пример #6
0
        internal AvroObjectContent ExtractAvroObjectContent(byte[] avroObject)
        {
            using (var stream = new MemoryStream(avroObject))
            {
                var reader = new Reader(stream);

                // validate header
                byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

                try
                {
                    reader.ReadFixed(firstBytes);
                }
                catch (EndOfStreamException)
                {
                    //stream shorter than AvroHeader
                }

                //does not contain header
                if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
                {
                    throw new InvalidAvroObjectException("Object does not contain Avro Header");
                }
                else
                {
                    AvroObjectContent result = new AvroObjectContent();
                    var header = reader.ReadHeader();
                    result.Codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));

                    reader.ReadFixed(header.SyncData);

                    result.Header        = header;
                    result.Header.Schema = Schema.Create(result.Header.GetMetadata(DataFileConstants.SchemaMetadataKey));

                    if (reader.IsReadToEnd())
                    {
                        return(result);
                    }

                    do
                    {
                        var blockContent = new DataBlock
                        {
                            ItemsCount = reader.ReadLong(),
                            Data       = reader.ReadDataBlock(header.SyncData, result.Codec)
                        };

                        result.DataBlocks.Add(blockContent);
                    } while (!reader.IsReadToEnd());

                    return(result);
                }
            }
        }
Пример #7
0
        public void CreateCodecFromString_NonExistingString_DefaultCodecIsReturned()
        {
            //Arrange


            //Act
            var result = AbstractCodec.CreateCodecFromString("NonExistingCodec");


            //Assert
            Assert.IsType <NullCodec>(result);
        }
Пример #8
0
        internal T Decode <T>(Stream stream, Schema.Schema readSchema)
        {
            var reader = new Reader(stream);
            var header = new Header();

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

            try
            {
                reader.ReadFixed(firstBytes);
            }
            catch (EndOfStreamException)
            {
                //stream shorter than AvroHeader
            }

            //does not contain header
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                throw new InvalidAvroObjectException("Object does not contain Avro Header");
            }
            else
            {
                // read meta data
                long len = reader.ReadMapStart();
                if (len > 0)
                {
                    do
                    {
                        for (long i = 0; i < len; i++)
                        {
                            string key = reader.ReadString();
                            byte[] val = reader.ReadBytes();
                            header.AddMetadata(key, val);
                        }
                    } while ((len = reader.ReadMapNext()) != 0);
                }

                readSchema = readSchema ?? Schema.Schema.Parse(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                Schema.Schema writeSchema = Schema.Schema.Parse(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                var           resolver    = new Resolver(writeSchema, readSchema);

                // read in sync data
                reader.ReadFixed(header.SyncData);
                var codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));


                return(Read <T>(reader, header, codec, resolver));
            }
        }
Пример #9
0
        internal BaseLineReader(Reader reader, byte[] syncDate, AbstractCodec codec, TypeSchema writeSchema, TypeSchema readSchema)
        {
            _reader      = reader;
            _syncDate    = syncDate;
            _codec       = codec;
            _writeSchema = writeSchema;
            _readSchema  = readSchema;

            if (_reader.IsReadToEnd())
            {
                return;
            }

            LoadNextDataBlock();
        }
Пример #10
0
        internal MergeEncoder(Stream outStream)
        {
            _codec  = new NullCodec();
            _stream = outStream;

            _syncInterval = DataFileConstants.DefaultSyncInterval;

            _blockCount = 0;
            _writer     = new Writer(_stream);

            _tempBuffer = new MemoryStream();
            _tempWriter = new Writer(_tempBuffer);

            _isOpen = true;
            _header = new Header();
        }
Пример #11
0
        private Decoder(Stream stream)
        {
            _stream     = stream;
            _header     = new Header();
            _reader     = new Reader(stream);
            _syncBuffer = new byte[DataFileConstants.SyncSize];

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];
            try
            {
                _reader.ReadFixed(firstBytes);
            }
            catch (Exception)
            {
                throw new InvalidAvroObjectException("Cannot read length of Avro Header");
            }
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                throw new InvalidAvroObjectException("Cannot read Avro Header");
            }

            // read meta data
            long len = _reader.ReadMapStart();

            if (len > 0)
            {
                do
                {
                    for (long i = 0; i < len; i++)
                    {
                        string key = _reader.ReadString();
                        byte[] val = _reader.ReadBytes();
                        _header.MetaData.Add(key, val);
                    }
                } while ((len = _reader.ReadMapNext()) != 0);
            }

            // read in sync data
            _reader.ReadFixed(_header.SyncData);

            // parse schema and set codec
            _header.Schema = Schema.Schema.Parse(GetMetaString(DataFileConstants.SchemaMetadataKey));
            _resolver      = new Resolver(_header.Schema, _readerSchema ?? _header.Schema);
            _codec         = AbstractCodec.CreateCodecFromString(GetMetaString(DataFileConstants.CodecMetadataKey));
        }
Пример #12
0
        internal Encoder(Schema.Schema schema, Stream outStream, CodecType codecType)
        {
            _codec        = AbstractCodec.CreateCodec(codecType);
            _stream       = outStream;
            _metadata     = new Metadata();
            _schema       = schema;
            _syncInterval = DataFileConstants.DefaultSyncInterval;

            _blockCount   = 0;
            _encoder      = new Writer(_stream);
            _blockStream  = new MemoryStream();
            _blockEncoder = new Writer(_blockStream);

            _writer = Resolver.ResolveWriter(schema);

            _isOpen = true;
        }
Пример #13
0
        internal static ILineReader <T> OpenReader <T>(Stream stream, TypeSchema readSchema)
        {
            var reader = new Reader(stream);

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

            try
            {
                reader.ReadFixed(firstBytes);
            }
            catch (EndOfStreamException)
            {
                //stream shorter than AvroHeader
            }

            //headless
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                if (readSchema == null)
                {
                    throw new MissingSchemaException("Provide valid schema for the Avro data");
                }
                var resolver = new Resolver(readSchema, readSchema);
                stream.Seek(0, SeekOrigin.Begin);
                return(new ListLineReader <T>(reader, resolver));
            }
            else
            {
                var header = reader.ReadHeader();

                readSchema = readSchema ?? Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                TypeSchema writeSchema = Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey));

                // read in sync data
                reader.ReadFixed(header.SyncData);
                var codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));


                return(new BaseLineReader <T>(reader, header.SyncData, codec, writeSchema, readSchema));
            }
        }
Пример #14
0
        internal T Read <T>(IReader reader, Header header, AbstractCodec codec, Resolver resolver)
        {
            var remainingBlocks = reader.ReadLong();
            var blockSize       = reader.ReadLong();
            var syncBuffer      = new byte[DataFileConstants.SyncSize];

            var dataBlock = new byte[blockSize];

            reader.ReadFixed(dataBlock, 0, (int)blockSize);
            reader.ReadFixed(syncBuffer);

            if (!syncBuffer.SequenceEqual(header.SyncData))
            {
                throw new AvroRuntimeException("Invalid sync!");
            }

            dataBlock = codec.Decompress(dataBlock);
            reader    = new Reader(new MemoryStream(dataBlock));

            return(resolver.Resolve <T>(reader, remainingBlocks));
        }
Пример #15
0
        internal Encoder(TypeSchema schema, Stream outStream, CodecType codecType)
        {
            _codec        = AbstractCodec.CreateCodec(codecType);
            _stream       = outStream;
            _header       = new Header();
            _schema       = schema;
            _syncInterval = DataFileConstants.DefaultSyncInterval;

            _blockCount = 0;
            _writer     = new Writer(_stream);
            _tempBuffer = new MemoryStream();
            _tempWriter = new Writer(_tempBuffer);

            GenerateSyncData();
            _header.AddMetadata(DataFileConstants.CodecMetadataKey, _codec.Name);
            _header.AddMetadata(DataFileConstants.SchemaMetadataKey, _schema.ToString());

            _writeItem = Resolver.ResolveWriter(schema);

            _isOpen = true;
        }
Пример #16
0
        internal object Decode(Stream stream, TypeSchema schema)
        {
            var reader = new Reader(stream);

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

            try
            {
                reader.ReadFixed(firstBytes);
            }
            catch (EndOfStreamException)
            {
                //stream shorter than AvroHeader
            }

            //does not contain header
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                if (schema == null)
                {
                    throw new MissingSchemaException("Provide valid schema for the Avro data");
                }
                var resolver = new Resolver(schema);
                stream.Seek(0, SeekOrigin.Begin);
                return(resolver.Resolve(reader));
            }
            else
            {
                var header = reader.ReadHeader();

                schema = schema ?? Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                var resolver = new Resolver(schema);

                reader.ReadFixed(header.SyncData);
                var codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));

                return(Read(reader, header, codec, resolver));
            }
        }
Пример #17
0
        internal T Decode <T>(Stream stream, TypeSchema readSchema)
        {
            var reader = new Reader(stream);

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

            try
            {
                reader.ReadFixed(firstBytes);
            }
            catch (EndOfStreamException)
            {
                //stream shorter than AvroHeader
            }

            //does not contain header
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                throw new InvalidAvroObjectException("Object does not contain Avro Header");
            }
            else
            {
                var header = reader.ReadHeader();

                TypeSchema writeSchema = Schema.Create(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                readSchema = readSchema ?? writeSchema;
                var resolver = new Resolver(writeSchema, readSchema);

                // read in sync data
                reader.ReadFixed(header.SyncData);
                var codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));


                return(Read <T>(reader, header, codec, resolver));
            }
        }
Пример #18
0
 internal T Read <T>(Reader reader, Header header, AbstractCodec codec, Resolver resolver)
 {
     if (reader.IsReadToEnd())
     {
         return(default);
Пример #19
0
        internal static ILineReader <T> OpenReader <T>(Stream stream, Schema.Schema readSchema)
        {
            var reader = new Reader(stream);
            var header = new Header();

            // validate header
            byte[] firstBytes = new byte[DataFileConstants.AvroHeader.Length];

            try
            {
                reader.ReadFixed(firstBytes);
            }
            catch (EndOfStreamException)
            {
                //stream shorter than AvroHeader
            }

            //does not contain header
            if (!firstBytes.SequenceEqual(DataFileConstants.AvroHeader))
            {
                if (readSchema == null)
                {
                    throw new MissingSchemaException("Provide valid schema for the Avro data");
                }
                var resolver = new Resolver(readSchema, readSchema);
                stream.Seek(0, SeekOrigin.Begin);
                return(new ListLineReader <T>(reader, resolver));
            }
            else
            {
                // read meta data
                long len = reader.ReadMapStart();
                if (len > 0)
                {
                    do
                    {
                        for (long i = 0; i < len; i++)
                        {
                            string key = reader.ReadString();
                            byte[] val = reader.ReadBytes();
                            header.AddMetadata(key, val);
                        }
                    } while ((len = reader.ReadMapNext()) != 0);
                }

                readSchema = readSchema ?? Schema.Schema.Parse(header.GetMetadata(DataFileConstants.SchemaMetadataKey));
                Schema.Schema writeSchema = Schema.Schema.Parse(header.GetMetadata(DataFileConstants.SchemaMetadataKey));

                var resolver = new Resolver(writeSchema, readSchema);

                // read in sync data
                reader.ReadFixed(header.SyncData);
                var codec = AbstractCodec.CreateCodecFromString(header.GetMetadata(DataFileConstants.CodecMetadataKey));


                var remainingBlocks = reader.ReadLong();
                var blockSize       = reader.ReadLong();
                var syncBuffer      = new byte[DataFileConstants.SyncSize];

                var dataBlock = new byte[blockSize];

                reader.ReadFixed(dataBlock, 0, (int)blockSize);
                reader.ReadFixed(syncBuffer);

                if (!syncBuffer.SequenceEqual(header.SyncData))
                {
                    throw new AvroRuntimeException("Invalid sync!");
                }

                dataBlock = codec.Decompress(dataBlock);
                reader    = new Reader(new MemoryStream(dataBlock));


                if (remainingBlocks > 1)
                {
                    return(new BlockLineReader <T>(reader, resolver, remainingBlocks));
                }

                if (writeSchema.Tag == Schema.Schema.Type.Array)
                {
                    return(new ListLineReader <T>(reader, new Resolver(((ArraySchema)writeSchema).ItemSchema, readSchema)));
                }

                return(new ListLineReader <T>(reader, new Resolver(writeSchema, readSchema)));
            }
        }