예제 #1
0
        public virtual void writeCompressed(ExtendedDataOutput output)
        {
            int dataType            = (int)this.getDataType();
            int unitLength          = getUnitLength(this.getDataType());
            int elementCount        = this.rows();
            int maxCompressedLength = this.rows() * sizeof(long) * 8 * 2 + 64 * 3;

            ByteBuffer outBuffer = ByteBuffer.Allocate(Math.Max(maxCompressedLength, 65536));

            outBuffer.order(output.GetType() == typeof(LittleEndianDataOutputStream));
            short flag = (short)((short)DATA_FORM.DF_VECTOR << 8 | (short)DATA_TYPE.DT_COMPRESS & 0xff);

            outBuffer.WriteShort(flag);
            outBuffer.WriteInt(0);                     // compressedBytes
            outBuffer.WriteInt(1);                     // cols
            outBuffer.WriteByte((byte)0);              // version
            outBuffer.WriteByte((byte)1);              // flag bit0:littleEndian bit1:containChecksum
            outBuffer.WriteByte(unchecked ((byte)-1)); // charcode
            outBuffer.WriteByte((byte)compressedMethod);
            outBuffer.WriteByte((byte)dataType);
            outBuffer.WriteByte((byte)unitLength);
            outBuffer.WriteByte((byte)0);
            outBuffer.WriteByte((byte)0);
            outBuffer.WriteInt(-1); //extra
            outBuffer.WriteInt(elementCount);
            outBuffer.WriteInt(-1); //TODO: checkSum
            EncoderFactory.Get(compressedMethod).compress(this, elementCount, unitLength, maxCompressedLength, outBuffer);
            int compressedLength = outBuffer.ReadableBytes - 10;

            outBuffer.PutInt(compressedLength, 2);
            byte[] tmp = new byte[outBuffer.ReadableBytes];
            output.write(outBuffer.ToArray());
        }
예제 #2
0
 /// <exception cref="System.IO.IOException"/>
 internal EventWriter(FSDataOutputStream @out)
 {
     this.@out = @out;
     @out.WriteBytes(Version);
     @out.WriteBytes("\n");
     @out.WriteBytes(Event.Schema$.ToString());
     @out.WriteBytes("\n");
     this.encoder = EncoderFactory.Get().JsonEncoder(Event.Schema$, @out);
 }
예제 #3
0
        /// <exception cref="System.Exception"/>
        public static void TestReflect(object value, Type type, string schema)
        {
            // check that schema matches expected
            Schema s = ((ReflectData)ReflectData.Get()).GetSchema(type);

            Assert.Equal(Schema.Parse(schema), s);
            // check that value is serialized correctly
            ReflectDatumWriter <object> writer = new ReflectDatumWriter <object>(s);
            ByteArrayOutputStream       @out   = new ByteArrayOutputStream();

            writer.Write(value, EncoderFactory.Get().DirectBinaryEncoder(@out, null));
            ReflectDatumReader <object> reader = new ReflectDatumReader <object>(s);
            object after = reader.Read(null, DecoderFactory.Get().BinaryDecoder(@out.ToByteArray
                                                                                    (), null));

            Assert.Equal(value, after);
        }
예제 #4
0
            /// <exception cref="System.IO.IOException"/>
            public AvroFileInputStream(FileStatus status)
            {
                pos    = 0;
                buffer = new byte[0];
                GenericDatumReader <object> reader = new GenericDatumReader <object>();
                FileContext fc = FileContext.GetFileContext(new Configuration());

                fileReader = DataFileReader.OpenReader(new AvroFSInput(fc, status.GetPath()), reader
                                                       );
                Schema schema = fileReader.GetSchema();

                writer = new GenericDatumWriter <object>(schema);
                output = new ByteArrayOutputStream();
                JsonGenerator generator = new JsonFactory().CreateJsonGenerator(output, JsonEncoding
                                                                                .Utf8);
                MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();

                prettyPrinter.SetRootValueSeparator(Runtime.GetProperty("line.separator"));
                generator.SetPrettyPrinter(prettyPrinter);
                encoder = EncoderFactory.Get().JsonEncoder(schema, generator);
            }
예제 #5
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Open(OutputStream @out)
 {
     this.outStream = @out;
     this.encoder   = EncoderFactory.Get().BinaryEncoder(@out, this.encoder);
 }