public virtual void writeCompressed(ExtendedDataOutput output)
{
int dataType = (int)this.getDataType();
int unitLength = getUnitLength(this.getDataType());
int elementCount = this.rows();
int maxCompressedLength = this.rows() * sizeof(long) * 8 * 2 + 64 * 3;
ByteBuffer outBuffer = ByteBuffer.Allocate(Math.Max(maxCompressedLength, 65536));
outBuffer.order(output.GetType() == typeof(LittleEndianDataOutputStream));
short flag = (short)((short)DATA_FORM.DF_VECTOR << 8 | (short)DATA_TYPE.DT_COMPRESS & 0xff);
outBuffer.WriteShort(flag);
outBuffer.WriteInt(0); // compressedBytes
outBuffer.WriteInt(1); // cols
outBuffer.WriteByte((byte)0); // version
outBuffer.WriteByte((byte)1); // flag bit0:littleEndian bit1:containChecksum
outBuffer.WriteByte(unchecked ((byte)-1)); // charcode
outBuffer.WriteByte((byte)compressedMethod);
outBuffer.WriteByte((byte)dataType);
outBuffer.WriteByte((byte)unitLength);
outBuffer.WriteByte((byte)0);
outBuffer.WriteByte((byte)0);
outBuffer.WriteInt(-1); //extra
outBuffer.WriteInt(elementCount);
outBuffer.WriteInt(-1); //TODO: checkSum
EncoderFactory.Get(compressedMethod).compress(this, elementCount, unitLength, maxCompressedLength, outBuffer);
int compressedLength = outBuffer.ReadableBytes - 10;
outBuffer.PutInt(compressedLength, 2);
byte[] tmp = new byte[outBuffer.ReadableBytes];
output.write(outBuffer.ToArray());
}
/// <exception cref="System.IO.IOException"/>
internal EventWriter(FSDataOutputStream @out)
{
this.@out = @out;
@out.WriteBytes(Version);
@out.WriteBytes("\n");
@out.WriteBytes(Event.Schema$.ToString());
@out.WriteBytes("\n");
this.encoder = EncoderFactory.Get().JsonEncoder(Event.Schema$, @out);
}
/// <exception cref="System.Exception"/>
public static void TestReflect(object value, Type type, string schema)
{
// check that schema matches expected
Schema s = ((ReflectData)ReflectData.Get()).GetSchema(type);
Assert.Equal(Schema.Parse(schema), s);
// check that value is serialized correctly
ReflectDatumWriter <object> writer = new ReflectDatumWriter <object>(s);
ByteArrayOutputStream @out = new ByteArrayOutputStream();
writer.Write(value, EncoderFactory.Get().DirectBinaryEncoder(@out, null));
ReflectDatumReader <object> reader = new ReflectDatumReader <object>(s);
object after = reader.Read(null, DecoderFactory.Get().BinaryDecoder(@out.ToByteArray
(), null));
Assert.Equal(value, after);
}
/// <exception cref="System.IO.IOException"/>
public AvroFileInputStream(FileStatus status)
{
pos = 0;
buffer = new byte[0];
GenericDatumReader <object> reader = new GenericDatumReader <object>();
FileContext fc = FileContext.GetFileContext(new Configuration());
fileReader = DataFileReader.OpenReader(new AvroFSInput(fc, status.GetPath()), reader
);
Schema schema = fileReader.GetSchema();
writer = new GenericDatumWriter <object>(schema);
output = new ByteArrayOutputStream();
JsonGenerator generator = new JsonFactory().CreateJsonGenerator(output, JsonEncoding
.Utf8);
MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
prettyPrinter.SetRootValueSeparator(Runtime.GetProperty("line.separator"));
generator.SetPrettyPrinter(prettyPrinter);
encoder = EncoderFactory.Get().JsonEncoder(schema, generator);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Open(OutputStream @out)
{
this.outStream = @out;
this.encoder = EncoderFactory.Get().BinaryEncoder(@out, this.encoder);
}