/// <summary>Serialize the shuffle port into a ByteBuffer for use later on.</summary>
/// <param name="port">the port to be sent to the ApplciationMaster</param>
/// <returns>the serialized form of the port.</returns>
/// <exception cref="System.IO.IOException"/>
public static ByteBuffer SerializeMetaData(int port)
{
//TODO these bytes should be versioned
DataOutputBuffer port_dob = new DataOutputBuffer();
port_dob.WriteInt(port);
return ByteBuffer.Wrap(port_dob.GetData(), 0, port_dob.GetLength());
}
/// <summary>Write infLen bytes (deflated) to file in test dir using codec.</summary>
/// <remarks>
/// Write infLen bytes (deflated) to file in test dir using codec.
/// Records are of the form
/// <i><b64 rand><i+i><b64 rand>
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private static Path WriteSplitTestFile(FileSystem fs, Random rand, CompressionCodec
codec, long infLen)
{
int RecSize = 1024;
Path wd = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
(fs), codec.GetType().Name);
Path file = new Path(wd, "test" + codec.GetDefaultExtension());
byte[] b = new byte[RecSize];
Base64 b64 = new Base64(0, null);
DataOutputStream fout = null;
Compressor cmp = CodecPool.GetCompressor(codec);
try
{
fout = new DataOutputStream(codec.CreateOutputStream(fs.Create(file, true), cmp));
DataOutputBuffer dob = new DataOutputBuffer(RecSize * 4 / 3 + 4);
int seq = 0;
while (infLen > 0)
{
rand.NextBytes(b);
byte[] b64enc = b64.Encode(b);
// ensures rand printable, no LF
dob.Reset();
dob.WriteInt(seq);
System.Array.Copy(dob.GetData(), 0, b64enc, 0, dob.GetLength());
fout.Write(b64enc);
fout.Write('\n');
++seq;
infLen -= b64enc.Length;
}
Log.Info("Wrote " + seq + " records to " + file);
}
finally
{
IOUtils.Cleanup(Log, fout);
CodecPool.ReturnCompressor(cmp);
}
return(file);
}