/// <summary>Read raw bytes from a SequenceFile.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override bool NextKeyValue()
{
lock (this)
{
if (done)
{
return(false);
}
long pos = @in.GetPosition();
bool eof = -1 == @in.NextRawKey(buffer);
if (!eof)
{
if (key == null)
{
key = new BytesWritable();
}
if (value == null)
{
value = new BytesWritable();
}
key.Set(buffer.GetData(), 0, buffer.GetLength());
buffer.Reset();
@in.NextRawValue(vbytes);
vbytes.WriteUncompressedBytes(buffer);
value.Set(buffer.GetData(), 0, buffer.GetLength());
buffer.Reset();
}
return(!(done = (eof || (pos >= end && @in.SyncSeen()))));
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override void FlushAndSync(bool durable)
{
int numReadyBytes = buf.CountReadyBytes();
if (numReadyBytes > 0)
{
int numReadyTxns = buf.CountReadyTxns();
long firstTxToFlush = buf.GetFirstReadyTxId();
System.Diagnostics.Debug.Assert(numReadyTxns > 0);
// Copy from our double-buffer into a new byte array. This is for
// two reasons:
// 1) The IPC code has no way of specifying to send only a slice of
// a larger array.
// 2) because the calls to the underlying nodes are asynchronous, we
// need a defensive copy to avoid accidentally mutating the buffer
// before it is sent.
DataOutputBuffer bufToSend = new DataOutputBuffer(numReadyBytes);
buf.FlushTo(bufToSend);
System.Diagnostics.Debug.Assert(bufToSend.GetLength() == numReadyBytes);
byte[] data = bufToSend.GetData();
System.Diagnostics.Debug.Assert(data.Length == bufToSend.GetLength());
QuorumCall <AsyncLogger, Void> qcall = loggers.SendEdits(segmentTxId, firstTxToFlush
, numReadyTxns, data);
loggers.WaitForWriteQuorum(qcall, writeTimeoutMs, "sendEdits");
// Since we successfully wrote this batch, let the loggers know. Any future
// RPCs will thus let the loggers know of the most recent transaction, even
// if a logger has fallen behind.
loggers.SetCommittedTxId(firstTxToFlush + numReadyTxns - 1);
}
}
public virtual void TestDoubleBuffer()
{
EditsDoubleBuffer buf = new EditsDoubleBuffer(1024);
NUnit.Framework.Assert.IsTrue(buf.IsFlushed());
byte[] data = new byte[100];
buf.WriteRaw(data, 0, data.Length);
NUnit.Framework.Assert.AreEqual("Should count new data correctly", data.Length, buf
.CountBufferedBytes());
NUnit.Framework.Assert.IsTrue("Writing to current buffer should not affect flush state"
, buf.IsFlushed());
// Swap the buffers
buf.SetReadyToFlush();
NUnit.Framework.Assert.AreEqual("Swapping buffers should still count buffered bytes"
, data.Length, buf.CountBufferedBytes());
NUnit.Framework.Assert.IsFalse(buf.IsFlushed());
// Flush to a stream
DataOutputBuffer outBuf = new DataOutputBuffer();
buf.FlushTo(outBuf);
NUnit.Framework.Assert.AreEqual(data.Length, outBuf.GetLength());
NUnit.Framework.Assert.IsTrue(buf.IsFlushed());
NUnit.Framework.Assert.AreEqual(0, buf.CountBufferedBytes());
// Write some more
buf.WriteRaw(data, 0, data.Length);
NUnit.Framework.Assert.AreEqual("Should count new data correctly", data.Length, buf
.CountBufferedBytes());
buf.SetReadyToFlush();
buf.FlushTo(outBuf);
NUnit.Framework.Assert.AreEqual(data.Length * 2, outBuf.GetLength());
NUnit.Framework.Assert.AreEqual(0, buf.CountBufferedBytes());
outBuf.Close();
}
public virtual void TestMany()
{
//Write a big set of data, one of each primitive type array
foreach (object x in bigSet)
{
//write each test object two ways
//First, transparently via ObjectWritable
ObjectWritable.WriteObject(@out, x, x.GetType(), null, true);
//Second, explicitly via ArrayPrimitiveWritable
(new ArrayPrimitiveWritable(x)).Write(@out);
}
//Now read the data back in
@in.Reset(@out.GetData(), @out.GetLength());
for (int x_1 = 0; x_1 < resultSet.Length;)
{
//First, transparently
resultSet[x_1++] = ObjectWritable.ReadObject(@in, null);
//Second, explicitly
ArrayPrimitiveWritable apw = new ArrayPrimitiveWritable();
apw.ReadFields(@in);
resultSet[x_1++] = apw.Get();
}
//validate data structures and values
Assert.Equal(expectedResultSet.Length, resultSet.Length);
for (int x_2 = 0; x_2 < resultSet.Length; x_2++)
{
Assert.Equal("ComponentType of array " + x_2, expectedResultSet
[x_2].GetType().GetElementType(), resultSet[x_2].GetType().GetElementType());
}
Assert.True("In and Out arrays didn't match values", Arrays.DeepEquals
(expectedResultSet, resultSet));
}
public virtual void TestGzipCompatibility()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Info("seed: " + seed);
DataOutputBuffer dflbuf = new DataOutputBuffer();
GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
byte[] b = new byte[r.Next(128 * 1024 + 1)];
r.NextBytes(b);
gzout.Write(b);
gzout.Close();
DataInputBuffer gzbuf = new DataInputBuffer();
gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength());
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf);
Decompressor decom = codec.CreateDecompressor();
NUnit.Framework.Assert.IsNotNull(decom);
Assert.Equal(typeof(BuiltInGzipDecompressor), decom.GetType());
InputStream gzin = codec.CreateInputStream(gzbuf, decom);
dflbuf.Reset();
IOUtils.CopyBytes(gzin, dflbuf, 4096);
byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength());
Assert.AssertArrayEquals(b, dflchk);
}
/// <exception cref="System.IO.IOException"/>
public override void Write(FSEditLogOp op)
{
writer.WriteOp(op);
if (bufCurrent.GetLength() > transmissionThreshold)
{
Transmit();
}
}
/// <summary>
/// Generate a string with the url-quoted base64 encoded serialized form
/// of the Writable.
/// </summary>
/// <param name="obj">the object to serialize</param>
/// <returns>the encoded string</returns>
/// <exception cref="System.IO.IOException"/>
private static string EncodeWritable(IWritable obj)
{
DataOutputBuffer buf = new DataOutputBuffer();
obj.Write(buf);
Base64 encoder = new Base64(0, null, true);
byte[] raw = new byte[buf.GetLength()];
System.Array.Copy(buf.GetData(), 0, raw, 0, buf.GetLength());
return(encoder.EncodeToString(raw));
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void PopulateHeaders(IList<string> mapIds, string outputBaseStr
, string user, int reduce, HttpRequest request, HttpResponse response, bool keepAliveParam
, IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap)
{
long contentLength = 0;
foreach (string mapId in mapIds)
{
string @base = outputBaseStr + mapId;
ShuffleHandler.Shuffle.MapOutputInfo outputInfo = this.GetMapOutputInfo(@base, mapId
, reduce, user);
if (mapOutputInfoMap.Count < this._enclosing.mapOutputMetaInfoCacheSize)
{
mapOutputInfoMap[mapId] = outputInfo;
}
// Index file
Path indexFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out.index",
this.conf);
IndexRecord info = this.indexCache.GetIndexInformation(mapId, reduce, indexFileName
, user);
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
contentLength += info.partLength;
contentLength += dob.GetLength();
}
// Now set the response headers.
this.SetResponseHeaders(response, keepAliveParam, contentLength);
}
/// <exception cref="System.IO.IOException"/>
private void SetupTokens(ContainerLaunchContext container, ContainerId containerID
)
{
IDictionary <string, string> environment = container.GetEnvironment();
environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase
();
// Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId();
environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId
].GetSubmitTime().ToString();
environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId
].GetMaxAppAttempts().ToString();
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
if (container.GetTokens() != null)
{
// TODO: Don't do this kind of checks everywhere.
dibb.Reset(container.GetTokens());
credentials.ReadTokenStorageStream(dibb);
}
// Add AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken
();
if (amrmToken != null)
{
credentials.AddToken(amrmToken.GetService(), amrmToken);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()));
}
/// <exception cref="System.Exception"/>
public virtual void TestLocalizerStatusSerDe()
{
LocalizerStatus rsrcS = CreateLocalizerStatus();
NUnit.Framework.Assert.IsTrue(rsrcS is LocalizerStatusPBImpl);
LocalizerStatusPBImpl rsrcPb = (LocalizerStatusPBImpl)rsrcS;
DataOutputBuffer @out = new DataOutputBuffer();
rsrcPb.GetProto().WriteDelimitedTo(@out);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), 0, @out.GetLength());
YarnServerNodemanagerServiceProtos.LocalizerStatusProto rsrcPbD = YarnServerNodemanagerServiceProtos.LocalizerStatusProto
.ParseDelimitedFrom(@in);
NUnit.Framework.Assert.IsNotNull(rsrcPbD);
LocalizerStatus rsrcD = new LocalizerStatusPBImpl(rsrcPbD);
NUnit.Framework.Assert.AreEqual(rsrcS, rsrcD);
NUnit.Framework.Assert.AreEqual("localizer0", rsrcS.GetLocalizerId());
NUnit.Framework.Assert.AreEqual("localizer0", rsrcD.GetLocalizerId());
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcS.GetResourceStatus
(0));
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcD.GetResourceStatus
(0));
}
private static ByteBuffer ConvertCredentialsToByteBuffer(Credentials credentials)
{
ByteBuffer appAttemptTokens = null;
DataOutputBuffer dob = new DataOutputBuffer();
try
{
if (credentials != null)
{
credentials.WriteTokenStorageToStream(dob);
appAttemptTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
}
return(appAttemptTokens);
}
catch (IOException)
{
Log.Error("Failed to convert Credentials to ByteBuffer.");
System.Diagnostics.Debug.Assert(false);
return(null);
}
finally
{
IOUtils.CloseStream(dob);
}
}
/// <summary>
/// Write a protobuf to a buffer 'numProtos' times, and then
/// read them back, making sure all data comes through correctly.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void DoTest(int numProtos)
{
Configuration conf = new Configuration();
DataOutputBuffer @out = new DataOutputBuffer();
// Write numProtos protobufs to the buffer
Message[] sent = new Message[numProtos];
for (int i = 0; i < numProtos; i++)
{
// Construct a test protocol buffer using one of the
// protos that ships with the protobuf library
Message testProto = ((DescriptorProtos.EnumValueDescriptorProto)DescriptorProtos.EnumValueDescriptorProto
.NewBuilder().SetName("test" + i).SetNumber(i).Build());
ObjectWritable.WriteObject(@out, testProto, typeof(DescriptorProtos.EnumValueDescriptorProto
), conf);
sent[i] = testProto;
}
// Read back the data
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
for (int i_1 = 0; i_1 < numProtos; i_1++)
{
Message received = (Message)ObjectWritable.ReadObject(@in, conf);
Assert.Equal(sent[i_1], received);
}
}
/// <summary>Serialize the shuffle port into a ByteBuffer for use later on.</summary>
/// <param name="port">the port to be sent to the ApplciationMaster</param>
/// <returns>the serialized form of the port.</returns>
/// <exception cref="System.IO.IOException"/>
public static ByteBuffer SerializeMetaData(int port)
{
//TODO these bytes should be versioned
DataOutputBuffer port_dob = new DataOutputBuffer();
port_dob.WriteInt(port);
return ByteBuffer.Wrap(port_dob.GetData(), 0, port_dob.GetLength());
}
/// <exception cref="System.IO.IOException"/>
public virtual void DoTestLimitedIO(string str, int len)
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
@out.Reset();
try
{
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len);
Fail("expected writeString to fail when told to write a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len + 1);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
@in.Mark(len);
string after;
try
{
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len);
Fail("expected readString to fail when told to read a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
@in.Reset();
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len + 1);
Assert.True(str.Equals(after));
}
/// <exception cref="System.IO.IOException"/>
public static void TestReadInRange(long val, int lower, int upper, bool expectSuccess
)
{
DataOutputBuffer buf = new DataOutputBuffer();
DataInputBuffer inbuf = new DataInputBuffer();
WritableUtils.WriteVLong(buf, val);
try
{
inbuf.Reset(buf.GetData(), 0, buf.GetLength());
long val2 = WritableUtils.ReadVIntInRange(inbuf, lower, upper);
if (!expectSuccess)
{
Fail("expected readVIntInRange to throw an exception");
}
Assert.Equal(val, val2);
}
catch (IOException e)
{
if (expectSuccess)
{
Log.Error("unexpected exception:", e);
Fail("readVIntInRange threw an unexpected exception");
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestIO()
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
for (int i = 0; i < NumIterations; i++)
{
// generate a random string
string before;
if (i == 0)
{
before = GetLongString();
}
else
{
before = GetTestString();
}
// write it
@out.Reset();
Org.Apache.Hadoop.IO.Text.WriteString(@out, before);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
string after = Org.Apache.Hadoop.IO.Text.ReadString(@in);
Assert.True(before.Equals(after));
// Test compatibility with Java's other decoder
int strLenSize = WritableUtils.GetVIntSize(Org.Apache.Hadoop.IO.Text.Utf8Length(before
));
string after2 = Runtime.GetStringForBytes(@out.GetData(), strLenSize, @out
.GetLength() - strLenSize, "UTF-8");
Assert.True(before.Equals(after2));
}
}
/// <summary>
/// If valueClass is undefined, readFields should throw an exception indicating
/// that the field is null.
/// </summary>
/// <remarks>
/// If valueClass is undefined, readFields should throw an exception indicating
/// that the field is null. Otherwise, readFields should succeed.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestThrowUndefinedValueException()
{
// Get a buffer containing a simple text array
Text[] elements = new Text[] { new Text("zero"), new Text("one"), new Text("two") };
TestArrayWritable.TextArrayWritable sourceArray = new TestArrayWritable.TextArrayWritable
();
sourceArray.Set(elements);
// Write it to a normal output buffer
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
sourceArray.Write(@out);
// Read the output buffer with TextReadable. Since the valueClass is defined,
// this should succeed
TestArrayWritable.TextArrayWritable destArray = new TestArrayWritable.TextArrayWritable
();
@in.Reset(@out.GetData(), @out.GetLength());
destArray.ReadFields(@in);
Writable[] destElements = destArray.Get();
Assert.True(destElements.Length == elements.Length);
for (int i = 0; i < elements.Length; i++)
{
Assert.Equal(destElements[i], elements[i]);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSerializeAndDeserializeNull()
{
bool gotException = false;
try
{
new EnumSetWritable <TestEnumSetWritable.TestEnumSet>(null);
}
catch (RuntimeException)
{
gotException = true;
}
Assert.True("Instantiation of empty EnumSetWritable with no element type class "
+ "provided should throw exception", gotException);
EnumSetWritable <TestEnumSetWritable.TestEnumSet> nullFlagWritable = new EnumSetWritable
<TestEnumSetWritable.TestEnumSet>(null, typeof(TestEnumSetWritable.TestEnumSet));
DataOutputBuffer @out = new DataOutputBuffer();
ObjectWritable.WriteObject(@out, nullFlagWritable, nullFlagWritable.GetType(), null
);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
EnumSet <TestEnumSetWritable.TestEnumSet> read = ((EnumSetWritable <TestEnumSetWritable.TestEnumSet
>)ObjectWritable.ReadObject(@in, null)).Get();
Assert.Equal(read, null);
}
public virtual void TestNodeHeartBeatResponse()
{
NodeHeartbeatResponse record = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHeartbeatResponse
>();
IDictionary <ApplicationId, ByteBuffer> appCredentials = new Dictionary <ApplicationId
, ByteBuffer>();
Credentials app1Cred = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token1.SetKind(new Text("kind1"));
app1Cred.AddToken(new Text("token1"), token1);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token2.SetKind(new Text("kind2"));
app1Cred.AddToken(new Text("token2"), token2);
DataOutputBuffer dob = new DataOutputBuffer();
app1Cred.WriteTokenStorageToStream(dob);
ByteBuffer byteBuffer1 = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
appCredentials[ApplicationId.NewInstance(1234, 1)] = byteBuffer1;
record.SetSystemCredentialsForApps(appCredentials);
NodeHeartbeatResponse proto = new NodeHeartbeatResponsePBImpl(((NodeHeartbeatResponsePBImpl
)record).GetProto());
NUnit.Framework.Assert.AreEqual(appCredentials, proto.GetSystemCredentialsForApps
());
}
public virtual void TestSerialization()
{
TestDelegationToken.TestDelegationTokenIdentifier origToken = new TestDelegationToken.TestDelegationTokenIdentifier
(new Text("alice"), new Text("bob"), new Text("colin"));
TestDelegationToken.TestDelegationTokenIdentifier newToken = new TestDelegationToken.TestDelegationTokenIdentifier
();
origToken.SetIssueDate(123);
origToken.SetMasterKeyId(321);
origToken.SetMaxDate(314);
origToken.SetSequenceNumber(12345);
// clone origToken into newToken
DataInputBuffer inBuf = new DataInputBuffer();
DataOutputBuffer outBuf = new DataOutputBuffer();
origToken.Write(outBuf);
inBuf.Reset(outBuf.GetData(), 0, outBuf.GetLength());
newToken.ReadFields(inBuf);
// now test the fields
Assert.Equal("alice", newToken.GetUser().GetUserName());
Assert.Equal(new Text("bob"), newToken.GetRenewer());
Assert.Equal("colin", newToken.GetUser().GetRealUser().GetUserName
());
Assert.Equal(123, newToken.GetIssueDate());
Assert.Equal(321, newToken.GetMasterKeyId());
Assert.Equal(314, newToken.GetMaxDate());
Assert.Equal(12345, newToken.GetSequenceNumber());
Assert.Equal(origToken, newToken);
}
/// <exception cref="System.IO.IOException"/>
internal static DataInputBuffer CreateFakeCredentials(Random r, int nTok)
{
Credentials creds = new Credentials();
byte[] password = new byte[20];
Text kind = new Text();
Text service = new Text();
Text alias = new Text();
for (int i = 0; i < nTok; ++i)
{
byte[] identifier = Sharpen.Runtime.GetBytesForString(("idef" + i));
r.NextBytes(password);
kind.Set("kind" + i);
service.Set("service" + i);
alias.Set("token" + i);
Org.Apache.Hadoop.Security.Token.Token token = new Org.Apache.Hadoop.Security.Token.Token
(identifier, password, kind, service);
creds.AddToken(alias, token);
}
DataOutputBuffer buf = new DataOutputBuffer();
creds.WriteTokenStorageToStream(buf);
DataInputBuffer ret = new DataInputBuffer();
ret.Reset(buf.GetData(), 0, buf.GetLength());
return(ret);
}
/// <summary>Write the given object to the stream.</summary>
/// <remarks>
/// Write the given object to the stream. If it is a Text or BytesWritable,
/// write it directly. Otherwise, write it to a buffer and then write the
/// length and data to the stream.
/// </remarks>
/// <param name="obj">the object to write</param>
/// <exception cref="System.IO.IOException"/>
private void WriteObject(Writable obj)
{
// For Text and BytesWritable, encode them directly, so that they end up
// in C++ as the natural translations.
if (obj is Text)
{
Text t = (Text)obj;
int len = t.GetLength();
WritableUtils.WriteVInt(stream, len);
stream.Write(t.GetBytes(), 0, len);
}
else
{
if (obj is BytesWritable)
{
BytesWritable b = (BytesWritable)obj;
int len = b.GetLength();
WritableUtils.WriteVInt(stream, len);
stream.Write(b.GetBytes(), 0, len);
}
else
{
buffer.Reset();
obj.Write(buffer);
int length = buffer.GetLength();
WritableUtils.WriteVInt(stream, length);
stream.Write(buffer.GetData(), 0, length);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual string ToString(T obj)
{
outBuf.Reset();
serializer.Serialize(obj);
byte[] buf = new byte[outBuf.GetLength()];
System.Array.Copy(outBuf.GetData(), 0, buf, 0, buf.Length);
return(new string(Base64.EncodeBase64(buf), Charsets.Utf8));
}
/// <summary>
/// A helper function to serialize the JobTokenIdentifier to be sent to the
/// ShuffleHandler as ServiceData.
/// </summary>
/// <param name="jobToken">
/// the job token to be used for authentication of
/// shuffle data requests.
/// </param>
/// <returns>the serialized version of the jobToken.</returns>
/// <exception cref="System.IO.IOException"/>
public static ByteBuffer SerializeServiceData(Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier> jobToken)
{
//TODO these bytes should be versioned
DataOutputBuffer jobToken_dob = new DataOutputBuffer();
jobToken.Write(jobToken_dob);
return ByteBuffer.Wrap(jobToken_dob.GetData(), 0, jobToken_dob.GetLength());
}
/// <exception cref="System.Exception"/>
public virtual void TestNullEncoding()
{
string s = new string(new char[] { 0 });
DataOutputBuffer dob = new DataOutputBuffer();
new UTF8(s).Write(dob);
Assert.Equal(s, Runtime.GetStringForBytes(dob.GetData(
), 2, dob.GetLength() - 2, "UTF-8"));
}
public virtual void TestBaseBuffers()
{
DataOutputBuffer dob = new DataOutputBuffer();
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
WriteJunk(dob, r, seed, 1000);
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(dob.GetData(), 0, dob.GetLength());
ReadJunk(dib, r, seed, 1000);
dob.Reset();
WriteJunk(dob, r, seed, 1000);
dib.Reset(dob.GetData(), 0, dob.GetLength());
ReadJunk(dib, r, seed, 1000);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Append(K key, V value)
{
if (key.GetType() != keyClass)
{
throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass
);
}
if (value.GetType() != valueClass)
{
throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass
);
}
// Append the 'key'
keySerializer.Serialize(key);
int keyLength = buffer.GetLength();
if (keyLength < 0)
{
throw new IOException("Negative key-length not allowed: " + keyLength + " for " +
key);
}
// Append the 'value'
valueSerializer.Serialize(value);
int valueLength = buffer.GetLength() - keyLength;
if (valueLength < 0)
{
throw new IOException("Negative value-length not allowed: " + valueLength + " for "
+ value);
}
// Write the record out
WritableUtils.WriteVInt(@out, keyLength);
// key length
WritableUtils.WriteVInt(@out, valueLength);
// value length
@out.Write(buffer.GetData(), 0, buffer.GetLength());
// data
// Reset
buffer.Reset();
// Update bytes written
decompressedBytesWritten += keyLength + valueLength + WritableUtils.GetVIntSize(keyLength
) + WritableUtils.GetVIntSize(valueLength);
++numRecordsWritten;
}
/// <exception cref="System.IO.IOException"/>
public static void TestValue(int val, int vintlen)
{
DataOutputBuffer buf = new DataOutputBuffer();
DataInputBuffer inbuf = new DataInputBuffer();
WritableUtils.WriteVInt(buf, val);
if (Log.IsDebugEnabled())
{
Log.Debug("Value = " + val);
BytesWritable printer = new BytesWritable();
printer.Set(buf.GetData(), 0, buf.GetLength());
Log.Debug("Buffer = " + printer);
}
inbuf.Reset(buf.GetData(), 0, buf.GetLength());
Assert.Equal(val, WritableUtils.ReadVInt(inbuf));
Assert.Equal(vintlen, buf.GetLength());
Assert.Equal(vintlen, WritableUtils.GetVIntSize(val));
Assert.Equal(vintlen, WritableUtils.DecodeVIntSize(buf.GetData
()[0]));
}
private static byte[] ByteArrayForBloomKey(DataOutputBuffer buf)
{
int cleanLength = buf.GetLength();
byte[] ba = buf.GetData();
if (cleanLength != ba.Length)
{
ba = new byte[cleanLength];
System.Array.Copy(buf.GetData(), 0, ba, 0, cleanLength);
}
return(ba);
}
public virtual void TestCompressorDecopressorLogicWithCompressionStreams()
{
DataOutputStream deflateOut = null;
DataInputStream inflateIn = null;
int ByteSize = 1024 * 100;
byte[] bytes = Generate(ByteSize);
int bufferSize = 262144;
int compressionOverhead = (bufferSize / 6) + 32;
try
{
DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
CompressionOutputStream deflateFilter = new BlockCompressorStream(compressedDataBuffer
, new Lz4Compressor(bufferSize), bufferSize, compressionOverhead);
deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
deflateOut.Write(bytes, 0, bytes.Length);
deflateOut.Flush();
deflateFilter.Finish();
DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer
.GetLength());
CompressionInputStream inflateFilter = new BlockDecompressorStream(deCompressedDataBuffer
, new Lz4Decompressor(bufferSize), bufferSize);
inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));
byte[] result = new byte[ByteSize];
inflateIn.Read(result);
Assert.AssertArrayEquals("original array not equals compress/decompressed array",
result, bytes);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("testLz4CompressorDecopressorLogicWithCompressionStreams ex error !!!"
);
}
finally
{
try
{
if (deflateOut != null)
{
deflateOut.Close();
}
if (inflateIn != null)
{
inflateIn.Close();
}
}
catch (Exception)
{
}
}
}