/// <summary>
/// Write a protobuf to a buffer 'numProtos' times, and then
/// read them back, making sure all data comes through correctly.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void DoTest(int numProtos)
{
Configuration conf = new Configuration();
DataOutputBuffer @out = new DataOutputBuffer();
// Write numProtos protobufs to the buffer
Message[] sent = new Message[numProtos];
for (int i = 0; i < numProtos; i++)
{
// Construct a test protocol buffer using one of the
// protos that ships with the protobuf library
Message testProto = ((DescriptorProtos.EnumValueDescriptorProto)DescriptorProtos.EnumValueDescriptorProto
.NewBuilder().SetName("test" + i).SetNumber(i).Build());
ObjectWritable.WriteObject(@out, testProto, typeof(DescriptorProtos.EnumValueDescriptorProto
), conf);
sent[i] = testProto;
}
// Read back the data
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
for (int i_1 = 0; i_1 < numProtos; i_1++)
{
Message received = (Message)ObjectWritable.ReadObject(@in, conf);
Assert.Equal(sent[i_1], received);
}
}
/// <exception cref="System.IO.IOException"/>
internal static DataInputBuffer CreateFakeCredentials(Random r, int nTok)
{
Credentials creds = new Credentials();
byte[] password = new byte[20];
Text kind = new Text();
Text service = new Text();
Text alias = new Text();
for (int i = 0; i < nTok; ++i)
{
byte[] identifier = Sharpen.Runtime.GetBytesForString(("idef" + i));
r.NextBytes(password);
kind.Set("kind" + i);
service.Set("service" + i);
alias.Set("token" + i);
Org.Apache.Hadoop.Security.Token.Token token = new Org.Apache.Hadoop.Security.Token.Token
(identifier, password, kind, service);
creds.AddToken(alias, token);
}
DataOutputBuffer buf = new DataOutputBuffer();
creds.WriteTokenStorageToStream(buf);
DataInputBuffer ret = new DataInputBuffer();
ret.Reset(buf.GetData(), 0, buf.GetLength());
return(ret);
}
/// <summary>
/// test
/// <c>Text</c>
/// readFields/write operations
/// </summary>
public virtual void TestReadWriteOperations()
{
string line = "adsawseeeeegqewgasddga";
byte[] inputBytes = Runtime.GetBytesForString(line);
inputBytes = Bytes.Concat(new byte[] { unchecked ((byte)22) }, inputBytes);
DataInputBuffer @in = new DataInputBuffer();
DataOutputBuffer @out = new DataOutputBuffer();
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(line);
try
{
@in.Reset(inputBytes, inputBytes.Length);
text.ReadFields(@in);
}
catch (Exception)
{
Fail("testReadFields error !!!");
}
try
{
text.Write(@out);
}
catch (IOException)
{
}
catch (Exception)
{
Fail("testReadWriteOperations error !!!");
}
}
/// <exception cref="System.IO.IOException"/>
public override bool NextRawKey(DataInputBuffer key)
{
try
{
if (!PositionToNextRecord(memDataIn))
{
return(false);
}
// Setup the key
int pos = memDataIn.GetPosition();
byte[] data = memDataIn.GetData();
key.Reset(data, pos, currentKeyLength);
// Position for the next value
long skipped = memDataIn.Skip(currentKeyLength);
if (skipped != currentKeyLength)
{
throw new IOException("Rec# " + recNo + ": Failed to skip past key of length: " +
currentKeyLength);
}
// Record the byte
bytesRead += currentKeyLength;
return(true);
}
catch (IOException ioe)
{
DumpOnError();
throw;
}
}
public virtual void TestNMTokenIdentifier()
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(1, 1), 1);
NodeId nodeId = NodeId.NewInstance("host0", 0);
string applicationSubmitter = "usr0";
int masterKeyId = 1;
NMTokenIdentifier token = new NMTokenIdentifier(appAttemptId, nodeId, applicationSubmitter
, masterKeyId);
NMTokenIdentifier anotherToken = new NMTokenIdentifier();
byte[] tokenContent = token.GetBytes();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenContent, tokenContent.Length);
anotherToken.ReadFields(dib);
// verify the whole record equals with original record
NUnit.Framework.Assert.AreEqual("Token is not the same after serialization " + "and deserialization."
, token, anotherToken);
// verify all properties are the same as original
NUnit.Framework.Assert.AreEqual("appAttemptId from proto is not the same with original token"
, anotherToken.GetApplicationAttemptId(), appAttemptId);
NUnit.Framework.Assert.AreEqual("NodeId from proto is not the same with original token"
, anotherToken.GetNodeId(), nodeId);
NUnit.Framework.Assert.AreEqual("applicationSubmitter from proto is not the same with original token"
, anotherToken.GetApplicationSubmitter(), applicationSubmitter);
NUnit.Framework.Assert.AreEqual("masterKeyId from proto is not the same with original token"
, anotherToken.GetKeyId(), masterKeyId);
}
public virtual void TestGzipCompatibility()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Info("seed: " + seed);
DataOutputBuffer dflbuf = new DataOutputBuffer();
GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
byte[] b = new byte[r.Next(128 * 1024 + 1)];
r.NextBytes(b);
gzout.Write(b);
gzout.Close();
DataInputBuffer gzbuf = new DataInputBuffer();
gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength());
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf);
Decompressor decom = codec.CreateDecompressor();
NUnit.Framework.Assert.IsNotNull(decom);
Assert.Equal(typeof(BuiltInGzipDecompressor), decom.GetType());
InputStream gzin = codec.CreateInputStream(gzbuf, decom);
dflbuf.Reset();
IOUtils.CopyBytes(gzin, dflbuf, 4096);
byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength());
Assert.AssertArrayEquals(b, dflchk);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSerializeAndDeserializeNull()
{
bool gotException = false;
try
{
new EnumSetWritable <TestEnumSetWritable.TestEnumSet>(null);
}
catch (RuntimeException)
{
gotException = true;
}
Assert.True("Instantiation of empty EnumSetWritable with no element type class "
+ "provided should throw exception", gotException);
EnumSetWritable <TestEnumSetWritable.TestEnumSet> nullFlagWritable = new EnumSetWritable
<TestEnumSetWritable.TestEnumSet>(null, typeof(TestEnumSetWritable.TestEnumSet));
DataOutputBuffer @out = new DataOutputBuffer();
ObjectWritable.WriteObject(@out, nullFlagWritable, nullFlagWritable.GetType(), null
);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
EnumSet <TestEnumSetWritable.TestEnumSet> read = ((EnumSetWritable <TestEnumSetWritable.TestEnumSet
>)ObjectWritable.ReadObject(@in, null)).Get();
Assert.Equal(read, null);
}
/// <summary>
/// If valueClass is undefined, readFields should throw an exception indicating
/// that the field is null.
/// </summary>
/// <remarks>
/// If valueClass is undefined, readFields should throw an exception indicating
/// that the field is null. Otherwise, readFields should succeed.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestThrowUndefinedValueException()
{
// Get a buffer containing a simple text array
Text[] elements = new Text[] { new Text("zero"), new Text("one"), new Text("two") };
TestArrayWritable.TextArrayWritable sourceArray = new TestArrayWritable.TextArrayWritable
();
sourceArray.Set(elements);
// Write it to a normal output buffer
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
sourceArray.Write(@out);
// Read the output buffer with TextReadable. Since the valueClass is defined,
// this should succeed
TestArrayWritable.TextArrayWritable destArray = new TestArrayWritable.TextArrayWritable
();
@in.Reset(@out.GetData(), @out.GetLength());
destArray.ReadFields(@in);
Writable[] destElements = destArray.Get();
Assert.True(destElements.Length == elements.Length);
for (int i = 0; i < elements.Length; i++)
{
Assert.Equal(destElements[i], elements[i]);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestBadLength()
{
int Dlen = 100;
DataOutputBuffer dob = new DataOutputBuffer(Dlen + 4);
IFileOutputStream ifos = new IFileOutputStream(dob);
for (int i = 0; i < Dlen; ++i)
{
ifos.Write(i);
}
ifos.Close();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(dob.GetData(), Dlen + 4);
IFileInputStream ifis = new IFileInputStream(dib, 100, new Configuration());
int i_1 = 0;
try
{
while (i_1 < Dlen - 8)
{
NUnit.Framework.Assert.AreEqual(i_1++, ifis.Read());
}
ifis.Close();
}
catch (ChecksumException)
{
NUnit.Framework.Assert.AreEqual("Checksum before close", i_1, Dlen - 8);
return;
}
Fail("Did not detect bad data in checksum");
}
/// <exception cref="System.IO.IOException"/>
public static void TestReadInRange(long val, int lower, int upper, bool expectSuccess
)
{
DataOutputBuffer buf = new DataOutputBuffer();
DataInputBuffer inbuf = new DataInputBuffer();
WritableUtils.WriteVLong(buf, val);
try
{
inbuf.Reset(buf.GetData(), 0, buf.GetLength());
long val2 = WritableUtils.ReadVIntInRange(inbuf, lower, upper);
if (!expectSuccess)
{
Fail("expected readVIntInRange to throw an exception");
}
Assert.Equal(val, val2);
}
catch (IOException e)
{
if (expectSuccess)
{
Log.Error("unexpected exception:", e);
Fail("readVIntInRange threw an unexpected exception");
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestIO()
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
for (int i = 0; i < NumIterations; i++)
{
// generate a random string
string before;
if (i == 0)
{
before = GetLongString();
}
else
{
before = GetTestString();
}
// write it
@out.Reset();
Org.Apache.Hadoop.IO.Text.WriteString(@out, before);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
string after = Org.Apache.Hadoop.IO.Text.ReadString(@in);
Assert.True(before.Equals(after));
// Test compatibility with Java's other decoder
int strLenSize = WritableUtils.GetVIntSize(Org.Apache.Hadoop.IO.Text.Utf8Length(before
));
string after2 = Runtime.GetStringForBytes(@out.GetData(), strLenSize, @out
.GetLength() - strLenSize, "UTF-8");
Assert.True(before.Equals(after2));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestLocalizerStatusSerDe()
{
LocalizerStatus rsrcS = CreateLocalizerStatus();
NUnit.Framework.Assert.IsTrue(rsrcS is LocalizerStatusPBImpl);
LocalizerStatusPBImpl rsrcPb = (LocalizerStatusPBImpl)rsrcS;
DataOutputBuffer @out = new DataOutputBuffer();
rsrcPb.GetProto().WriteDelimitedTo(@out);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), 0, @out.GetLength());
YarnServerNodemanagerServiceProtos.LocalizerStatusProto rsrcPbD = YarnServerNodemanagerServiceProtos.LocalizerStatusProto
.ParseDelimitedFrom(@in);
NUnit.Framework.Assert.IsNotNull(rsrcPbD);
LocalizerStatus rsrcD = new LocalizerStatusPBImpl(rsrcPbD);
NUnit.Framework.Assert.AreEqual(rsrcS, rsrcD);
NUnit.Framework.Assert.AreEqual("localizer0", rsrcS.GetLocalizerId());
NUnit.Framework.Assert.AreEqual("localizer0", rsrcD.GetLocalizerId());
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcS.GetResourceStatus
(0));
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcD.GetResourceStatus
(0));
}
/// <exception cref="System.IO.IOException"/>
public virtual void DoTestLimitedIO(string str, int len)
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
@out.Reset();
try
{
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len);
Fail("expected writeString to fail when told to write a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len + 1);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
@in.Mark(len);
string after;
try
{
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len);
Fail("expected readString to fail when told to read a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
@in.Reset();
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len + 1);
Assert.True(str.Equals(after));
}
public virtual void TestSerialization()
{
TestDelegationToken.TestDelegationTokenIdentifier origToken = new TestDelegationToken.TestDelegationTokenIdentifier
(new Text("alice"), new Text("bob"), new Text("colin"));
TestDelegationToken.TestDelegationTokenIdentifier newToken = new TestDelegationToken.TestDelegationTokenIdentifier
();
origToken.SetIssueDate(123);
origToken.SetMasterKeyId(321);
origToken.SetMaxDate(314);
origToken.SetSequenceNumber(12345);
// clone origToken into newToken
DataInputBuffer inBuf = new DataInputBuffer();
DataOutputBuffer outBuf = new DataOutputBuffer();
origToken.Write(outBuf);
inBuf.Reset(outBuf.GetData(), 0, outBuf.GetLength());
newToken.ReadFields(inBuf);
// now test the fields
Assert.Equal("alice", newToken.GetUser().GetUserName());
Assert.Equal(new Text("bob"), newToken.GetRenewer());
Assert.Equal("colin", newToken.GetUser().GetRealUser().GetUserName
());
Assert.Equal(123, newToken.GetIssueDate());
Assert.Equal(321, newToken.GetMasterKeyId());
Assert.Equal(314, newToken.GetMaxDate());
Assert.Equal(12345, newToken.GetSequenceNumber());
Assert.Equal(origToken, newToken);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Append(DataInputBuffer key, DataInputBuffer value)
{
int keyLength = key.GetLength() - key.GetPosition();
if (keyLength < 0)
{
throw new IOException("Negative key-length not allowed: " + keyLength + " for " +
key);
}
int valueLength = value.GetLength() - value.GetPosition();
if (valueLength < 0)
{
throw new IOException("Negative value-length not allowed: " + valueLength + " for "
+ value);
}
WritableUtils.WriteVInt(@out, keyLength);
WritableUtils.WriteVInt(@out, valueLength);
@out.Write(key.GetData(), key.GetPosition(), keyLength);
@out.Write(value.GetData(), value.GetPosition(), valueLength);
// Update bytes written
decompressedBytesWritten += keyLength + valueLength + WritableUtils.GetVIntSize(keyLength
) + WritableUtils.GetVIntSize(valueLength);
++numRecordsWritten;
}
/// <summary>Modify the writable to the value from the newValue</summary>
/// <param name="obj">the object to read into</param>
/// <param name="newValue">the string with the url-safe base64 encoded bytes</param>
/// <exception cref="System.IO.IOException"/>
private static void DecodeWritable(IWritable obj, string newValue)
{
Base64 decoder = new Base64(0, null, true);
DataInputBuffer buf = new DataInputBuffer();
byte[] decoded = decoder.Decode(newValue);
buf.Reset(decoded, decoded.Length);
obj.ReadFields(buf);
}
/// <exception cref="System.IO.IOException"/>
public override void NextRawValue(DataInputBuffer value)
{
DataInputBuffer vb = this.kvIter.GetValue();
int vp = vb.GetPosition();
int vlen = vb.GetLength() - vp;
value.Reset(vb.GetData(), vp, vlen);
this.bytesRead += vlen;
}
public Void Answer(InvocationOnMock invocation)
{
object[] args = invocation.GetArguments();
DataInputBuffer key = (DataInputBuffer)args[0];
key.Reset(Sharpen.Runtime.GetBytesForString(("Segment Value " + segmentName + this
.i)), 20);
return(null);
}
/// <exception cref="System.IO.IOException"/>
internal virtual bool ReserveSpace(DataInputBuffer key, DataInputBuffer value)
{
int keyLength = key.GetLength() - key.GetPosition();
int valueLength = value.GetLength() - value.GetPosition();
int requestedSize = keyLength + valueLength + WritableUtils.GetVIntSize(keyLength
) + WritableUtils.GetVIntSize(valueLength);
return(this.ReserveSpace(requestedSize));
}
/// <exception cref="System.IO.IOException"/>
protected internal override InputStream GetInputStream(int bufferSize, byte[] key
, byte[] iv)
{
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(buf, 0, bufLen);
return(new CryptoInputStream(new TestCryptoStreams.FakeInputStream(@in), codec, bufferSize
, key, iv));
}
/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerTokenWithEpoch(Configuration conf)
{
Log.Info("Running test for serializing/deserializing containerIds");
NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager
().GetRMContext().GetNMTokenSecretManager();
ApplicationId appId = ApplicationId.NewInstance(1, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L);
NodeManager nm = yarnCluster.GetNodeManager(0);
NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager
();
string user = "******";
WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
NodeId nodeId = nm.GetNMContext().GetNodeId();
// Both id should be equal.
NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId
(), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId());
// Creating a normal Container Token
RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
().GetRMContext().GetContainerTokenSecretManager();
Resource r = Resource.NewInstance(1230, 2);
Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager
.CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
);
byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
containerTokenIdentifier.ReadFields(dib);
NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID());
NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID
().ToString());
Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken
(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.Create(conf);
TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
IList <ContainerId> containerIds = new List <ContainerId>();
containerIds.AddItem(cId);
ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken
, nodeId, user);
GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest
.NewInstance(containerIds));
NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]);
NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId
());
NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId
().ToString());
}
protected override bool LessThan(object a, object b)
{
DataInputBuffer key1 = ((Merger.Segment <K, V>)a).GetKey();
DataInputBuffer key2 = ((Merger.Segment <K, V>)b).GetKey();
int s1 = key1.GetPosition();
int l1 = key1.GetLength() - s1;
int s2 = key2.GetPosition();
int l2 = key2.GetLength() - s2;
return(comparator.Compare(key1.GetData(), s1, l1, key2.GetData(), s2, l2) < 0);
}
public virtual void TestCompressorDecopressorLogicWithCompressionStreams()
{
DataOutputStream deflateOut = null;
DataInputStream inflateIn = null;
int ByteSize = 1024 * 100;
byte[] bytes = Generate(ByteSize);
int bufferSize = 262144;
int compressionOverhead = (bufferSize / 6) + 32;
try
{
DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
CompressionOutputStream deflateFilter = new BlockCompressorStream(compressedDataBuffer
, new Lz4Compressor(bufferSize), bufferSize, compressionOverhead);
deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
deflateOut.Write(bytes, 0, bytes.Length);
deflateOut.Flush();
deflateFilter.Finish();
DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer
.GetLength());
CompressionInputStream inflateFilter = new BlockDecompressorStream(deCompressedDataBuffer
, new Lz4Decompressor(bufferSize), bufferSize);
inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));
byte[] result = new byte[ByteSize];
inflateIn.Read(result);
Assert.AssertArrayEquals("original array not equals compress/decompressed array",
result, bytes);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("testLz4CompressorDecopressorLogicWithCompressionStreams ex error !!!"
);
}
finally
{
try
{
if (deflateOut != null)
{
deflateOut.Close();
}
if (inflateIn != null)
{
inflateIn.Close();
}
}
catch (Exception)
{
}
}
}
/// <exception cref="System.IO.IOException"/>
public override bool NextRawKey(DataInputBuffer key)
{
if (this.kvIter.Next())
{
DataInputBuffer kb = this.kvIter.GetKey();
int kp = kb.GetPosition();
int klen = kb.GetLength() - kp;
key.Reset(kb.GetData(), kp, klen);
this.bytesRead += klen;
return(true);
}
return(false);
}
/// <exception cref="System.IO.IOException"/>
private ContainerTokenIdentifier GetContainerTokenIdentifierFromToken(Org.Apache.Hadoop.Yarn.Api.Records.Token
containerToken)
{
ContainerTokenIdentifier containerTokenIdentifier;
containerTokenIdentifier = new ContainerTokenIdentifier();
byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
containerTokenIdentifier.ReadFields(dib);
return(containerTokenIdentifier);
}
/// <exception cref="System.IO.IOException"/>
public TextRecordInputStream(Display _enclosing, FileStatus f)
{
this._enclosing = _enclosing;
Path fpath = f.GetPath();
Configuration lconf = this._enclosing.GetConf();
this.r = new SequenceFile.Reader(lconf, SequenceFile.Reader.File(fpath));
this.key = ReflectionUtils.NewInstance(this.r.GetKeyClass().AsSubclass <IWritableComparable <> >(), lconf);
this.val = ReflectionUtils.NewInstance(this.r.GetValueClass().AsSubclass <IWritable
>(), lconf);
this.inbuf = new DataInputBuffer();
this.outbuf = new DataOutputBuffer();
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Write(DataInputBuffer key, DataInputBuffer value)
{
if (this.writer == null)
{
// If spillNumber is 0, we should have called activate and not
// come here at all
System.Diagnostics.Debug.Assert((this.spillNumber != 0));
this.writer = this.CreateSpillFile();
}
this.writer.Append(key, value);
BackupStore.Log.Debug("ID: " + this._enclosing.segmentList.Count + " WRITE TO DISK"
);
}
/// <summary>Utility method for testing writables.</summary>
/// <exception cref="System.Exception"/>
public static Writable TestWritable(Writable before, Configuration conf)
{
DataOutputBuffer dob = new DataOutputBuffer();
before.Write(dob);
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(dob.GetData(), dob.GetLength());
Writable after = (Writable)ReflectionUtils.NewInstance(before.GetType(), conf);
after.ReadFields(dib);
Assert.Equal(before, after);
return(after);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSerializeAndDeserializeNonEmpty()
{
DataOutputBuffer @out = new DataOutputBuffer();
ObjectWritable.WriteObject(@out, nonEmptyFlagWritable, nonEmptyFlagWritable.GetType
(), null);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
EnumSet <TestEnumSetWritable.TestEnumSet> read = ((EnumSetWritable <TestEnumSetWritable.TestEnumSet
>)ObjectWritable.ReadObject(@in, null)).Get();
Assert.Equal(read, nonEmptyFlag);
}
/// <summary>Write the key and value to the cache in the IFile format</summary>
/// <param name="key"/>
/// <param name="value"/>
/// <exception cref="System.IO.IOException"/>
public virtual void Write(DataInputBuffer key, DataInputBuffer value)
{
int keyLength = key.GetLength() - key.GetPosition();
int valueLength = value.GetLength() - value.GetPosition();
WritableUtils.WriteVInt(this.dataOut, keyLength);
WritableUtils.WriteVInt(this.dataOut, valueLength);
this.dataOut.Write(key.GetData(), key.GetPosition(), keyLength);
this.dataOut.Write(value.GetData(), value.GetPosition(), valueLength);
this.usedSize += keyLength + valueLength + WritableUtils.GetVIntSize(keyLength) +
WritableUtils.GetVIntSize(valueLength);
BackupStore.Log.Debug("ID: " + this._enclosing.segmentList.Count + " WRITE TO MEM"
);
}
