public virtual void TestDataOutputByteBufferCompatibility()
{
DataOutputBuffer dob = new DataOutputBuffer();
DataOutputByteBuffer dobb = new DataOutputByteBuffer();
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
WriteJunk(dob, r, seed, 1000);
WriteJunk(dobb, r, seed, 1000);
byte[] check = ToBytes(dobb.GetData(), dobb.GetLength());
Assert.Equal(check.Length, dob.GetLength());
Assert.AssertArrayEquals(check, Arrays.CopyOf(dob.GetData(), dob.GetLength()));
dob.Reset();
dobb.Reset();
WriteJunk(dob, r, seed, 3000);
WriteJunk(dobb, r, seed, 3000);
check = ToBytes(dobb.GetData(), dobb.GetLength());
Assert.Equal(check.Length, dob.GetLength());
Assert.AssertArrayEquals(check, Arrays.CopyOf(dob.GetData(), dob.GetLength()));
dob.Reset();
dobb.Reset();
WriteJunk(dob, r, seed, 1000);
WriteJunk(dobb, r, seed, 1000);
check = ToBytes(dobb.GetData(), dobb.GetLength());
Assert.Equal("Failed Checking length = " + check.Length, check
.Length, dob.GetLength());
Assert.AssertArrayEquals(check, Arrays.CopyOf(dob.GetData(), dob.GetLength()));
}
public virtual void TestPacket()
{
Random r = new Random(12345L);
byte[] data = new byte[chunkSize];
r.NextBytes(data);
byte[] checksum = new byte[checksumSize];
r.NextBytes(checksum);
DataOutputBuffer os = new DataOutputBuffer(data.Length * 2);
byte[] packetBuf = new byte[data.Length * 2];
DFSPacket p = new DFSPacket(packetBuf, maxChunksPerPacket, 0, 0, checksumSize, false
);
p.SetSyncBlock(true);
p.WriteData(data, 0, data.Length);
p.WriteChecksum(checksum, 0, checksum.Length);
p.WriteTo(os);
//we have set syncBlock to true, so the header has the maximum length
int headerLen = PacketHeader.PktMaxHeaderLen;
byte[] readBuf = os.GetData();
AssertArrayRegionsEqual(readBuf, headerLen, checksum, 0, checksum.Length);
AssertArrayRegionsEqual(readBuf, headerLen + checksum.Length, data, 0, data.Length
);
}
/// <summary>
/// Write a protobuf to a buffer 'numProtos' times, and then
/// read them back, making sure all data comes through correctly.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void DoTest(int numProtos)
{
Configuration conf = new Configuration();
DataOutputBuffer @out = new DataOutputBuffer();
// Write numProtos protobufs to the buffer
Message[] sent = new Message[numProtos];
for (int i = 0; i < numProtos; i++)
{
// Construct a test protocol buffer using one of the
// protos that ships with the protobuf library
Message testProto = ((DescriptorProtos.EnumValueDescriptorProto)DescriptorProtos.EnumValueDescriptorProto
.NewBuilder().SetName("test" + i).SetNumber(i).Build());
ObjectWritable.WriteObject(@out, testProto, typeof(DescriptorProtos.EnumValueDescriptorProto
), conf);
sent[i] = testProto;
}
// Read back the data
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
for (int i_1 = 0; i_1 < numProtos; i_1++)
{
Message received = (Message)ObjectWritable.ReadObject(@in, conf);
Assert.Equal(sent[i_1], received);
}
}
/// <summary>Serialize the shuffle port into a ByteBuffer for use later on.</summary>
/// <param name="port">the port to be sent to the ApplciationMaster</param>
/// <returns>the serialized form of the port.</returns>
/// <exception cref="System.IO.IOException"/>
public static ByteBuffer SerializeMetaData(int port)
{
//TODO these bytes should be versioned
DataOutputBuffer port_dob = new DataOutputBuffer();
port_dob.WriteInt(port);
return ByteBuffer.Wrap(port_dob.GetData(), 0, port_dob.GetLength());
}
/// <summary>
/// test
/// <c>Text</c>
/// readFields/write operations
/// </summary>
public virtual void TestReadWriteOperations()
{
string line = "adsawseeeeegqewgasddga";
byte[] inputBytes = Runtime.GetBytesForString(line);
inputBytes = Bytes.Concat(new byte[] { unchecked ((byte)22) }, inputBytes);
DataInputBuffer @in = new DataInputBuffer();
DataOutputBuffer @out = new DataOutputBuffer();
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(line);
try
{
@in.Reset(inputBytes, inputBytes.Length);
text.ReadFields(@in);
}
catch (Exception)
{
Fail("testReadFields error !!!");
}
try
{
text.Write(@out);
}
catch (IOException)
{
}
catch (Exception)
{
Fail("testReadWriteOperations error !!!");
}
}
/// <exception cref="System.Exception"/>
public virtual void TestLocalizerStatusSerDe()
{
LocalizerStatus rsrcS = CreateLocalizerStatus();
NUnit.Framework.Assert.IsTrue(rsrcS is LocalizerStatusPBImpl);
LocalizerStatusPBImpl rsrcPb = (LocalizerStatusPBImpl)rsrcS;
DataOutputBuffer @out = new DataOutputBuffer();
rsrcPb.GetProto().WriteDelimitedTo(@out);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), 0, @out.GetLength());
YarnServerNodemanagerServiceProtos.LocalizerStatusProto rsrcPbD = YarnServerNodemanagerServiceProtos.LocalizerStatusProto
.ParseDelimitedFrom(@in);
NUnit.Framework.Assert.IsNotNull(rsrcPbD);
LocalizerStatus rsrcD = new LocalizerStatusPBImpl(rsrcPbD);
NUnit.Framework.Assert.AreEqual(rsrcS, rsrcD);
NUnit.Framework.Assert.AreEqual("localizer0", rsrcS.GetLocalizerId());
NUnit.Framework.Assert.AreEqual("localizer0", rsrcD.GetLocalizerId());
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcS.GetResourceStatus
(0));
NUnit.Framework.Assert.AreEqual(CreateLocalResourceStatus(), rsrcD.GetResourceStatus
(0));
}
private static ByteBuffer ConvertCredentialsToByteBuffer(Credentials credentials)
{
ByteBuffer appAttemptTokens = null;
DataOutputBuffer dob = new DataOutputBuffer();
try
{
if (credentials != null)
{
credentials.WriteTokenStorageToStream(dob);
appAttemptTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
}
return(appAttemptTokens);
}
catch (IOException)
{
Log.Error("Failed to convert Credentials to ByteBuffer.");
System.Diagnostics.Debug.Assert(false);
return(null);
}
finally
{
IOUtils.CloseStream(dob);
}
}
/// <exception cref="System.IO.IOException"/>
internal static DataInputBuffer CreateFakeCredentials(Random r, int nTok)
{
Credentials creds = new Credentials();
byte[] password = new byte[20];
Text kind = new Text();
Text service = new Text();
Text alias = new Text();
for (int i = 0; i < nTok; ++i)
{
byte[] identifier = Sharpen.Runtime.GetBytesForString(("idef" + i));
r.NextBytes(password);
kind.Set("kind" + i);
service.Set("service" + i);
alias.Set("token" + i);
Org.Apache.Hadoop.Security.Token.Token token = new Org.Apache.Hadoop.Security.Token.Token
(identifier, password, kind, service);
creds.AddToken(alias, token);
}
DataOutputBuffer buf = new DataOutputBuffer();
creds.WriteTokenStorageToStream(buf);
DataInputBuffer ret = new DataInputBuffer();
ret.Reset(buf.GetData(), 0, buf.GetLength());
return(ret);
}
/// <exception cref="System.IO.IOException"/>
protected internal override void FlushAndSync(bool durable)
{
int numReadyBytes = buf.CountReadyBytes();
if (numReadyBytes > 0)
{
int numReadyTxns = buf.CountReadyTxns();
long firstTxToFlush = buf.GetFirstReadyTxId();
System.Diagnostics.Debug.Assert(numReadyTxns > 0);
// Copy from our double-buffer into a new byte array. This is for
// two reasons:
// 1) The IPC code has no way of specifying to send only a slice of
// a larger array.
// 2) because the calls to the underlying nodes are asynchronous, we
// need a defensive copy to avoid accidentally mutating the buffer
// before it is sent.
DataOutputBuffer bufToSend = new DataOutputBuffer(numReadyBytes);
buf.FlushTo(bufToSend);
System.Diagnostics.Debug.Assert(bufToSend.GetLength() == numReadyBytes);
byte[] data = bufToSend.GetData();
System.Diagnostics.Debug.Assert(data.Length == bufToSend.GetLength());
QuorumCall <AsyncLogger, Void> qcall = loggers.SendEdits(segmentTxId, firstTxToFlush
, numReadyTxns, data);
loggers.WaitForWriteQuorum(qcall, writeTimeoutMs, "sendEdits");
// Since we successfully wrote this batch, let the loggers know. Any future
// RPCs will thus let the loggers know of the most recent transaction, even
// if a logger has fallen behind.
loggers.SetCommittedTxId(firstTxToFlush + numReadyTxns - 1);
}
}
public virtual void TestSerialization()
{
TestDelegationToken.TestDelegationTokenIdentifier origToken = new TestDelegationToken.TestDelegationTokenIdentifier
(new Text("alice"), new Text("bob"), new Text("colin"));
TestDelegationToken.TestDelegationTokenIdentifier newToken = new TestDelegationToken.TestDelegationTokenIdentifier
();
origToken.SetIssueDate(123);
origToken.SetMasterKeyId(321);
origToken.SetMaxDate(314);
origToken.SetSequenceNumber(12345);
// clone origToken into newToken
DataInputBuffer inBuf = new DataInputBuffer();
DataOutputBuffer outBuf = new DataOutputBuffer();
origToken.Write(outBuf);
inBuf.Reset(outBuf.GetData(), 0, outBuf.GetLength());
newToken.ReadFields(inBuf);
// now test the fields
Assert.Equal("alice", newToken.GetUser().GetUserName());
Assert.Equal(new Text("bob"), newToken.GetRenewer());
Assert.Equal("colin", newToken.GetUser().GetRealUser().GetUserName
());
Assert.Equal(123, newToken.GetIssueDate());
Assert.Equal(321, newToken.GetMasterKeyId());
Assert.Equal(314, newToken.GetMaxDate());
Assert.Equal(12345, newToken.GetSequenceNumber());
Assert.Equal(origToken, newToken);
}
/// <exception cref="System.Exception"/>
public virtual void TestIO()
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
for (int i = 0; i < NumIterations; i++)
{
// generate a random string
string before;
if (i == 0)
{
before = GetLongString();
}
else
{
before = GetTestString();
}
// write it
@out.Reset();
Org.Apache.Hadoop.IO.Text.WriteString(@out, before);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
string after = Org.Apache.Hadoop.IO.Text.ReadString(@in);
Assert.True(before.Equals(after));
// Test compatibility with Java's other decoder
int strLenSize = WritableUtils.GetVIntSize(Org.Apache.Hadoop.IO.Text.Utf8Length(before
));
string after2 = Runtime.GetStringForBytes(@out.GetData(), strLenSize, @out
.GetLength() - strLenSize, "UTF-8");
Assert.True(before.Equals(after2));
}
}
/// <summary>
/// If valueClass is undefined, readFields should throw an exception indicating
/// that the field is null.
/// </summary>
/// <remarks>
/// If valueClass is undefined, readFields should throw an exception indicating
/// that the field is null. Otherwise, readFields should succeed.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestThrowUndefinedValueException()
{
// Get a buffer containing a simple text array
Text[] elements = new Text[] { new Text("zero"), new Text("one"), new Text("two") };
TestArrayWritable.TextArrayWritable sourceArray = new TestArrayWritable.TextArrayWritable
();
sourceArray.Set(elements);
// Write it to a normal output buffer
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
sourceArray.Write(@out);
// Read the output buffer with TextReadable. Since the valueClass is defined,
// this should succeed
TestArrayWritable.TextArrayWritable destArray = new TestArrayWritable.TextArrayWritable
();
@in.Reset(@out.GetData(), @out.GetLength());
destArray.ReadFields(@in);
Writable[] destElements = destArray.Get();
Assert.True(destElements.Length == elements.Length);
for (int i = 0; i < elements.Length; i++)
{
Assert.Equal(destElements[i], elements[i]);
}
}
/// <exception cref="System.IO.IOException"/>
private void SetupTokens(ContainerLaunchContext container, ContainerId containerID
)
{
IDictionary <string, string> environment = container.GetEnvironment();
environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase
();
// Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId();
environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId
].GetSubmitTime().ToString();
environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId
].GetMaxAppAttempts().ToString();
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
if (container.GetTokens() != null)
{
// TODO: Don't do this kind of checks everywhere.
dibb.Reset(container.GetTokens());
credentials.ReadTokenStorageStream(dibb);
}
// Add AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken
();
if (amrmToken != null)
{
credentials.AddToken(amrmToken.GetService(), amrmToken);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSerializeAndDeserializeNull()
{
bool gotException = false;
try
{
new EnumSetWritable <TestEnumSetWritable.TestEnumSet>(null);
}
catch (RuntimeException)
{
gotException = true;
}
Assert.True("Instantiation of empty EnumSetWritable with no element type class "
+ "provided should throw exception", gotException);
EnumSetWritable <TestEnumSetWritable.TestEnumSet> nullFlagWritable = new EnumSetWritable
<TestEnumSetWritable.TestEnumSet>(null, typeof(TestEnumSetWritable.TestEnumSet));
DataOutputBuffer @out = new DataOutputBuffer();
ObjectWritable.WriteObject(@out, nullFlagWritable, nullFlagWritable.GetType(), null
);
DataInputBuffer @in = new DataInputBuffer();
@in.Reset(@out.GetData(), @out.GetLength());
EnumSet <TestEnumSetWritable.TestEnumSet> read = ((EnumSetWritable <TestEnumSetWritable.TestEnumSet
>)ObjectWritable.ReadObject(@in, null)).Get();
Assert.Equal(read, null);
}
/// <exception cref="System.IO.IOException"/>
public static void TestReadInRange(long val, int lower, int upper, bool expectSuccess
)
{
DataOutputBuffer buf = new DataOutputBuffer();
DataInputBuffer inbuf = new DataInputBuffer();
WritableUtils.WriteVLong(buf, val);
try
{
inbuf.Reset(buf.GetData(), 0, buf.GetLength());
long val2 = WritableUtils.ReadVIntInRange(inbuf, lower, upper);
if (!expectSuccess)
{
Fail("expected readVIntInRange to throw an exception");
}
Assert.Equal(val, val2);
}
catch (IOException e)
{
if (expectSuccess)
{
Log.Error("unexpected exception:", e);
Fail("readVIntInRange threw an unexpected exception");
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void DoTestLimitedIO(string str, int len)
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
@out.Reset();
try
{
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len);
Fail("expected writeString to fail when told to write a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len + 1);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
@in.Mark(len);
string after;
try
{
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len);
Fail("expected readString to fail when told to read a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
@in.Reset();
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len + 1);
Assert.True(str.Equals(after));
}
/// <exception cref="System.Exception"/>
public virtual void TestBadLength()
{
int Dlen = 100;
DataOutputBuffer dob = new DataOutputBuffer(Dlen + 4);
IFileOutputStream ifos = new IFileOutputStream(dob);
for (int i = 0; i < Dlen; ++i)
{
ifos.Write(i);
}
ifos.Close();
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(dob.GetData(), Dlen + 4);
IFileInputStream ifis = new IFileInputStream(dib, 100, new Configuration());
int i_1 = 0;
try
{
while (i_1 < Dlen - 8)
{
NUnit.Framework.Assert.AreEqual(i_1++, ifis.Read());
}
ifis.Close();
}
catch (ChecksumException)
{
NUnit.Framework.Assert.AreEqual("Checksum before close", i_1, Dlen - 8);
return;
}
Fail("Did not detect bad data in checksum");
}
public virtual void TestGzipCompatibility()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Info("seed: " + seed);
DataOutputBuffer dflbuf = new DataOutputBuffer();
GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
byte[] b = new byte[r.Next(128 * 1024 + 1)];
r.NextBytes(b);
gzout.Write(b);
gzout.Close();
DataInputBuffer gzbuf = new DataInputBuffer();
gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength());
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf);
Decompressor decom = codec.CreateDecompressor();
NUnit.Framework.Assert.IsNotNull(decom);
Assert.Equal(typeof(BuiltInGzipDecompressor), decom.GetType());
InputStream gzin = codec.CreateInputStream(gzbuf, decom);
dflbuf.Reset();
IOUtils.CopyBytes(gzin, dflbuf, 4096);
byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength());
Assert.AssertArrayEquals(b, dflchk);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void PopulateHeaders(IList<string> mapIds, string outputBaseStr
, string user, int reduce, HttpRequest request, HttpResponse response, bool keepAliveParam
, IDictionary<string, ShuffleHandler.Shuffle.MapOutputInfo> mapOutputInfoMap)
{
long contentLength = 0;
foreach (string mapId in mapIds)
{
string @base = outputBaseStr + mapId;
ShuffleHandler.Shuffle.MapOutputInfo outputInfo = this.GetMapOutputInfo(@base, mapId
, reduce, user);
if (mapOutputInfoMap.Count < this._enclosing.mapOutputMetaInfoCacheSize)
{
mapOutputInfoMap[mapId] = outputInfo;
}
// Index file
Path indexFileName = this.lDirAlloc.GetLocalPathToRead(@base + "/file.out.index",
this.conf);
IndexRecord info = this.indexCache.GetIndexInformation(mapId, reduce, indexFileName
, user);
ShuffleHeader header = new ShuffleHeader(mapId, info.partLength, info.rawLength,
reduce);
DataOutputBuffer dob = new DataOutputBuffer();
header.Write(dob);
contentLength += info.partLength;
contentLength += dob.GetLength();
}
// Now set the response headers.
this.SetResponseHeaders(response, keepAliveParam, contentLength);
}
/// <exception cref="System.IO.IOException"/>
internal EditLogBackupOutputStream(NamenodeRegistration bnReg, JournalInfo journalInfo
)
: base()
{
// RPC proxy to backup node
// backup node registration
// active node registration
// serialized output sent to backup node
// backup node
// active name-node
this.bnRegistration = bnReg;
this.journalInfo = journalInfo;
IPEndPoint bnAddress = NetUtils.CreateSocketAddr(bnRegistration.GetAddress());
try
{
this.backupNode = NameNodeProxies.CreateNonHAProxy <JournalProtocol>(new HdfsConfiguration
(), bnAddress, UserGroupInformation.GetCurrentUser(), true).GetProxy();
}
catch (IOException e)
{
Storage.Log.Error("Error connecting to: " + bnAddress, e);
throw;
}
this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize);
this.@out = new DataOutputBuffer(DefaultBufferSize);
}
public virtual void TestNodeHeartBeatResponse()
{
NodeHeartbeatResponse record = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHeartbeatResponse
>();
IDictionary <ApplicationId, ByteBuffer> appCredentials = new Dictionary <ApplicationId
, ByteBuffer>();
Credentials app1Cred = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token1 = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token1.SetKind(new Text("kind1"));
app1Cred.AddToken(new Text("token1"), token1);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token2 = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token2.SetKind(new Text("kind2"));
app1Cred.AddToken(new Text("token2"), token2);
DataOutputBuffer dob = new DataOutputBuffer();
app1Cred.WriteTokenStorageToStream(dob);
ByteBuffer byteBuffer1 = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
appCredentials[ApplicationId.NewInstance(1234, 1)] = byteBuffer1;
record.SetSystemCredentialsForApps(appCredentials);
NodeHeartbeatResponse proto = new NodeHeartbeatResponsePBImpl(((NodeHeartbeatResponsePBImpl
)record).GetProto());
NUnit.Framework.Assert.AreEqual(appCredentials, proto.GetSystemCredentialsForApps
());
}
public virtual void TestDoubleBuffer()
{
EditsDoubleBuffer buf = new EditsDoubleBuffer(1024);
NUnit.Framework.Assert.IsTrue(buf.IsFlushed());
byte[] data = new byte[100];
buf.WriteRaw(data, 0, data.Length);
NUnit.Framework.Assert.AreEqual("Should count new data correctly", data.Length, buf
.CountBufferedBytes());
NUnit.Framework.Assert.IsTrue("Writing to current buffer should not affect flush state"
, buf.IsFlushed());
// Swap the buffers
buf.SetReadyToFlush();
NUnit.Framework.Assert.AreEqual("Swapping buffers should still count buffered bytes"
, data.Length, buf.CountBufferedBytes());
NUnit.Framework.Assert.IsFalse(buf.IsFlushed());
// Flush to a stream
DataOutputBuffer outBuf = new DataOutputBuffer();
buf.FlushTo(outBuf);
NUnit.Framework.Assert.AreEqual(data.Length, outBuf.GetLength());
NUnit.Framework.Assert.IsTrue(buf.IsFlushed());
NUnit.Framework.Assert.AreEqual(0, buf.CountBufferedBytes());
// Write some more
buf.WriteRaw(data, 0, data.Length);
NUnit.Framework.Assert.AreEqual("Should count new data correctly", data.Length, buf
.CountBufferedBytes());
buf.SetReadyToFlush();
buf.FlushTo(outBuf);
NUnit.Framework.Assert.AreEqual(data.Length * 2, outBuf.GetLength());
NUnit.Framework.Assert.AreEqual(0, buf.CountBufferedBytes());
outBuf.Close();
}
/// <summary>
/// A helper function to serialize the JobTokenIdentifier to be sent to the
/// ShuffleHandler as ServiceData.
/// </summary>
/// <param name="jobToken">
/// the job token to be used for authentication of
/// shuffle data requests.
/// </param>
/// <returns>the serialized version of the jobToken.</returns>
/// <exception cref="System.IO.IOException"/>
public static ByteBuffer SerializeServiceData(Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier> jobToken)
{
//TODO these bytes should be versioned
DataOutputBuffer jobToken_dob = new DataOutputBuffer();
jobToken.Write(jobToken_dob);
return ByteBuffer.Wrap(jobToken_dob.GetData(), 0, jobToken_dob.GetLength());
}
/// <exception cref="System.Exception"/>
public virtual void TestNullEncoding()
{
string s = new string(new char[] { 0 });
DataOutputBuffer dob = new DataOutputBuffer();
new UTF8(s).Write(dob);
Assert.Equal(s, Runtime.GetStringForBytes(dob.GetData(
), 2, dob.GetLength() - 2, "UTF-8"));
}
/// <summary>
/// Generate a string with the url-quoted base64 encoded serialized form
/// of the Writable.
/// </summary>
/// <param name="obj">the object to serialize</param>
/// <returns>the encoded string</returns>
/// <exception cref="System.IO.IOException"/>
private static string EncodeWritable(IWritable obj)
{
DataOutputBuffer buf = new DataOutputBuffer();
obj.Write(buf);
Base64 encoder = new Base64(0, null, true);
byte[] raw = new byte[buf.GetLength()];
System.Array.Copy(buf.GetData(), 0, raw, 0, buf.GetLength());
return(encoder.EncodeToString(raw));
}
/// <summary>Construct an edit log output stream which writes to a ledger.</summary>
/// <exception cref="System.IO.IOException"/>
protected internal BookKeeperEditLogOutputStream(Configuration conf, LedgerHandle
lh)
: base()
{
bufCurrent = new DataOutputBuffer();
outstandingRequests = new AtomicInteger(0);
syncLatch = null;
this.lh = lh;
this.writer = new FSEditLogOp.Writer(bufCurrent);
this.transmissionThreshold = conf.GetInt(BookKeeperJournalManager.BkjmOutputBufferSize
, BookKeeperJournalManager.BkjmOutputBufferSizeDefault);
}
private static byte[] ByteArrayForBloomKey(DataOutputBuffer buf)
{
int cleanLength = buf.GetLength();
byte[] ba = buf.GetData();
if (cleanLength != ba.Length)
{
ba = new byte[cleanLength];
System.Array.Copy(buf.GetData(), 0, ba, 0, cleanLength);
}
return(ba);
}
public virtual void TestCompressorDecopressorLogicWithCompressionStreams()
{
DataOutputStream deflateOut = null;
DataInputStream inflateIn = null;
int ByteSize = 1024 * 100;
byte[] bytes = Generate(ByteSize);
int bufferSize = 262144;
int compressionOverhead = (bufferSize / 6) + 32;
try
{
DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
CompressionOutputStream deflateFilter = new BlockCompressorStream(compressedDataBuffer
, new Lz4Compressor(bufferSize), bufferSize, compressionOverhead);
deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
deflateOut.Write(bytes, 0, bytes.Length);
deflateOut.Flush();
deflateFilter.Finish();
DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer
.GetLength());
CompressionInputStream inflateFilter = new BlockDecompressorStream(deCompressedDataBuffer
, new Lz4Decompressor(bufferSize), bufferSize);
inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));
byte[] result = new byte[ByteSize];
inflateIn.Read(result);
Assert.AssertArrayEquals("original array not equals compress/decompressed array",
result, bytes);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("testLz4CompressorDecopressorLogicWithCompressionStreams ex error !!!"
);
}
finally
{
try
{
if (deflateOut != null)
{
deflateOut.Close();
}
if (inflateIn != null)
{
inflateIn.Close();
}
}
catch (Exception)
{
}
}
}
/// <summary>Generate byte array representing a set of GarbageMkdirOp</summary>
/// <exception cref="System.IO.IOException"/>
public static byte[] CreateGabageTxns(long startTxId, int numTxns)
{
DataOutputBuffer buf = new DataOutputBuffer();
FSEditLogOp.Writer writer = new FSEditLogOp.Writer(buf);
for (long txid = startTxId; txid < startTxId + numTxns; txid++)
{
FSEditLogOp op = new TestEditLog.GarbageMkdirOp();
op.SetTransactionId(txid);
writer.WriteOp(op);
}
return(Arrays.CopyOf(buf.GetData(), buf.GetLength()));
}
/// <exception cref="System.Exception"/>
public static byte[] CreateTxnData(int startTxn, int numTxns)
{
DataOutputBuffer buf = new DataOutputBuffer();
FSEditLogOp.Writer writer = new FSEditLogOp.Writer(buf);
for (long txid = startTxn; txid < startTxn + numTxns; txid++)
{
FSEditLogOp op = NameNodeAdapter.CreateMkdirOp("tx " + txid);
op.SetTransactionId(txid);
writer.WriteOp(op);
}
return(Arrays.CopyOf(buf.GetData(), buf.GetLength()));
}
public void testMaxLengthToReader()
{
Configuration conf = new Configuration();
OrcProto.Type rowType = OrcProto.Type.CreateBuilder()
.SetKind(OrcProto.Type.Types.Kind.STRUCT).Build();
OrcProto.Footer footer = OrcProto.Footer.CreateBuilder()
.SetHeaderLength(0).SetContentLength(0).SetNumberOfRows(0)
.SetRowIndexStride(0).AddTypes(rowType).Build();
OrcProto.PostScript ps = OrcProto.PostScript.CreateBuilder()
.SetCompression(OrcProto.CompressionKind.NONE)
.SetFooterLength((ulong)footer.SerializedSize)
.SetMagic("ORC").AddVersion(0).AddVersion(11).Build();
DataOutputBuffer buffer = new DataOutputBuffer();
footer.WriteTo(buffer);
ps.WriteTo(buffer);
buffer.write(ps.SerializedSize);
FileSystem fs = Mockito.mock(typeof(FileSystem), settings);
FSDataInputStream file = new FSDataInputStream(new BufferInStream(buffer.getData(),
buffer.getLength()));
string p = "/dir/file.orc";
Mockito.when(fs.open(p)).thenReturn(file);
OrcFile.ReaderOptions options = OrcFile.readerOptions(conf);
options.filesystem(fs);
options.maxLength(buffer.getLength());
Mockito.when(fs.getFileStatus(p))
.thenReturn(new FileStatus(10, false, 3, 3000, 0, p));
Reader reader = OrcFile.createReader(p, options);
}
