public virtual void TestDataOutputByteBufferCompatibility()
{
DataOutputBuffer dob = new DataOutputBuffer();
DataOutputByteBuffer dobb = new DataOutputByteBuffer();
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
WriteJunk(dob, r, seed, 1000);
WriteJunk(dobb, r, seed, 1000);
byte[] check = ToBytes(dobb.GetData(), dobb.GetLength());
Assert.Equal(check.Length, dob.GetLength());
Assert.AssertArrayEquals(check, Arrays.CopyOf(dob.GetData(), dob.GetLength()));
dob.Reset();
dobb.Reset();
WriteJunk(dob, r, seed, 3000);
WriteJunk(dobb, r, seed, 3000);
check = ToBytes(dobb.GetData(), dobb.GetLength());
Assert.Equal(check.Length, dob.GetLength());
Assert.AssertArrayEquals(check, Arrays.CopyOf(dob.GetData(), dob.GetLength()));
dob.Reset();
dobb.Reset();
WriteJunk(dob, r, seed, 1000);
WriteJunk(dobb, r, seed, 1000);
check = ToBytes(dobb.GetData(), dobb.GetLength());
Assert.Equal("Failed Checking length = " + check.Length, check
.Length, dob.GetLength());
Assert.AssertArrayEquals(check, Arrays.CopyOf(dob.GetData(), dob.GetLength()));
}
/// <summary>Read raw bytes from a SequenceFile.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override bool NextKeyValue()
{
lock (this)
{
if (done)
{
return(false);
}
long pos = @in.GetPosition();
bool eof = -1 == @in.NextRawKey(buffer);
if (!eof)
{
if (key == null)
{
key = new BytesWritable();
}
if (value == null)
{
value = new BytesWritable();
}
key.Set(buffer.GetData(), 0, buffer.GetLength());
buffer.Reset();
@in.NextRawValue(vbytes);
vbytes.WriteUncompressedBytes(buffer);
value.Set(buffer.GetData(), 0, buffer.GetLength());
buffer.Reset();
}
return(!(done = (eof || (pos >= end && @in.SyncSeen()))));
}
}
public virtual void TestGzipCompatibility()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Info("seed: " + seed);
DataOutputBuffer dflbuf = new DataOutputBuffer();
GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
byte[] b = new byte[r.Next(128 * 1024 + 1)];
r.NextBytes(b);
gzout.Write(b);
gzout.Close();
DataInputBuffer gzbuf = new DataInputBuffer();
gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength());
Configuration conf = new Configuration();
conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false);
CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf);
Decompressor decom = codec.CreateDecompressor();
NUnit.Framework.Assert.IsNotNull(decom);
Assert.Equal(typeof(BuiltInGzipDecompressor), decom.GetType());
InputStream gzin = codec.CreateInputStream(gzbuf, decom);
dflbuf.Reset();
IOUtils.CopyBytes(gzin, dflbuf, 4096);
byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength());
Assert.AssertArrayEquals(b, dflchk);
}
/// <exception cref="System.Exception"/>
public virtual void TestIO()
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
for (int i = 0; i < NumIterations; i++)
{
// generate a random string
string before;
if (i == 0)
{
before = GetLongString();
}
else
{
before = GetTestString();
}
// write it
@out.Reset();
Org.Apache.Hadoop.IO.Text.WriteString(@out, before);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
string after = Org.Apache.Hadoop.IO.Text.ReadString(@in);
Assert.True(before.Equals(after));
// Test compatibility with Java's other decoder
int strLenSize = WritableUtils.GetVIntSize(Org.Apache.Hadoop.IO.Text.Utf8Length(before
));
string after2 = Runtime.GetStringForBytes(@out.GetData(), strLenSize, @out
.GetLength() - strLenSize, "UTF-8");
Assert.True(before.Equals(after2));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void DoTestLimitedIO(string str, int len)
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
@out.Reset();
try
{
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len);
Fail("expected writeString to fail when told to write a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
Org.Apache.Hadoop.IO.Text.WriteString(@out, str, len + 1);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
@in.Mark(len);
string after;
try
{
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len);
Fail("expected readString to fail when told to read a string " + "that was too long! The string was '"
+ str + "'");
}
catch (IOException)
{
}
@in.Reset();
after = Org.Apache.Hadoop.IO.Text.ReadString(@in, len + 1);
Assert.True(str.Equals(after));
}
/// <summary>Write the given object to the stream.</summary>
/// <remarks>
/// Write the given object to the stream. If it is a Text or BytesWritable,
/// write it directly. Otherwise, write it to a buffer and then write the
/// length and data to the stream.
/// </remarks>
/// <param name="obj">the object to write</param>
/// <exception cref="System.IO.IOException"/>
private void WriteObject(Writable obj)
{
// For Text and BytesWritable, encode them directly, so that they end up
// in C++ as the natural translations.
if (obj is Text)
{
Text t = (Text)obj;
int len = t.GetLength();
WritableUtils.WriteVInt(stream, len);
stream.Write(t.GetBytes(), 0, len);
}
else
{
if (obj is BytesWritable)
{
BytesWritable b = (BytesWritable)obj;
int len = b.GetLength();
WritableUtils.WriteVInt(stream, len);
stream.Write(b.GetBytes(), 0, len);
}
else
{
buffer.Reset();
obj.Write(buffer);
int length = buffer.GetLength();
WritableUtils.WriteVInt(stream, length);
stream.Write(buffer.GetData(), 0, length);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual string ToString(T obj)
{
outBuf.Reset();
serializer.Serialize(obj);
byte[] buf = new byte[outBuf.GetLength()];
System.Array.Copy(outBuf.GetData(), 0, buf, 0, buf.Length);
return(new string(Base64.EncodeBase64(buf), Charsets.Utf8));
}
/// <summary>Read raw bytes from a SequenceFile.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual bool Next(BytesWritable key, BytesWritable val)
{
lock (this)
{
if (done)
{
return(false);
}
long pos = @in.GetPosition();
bool eof = -1 == @in.NextRawKey(buffer);
if (!eof)
{
key.Set(buffer.GetData(), 0, buffer.GetLength());
buffer.Reset();
@in.NextRawValue(vbytes);
vbytes.WriteUncompressedBytes(buffer);
val.Set(buffer.GetData(), 0, buffer.GetLength());
buffer.Reset();
}
return(!(done = (eof || (pos >= end && @in.SyncSeen()))));
}
}
// wait for whatever we wait on
/// <summary>Transmit the current buffer to bookkeeper.</summary>
/// <remarks>
/// Transmit the current buffer to bookkeeper.
/// Synchronised at the FSEditLog level. #write() and #setReadyToFlush()
/// are never called at the same time.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private void Transmit()
{
if (!transmitResult.CompareAndSet(BKException.Code.Ok, BKException.Code.Ok))
{
throw new IOException("Trying to write to an errored stream;" + " Error code : ("
+ transmitResult.Get() + ") " + BKException.GetMessage(transmitResult.Get()));
}
if (bufCurrent.GetLength() > 0)
{
byte[] entry = Arrays.CopyOf(bufCurrent.GetData(), bufCurrent.GetLength());
lh.AsyncAddEntry(entry, this, null);
bufCurrent.Reset();
outstandingRequests.IncrementAndGet();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCompare()
{
DataOutputBuffer out1 = new DataOutputBuffer();
DataOutputBuffer out2 = new DataOutputBuffer();
DataOutputBuffer out3 = new DataOutputBuffer();
Text.Comparator comparator = new Text.Comparator();
for (int i = 0; i < NumIterations; i++)
{
// reset output buffer
out1.Reset();
out2.Reset();
out3.Reset();
// generate two random strings
string str1 = GetTestString();
string str2 = GetTestString();
if (i == 0)
{
str1 = GetLongString();
str2 = GetLongString();
}
else
{
str1 = GetTestString();
str2 = GetTestString();
}
// convert to texts
Org.Apache.Hadoop.IO.Text txt1 = new Org.Apache.Hadoop.IO.Text(str1);
Org.Apache.Hadoop.IO.Text txt2 = new Org.Apache.Hadoop.IO.Text(str2);
Org.Apache.Hadoop.IO.Text txt3 = new Org.Apache.Hadoop.IO.Text(str1);
// serialize them
txt1.Write(out1);
txt2.Write(out2);
txt3.Write(out3);
// compare two strings by looking at their binary formats
int ret1 = comparator.Compare(out1.GetData(), 0, out1.GetLength(), out2.GetData()
, 0, out2.GetLength());
// compare two strings
int ret2 = txt1.CompareTo(txt2);
Assert.Equal(ret1, ret2);
Assert.Equal("Equivalence of different txt objects, same content"
, 0, txt1.CompareTo(txt3));
Assert.Equal("Equvalence of data output buffers", 0, comparator
.Compare(out1.GetData(), 0, out3.GetLength(), out3.GetData(), 0, out3.GetLength(
)));
}
}
public virtual void TestBaseBuffers()
{
DataOutputBuffer dob = new DataOutputBuffer();
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
WriteJunk(dob, r, seed, 1000);
DataInputBuffer dib = new DataInputBuffer();
dib.Reset(dob.GetData(), 0, dob.GetLength());
ReadJunk(dib, r, seed, 1000);
dob.Reset();
WriteJunk(dob, r, seed, 1000);
dib.Reset(dob.GetData(), 0, dob.GetLength());
ReadJunk(dib, r, seed, 1000);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Append(K key, V value)
{
if (key.GetType() != keyClass)
{
throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass
);
}
if (value.GetType() != valueClass)
{
throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass
);
}
// Append the 'key'
keySerializer.Serialize(key);
int keyLength = buffer.GetLength();
if (keyLength < 0)
{
throw new IOException("Negative key-length not allowed: " + keyLength + " for " +
key);
}
// Append the 'value'
valueSerializer.Serialize(value);
int valueLength = buffer.GetLength() - keyLength;
if (valueLength < 0)
{
throw new IOException("Negative value-length not allowed: " + valueLength + " for "
+ value);
}
// Write the record out
WritableUtils.WriteVInt(@out, keyLength);
// key length
WritableUtils.WriteVInt(@out, valueLength);
// value length
@out.Write(buffer.GetData(), 0, buffer.GetLength());
// data
// Reset
buffer.Reset();
// Update bytes written
decompressedBytesWritten += keyLength + valueLength + WritableUtils.GetVIntSize(keyLength
) + WritableUtils.GetVIntSize(valueLength);
++numRecordsWritten;
}
/// <exception cref="System.IO.IOException"/>
private E MakeCopyForPassByValue <E>(Serialization <E> serialization, E obj)
{
Org.Apache.Hadoop.IO.Serializer.Serializer <E> ser = serialization.GetSerializer(GenericsUtil
.GetClass(obj));
Deserializer <E> deser = serialization.GetDeserializer(GenericsUtil.GetClass(obj));
DataOutputBuffer dof = this._enclosing.threadLocalDataOutputBuffer.Get();
dof.Reset();
ser.Open(dof);
ser.Serialize(obj);
ser.Close();
obj = ReflectionUtils.NewInstance(GenericsUtil.GetClass(obj), this._enclosing.GetChainJobConf
());
ByteArrayInputStream bais = new ByteArrayInputStream(dof.GetData(), 0, dof.GetLength
());
deser.Open(bais);
deser.Deserialize(obj);
deser.Close();
return(obj);
}
/// <summary>Write infLen bytes (deflated) to file in test dir using codec.</summary>
/// <remarks>
/// Write infLen bytes (deflated) to file in test dir using codec.
/// Records are of the form
/// <i><b64 rand><i+i><b64 rand>
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private static Path WriteSplitTestFile(FileSystem fs, Random rand, CompressionCodec
codec, long infLen)
{
int RecSize = 1024;
Path wd = new Path(new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
(fs), codec.GetType().Name);
Path file = new Path(wd, "test" + codec.GetDefaultExtension());
byte[] b = new byte[RecSize];
Base64 b64 = new Base64(0, null);
DataOutputStream fout = null;
Compressor cmp = CodecPool.GetCompressor(codec);
try
{
fout = new DataOutputStream(codec.CreateOutputStream(fs.Create(file, true), cmp));
DataOutputBuffer dob = new DataOutputBuffer(RecSize * 4 / 3 + 4);
int seq = 0;
while (infLen > 0)
{
rand.NextBytes(b);
byte[] b64enc = b64.Encode(b);
// ensures rand printable, no LF
dob.Reset();
dob.WriteInt(seq);
System.Array.Copy(dob.GetData(), 0, b64enc, 0, dob.GetLength());
fout.Write(b64enc);
fout.Write('\n');
++seq;
infLen -= b64enc.Length;
}
Log.Info("Wrote " + seq + " records to " + file);
}
finally
{
IOUtils.Cleanup(Log, fout);
CodecPool.ReturnCompressor(cmp);
}
return(file);
}
public override void Run()
{
string name = this.GetName();
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
for (int i = 0; i < 1000; ++i)
{
try
{
@out.Reset();
WritableUtils.WriteString(@out, name);
@in.Reset(@out.GetData(), @out.GetLength());
string s = WritableUtils.ReadString(@in);
Assert.Equal("input buffer reset contents = " + name, name, s);
}
catch (Exception ioe)
{
throw new RuntimeException(ioe);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestIO()
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
for (int i = 0; i < 10000; i++)
{
// generate a random string
string before = GetTestString();
// write it
@out.Reset();
UTF8.WriteString(@out, before);
// test that it reads correctly
@in.Reset(@out.GetData(), @out.GetLength());
string after = UTF8.ReadString(@in);
Assert.Equal(before, after);
// test that it reads correctly with BinaryReader
@in.Reset(@out.GetData(), @out.GetLength());
string after2 = @in.ReadUTF();
Assert.Equal(before, after2);
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void GzipConcatTest(Configuration conf, Type decomClass)
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Info(decomClass + " seed: " + seed);
int Concat = r.Next(4) + 3;
int Buflen = 128 * 1024;
DataOutputBuffer dflbuf = new DataOutputBuffer();
DataOutputBuffer chkbuf = new DataOutputBuffer();
byte[] b = new byte[Buflen];
for (int i = 0; i < Concat; ++i)
{
GZIPOutputStream gzout = new GZIPOutputStream(dflbuf);
r.NextBytes(b);
int len = r.Next(Buflen);
int off = r.Next(Buflen - len);
chkbuf.Write(b, off, len);
gzout.Write(b, off, len);
gzout.Close();
}
byte[] chk = Arrays.CopyOf(chkbuf.GetData(), chkbuf.GetLength());
CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf);
Decompressor decom = codec.CreateDecompressor();
NUnit.Framework.Assert.IsNotNull(decom);
Assert.Equal(decomClass, decom.GetType());
DataInputBuffer gzbuf = new DataInputBuffer();
gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength());
InputStream gzin = codec.CreateInputStream(gzbuf, decom);
dflbuf.Reset();
IOUtils.CopyBytes(gzin, dflbuf, 4096);
byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength());
Assert.AssertArrayEquals(chk, dflchk);
}
public virtual void TestReadWriteReplicaState()
{
try
{
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
foreach (HdfsServerConstants.ReplicaState repState in HdfsServerConstants.ReplicaState
.Values())
{
repState.Write(@out);
@in.Reset(@out.GetData(), @out.GetLength());
HdfsServerConstants.ReplicaState result = HdfsServerConstants.ReplicaState.Read(@in
);
NUnit.Framework.Assert.IsTrue("testReadWrite error !!!", repState == result);
@out.Reset();
@in.Reset();
}
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("testReadWrite ex error ReplicaState");
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Write(K key, V value)
{
if (key.GetType() != keyClass)
{
throw new IOException("wrong key class: " + key.GetType() + " is not " + keyClass
);
}
if (value.GetType() != valueClass)
{
throw new IOException("wrong value class: " + value.GetType() + " is not " + valueClass
);
}
// Append the 'key'
keySerializer.Serialize(key);
int keyLength = dataBuffer.GetLength();
if (keyLength < 0)
{
throw new IOException("Negative key-length not allowed: " + keyLength + " for " +
key);
}
// Append the 'value'
valueSerializer.Serialize(value);
int valueLength = dataBuffer.GetLength() - keyLength;
if (valueLength < 0)
{
throw new IOException("Negative value-length not allowed: " + valueLength + " for "
+ value);
}
// Write the record out
WritableUtils.WriteVInt(outputStream, keyLength);
WritableUtils.WriteVInt(outputStream, valueLength);
outputStream.Write(dataBuffer.GetData(), 0, dataBuffer.GetLength());
// Reset
dataBuffer.Reset();
}
/// <exception cref="System.IO.IOException"/>
protected internal override void FlushAndSync(bool durable)
{
// EditLogOutputStream
System.Diagnostics.Debug.Assert(@out.GetLength() == 0, "Output buffer is not empty"
);
if (doubleBuf.IsFlushed())
{
Log.Info("Nothing to flush");
return;
}
int numReadyTxns = doubleBuf.CountReadyTxns();
long firstTxToFlush = doubleBuf.GetFirstReadyTxId();
doubleBuf.FlushTo(@out);
if (@out.GetLength() > 0)
{
System.Diagnostics.Debug.Assert(numReadyTxns > 0);
byte[] data = Arrays.CopyOf(@out.GetData(), @out.GetLength());
@out.Reset();
System.Diagnostics.Debug.Assert(@out.GetLength() == 0, "Output buffer is not empty"
);
backupNode.Journal(journalInfo, 0, firstTxToFlush, numReadyTxns, data);
}
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void WriteObject(Writable obj, DataOutputStream stream
)
{
// For Text and BytesWritable, encode them directly, so that they end up
// in C++ as the natural translations.
DataOutputBuffer buffer = new DataOutputBuffer();
if (obj is Text)
{
Text t = (Text)obj;
int len = t.GetLength();
WritableUtils.WriteVLong(stream, len);
stream.Flush();
stream.Write(t.GetBytes(), 0, len);
stream.Flush();
}
else
{
if (obj is BytesWritable)
{
BytesWritable b = (BytesWritable)obj;
int len = b.GetLength();
WritableUtils.WriteVLong(stream, len);
stream.Write(b.GetBytes(), 0, len);
}
else
{
buffer.Reset();
obj.Write(buffer);
int length = buffer.GetLength();
WritableUtils.WriteVInt(stream, length);
stream.Write(buffer.GetData(), 0, length);
}
}
stream.Flush();
}
private void CheckOnReadWrite()
{
string line = "qryqeb354645rghdfvbaq23312fg";
DataOutputBuffer @out = new DataOutputBuffer();
DataInputBuffer @in = new DataInputBuffer();
Key originKey = new Key(Runtime.GetBytesForString(line), 100d);
try
{
originKey.Write(@out);
@in.Reset(@out.GetData(), @out.GetData().Length);
Key restoredKey = new Key(new byte[] { 0 });
NUnit.Framework.Assert.IsFalse("checkOnReadWrite equals error", restoredKey.Equals
(originKey));
restoredKey.ReadFields(@in);
Assert.True("checkOnReadWrite equals error", restoredKey.Equals
(originKey));
@out.Reset();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("checkOnReadWrite ex error");
}
}
/// <exception cref="System.IO.IOException"/>
private void ReadTest(FileSystem fs, int count, int seed, Path file)
{
Log.Debug("reading " + count + " records");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
RandomDatum k = new RandomDatum();
RandomDatum v = new RandomDatum();
DataOutputBuffer rawKey = new DataOutputBuffer();
SequenceFile.ValueBytes rawValue = reader.CreateValueBytes();
for (int i = 0; i < count; i++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
try
{
if ((i % 5) == 0)
{
// Testing 'raw' apis
rawKey.Reset();
reader.NextRaw(rawKey, rawValue);
}
else
{
// Testing 'non-raw' apis
if ((i % 2) == 0)
{
reader.Next(k);
reader.GetCurrentValue(v);
}
else
{
reader.Next(k, v);
}
// Check
if (!k.Equals(key))
{
throw new RuntimeException("wrong key at " + i);
}
if (!v.Equals(value))
{
throw new RuntimeException("wrong value at " + i);
}
}
}
catch (IOException ioe)
{
Log.Info("Problem on row " + i);
Log.Info("Expected key = " + key);
Log.Info("Expected len = " + key.GetLength());
Log.Info("Actual key = " + k);
Log.Info("Actual len = " + k.GetLength());
Log.Info("Expected value = " + value);
Log.Info("Expected len = " + value.GetLength());
Log.Info("Actual value = " + v);
Log.Info("Actual len = " + v.GetLength());
Log.Info("Key equals: " + k.Equals(key));
Log.Info("value equals: " + v.Equals(value));
throw;
}
}
reader.Close();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestBinary()
{
Configuration conf = new Configuration();
Job job = Job.GetInstance(conf);
Path outdir = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "outseq");
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
FileOutputFormat.SetOutputPath(job, outdir);
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputKeyClass(job, typeof(IntWritable
));
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputValueClass(job, typeof(DoubleWritable
));
SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true);
SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
.Block);
BytesWritable bkey = new BytesWritable();
BytesWritable bval = new BytesWritable();
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
.GetConfiguration());
OutputFormat <BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat
();
OutputCommitter committer = outputFormat.GetOutputCommitter(context);
committer.SetupJob(job);
RecordWriter <BytesWritable, BytesWritable> writer = outputFormat.GetRecordWriter(
context);
IntWritable iwritable = new IntWritable();
DoubleWritable dwritable = new DoubleWritable();
DataOutputBuffer outbuf = new DataOutputBuffer();
Log.Info("Creating data by SequenceFileAsBinaryOutputFormat");
try
{
for (int i = 0; i < Records; ++i)
{
iwritable = new IntWritable(r.Next());
iwritable.Write(outbuf);
bkey.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
dwritable = new DoubleWritable(r.NextDouble());
dwritable.Write(outbuf);
bval.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
writer.Write(bkey, bval);
}
}
finally
{
writer.Close(context);
}
committer.CommitTask(context);
committer.CommitJob(job);
InputFormat <IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat <IntWritable
, DoubleWritable>();
int count = 0;
r.SetSeed(seed);
SequenceFileInputFormat.SetInputPaths(job, outdir);
Log.Info("Reading data by SequenceFileInputFormat");
foreach (InputSplit split in iformat.GetSplits(job))
{
RecordReader <IntWritable, DoubleWritable> reader = iformat.CreateRecordReader(split
, context);
MapContext <IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext =
new MapContextImpl <IntWritable, DoubleWritable, BytesWritable, BytesWritable>(job
.GetConfiguration(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil
.CreateDummyReporter(), split);
reader.Initialize(split, mcontext);
try
{
int sourceInt;
double sourceDouble;
while (reader.NextKeyValue())
{
sourceInt = r.Next();
sourceDouble = r.NextDouble();
iwritable = reader.GetCurrentKey();
dwritable = reader.GetCurrentValue();
NUnit.Framework.Assert.AreEqual("Keys don't match: " + "*" + iwritable.Get() + ":"
+ sourceInt + "*", sourceInt, iwritable.Get());
NUnit.Framework.Assert.IsTrue("Vals don't match: " + "*" + dwritable.Get() + ":"
+ sourceDouble + "*", double.Compare(dwritable.Get(), sourceDouble) == 0);
++count;
}
}
finally
{
reader.Close();
}
}
NUnit.Framework.Assert.AreEqual("Some records not found", Records, count);
}
// A random task attempt id for testing.
/// <exception cref="System.IO.IOException"/>
public virtual void TestBinary()
{
JobConf job = new JobConf();
FileSystem fs = FileSystem.GetLocal(job);
Path dir = new Path(new Path(new Path(Runtime.GetProperty("test.build.data", ".")
), FileOutputCommitter.TempDirName), "_" + attempt);
Path file = new Path(dir, "testbinary.seq");
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
fs.Delete(dir, true);
if (!fs.Mkdirs(dir))
{
Fail("Failed to create output directory");
}
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, dir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, dir);
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputKeyClass(job, typeof(IntWritable
));
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputValueClass(job, typeof(DoubleWritable
));
SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true);
SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
.Block);
BytesWritable bkey = new BytesWritable();
BytesWritable bval = new BytesWritable();
RecordWriter <BytesWritable, BytesWritable> writer = new SequenceFileAsBinaryOutputFormat
().GetRecordWriter(fs, job, file.ToString(), Reporter.Null);
IntWritable iwritable = new IntWritable();
DoubleWritable dwritable = new DoubleWritable();
DataOutputBuffer outbuf = new DataOutputBuffer();
Log.Info("Creating data by SequenceFileAsBinaryOutputFormat");
try
{
for (int i = 0; i < Records; ++i)
{
iwritable = new IntWritable(r.Next());
iwritable.Write(outbuf);
bkey.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
dwritable = new DoubleWritable(r.NextDouble());
dwritable.Write(outbuf);
bval.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
writer.Write(bkey, bval);
}
}
finally
{
writer.Close(Reporter.Null);
}
InputFormat <IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat <IntWritable
, DoubleWritable>();
int count = 0;
r.SetSeed(seed);
DataInputBuffer buf = new DataInputBuffer();
int NumSplits = 3;
SequenceFileInputFormat.AddInputPath(job, file);
Log.Info("Reading data by SequenceFileInputFormat");
foreach (InputSplit split in iformat.GetSplits(job, NumSplits))
{
RecordReader <IntWritable, DoubleWritable> reader = iformat.GetRecordReader(split,
job, Reporter.Null);
try
{
int sourceInt;
double sourceDouble;
while (reader.Next(iwritable, dwritable))
{
sourceInt = r.Next();
sourceDouble = r.NextDouble();
NUnit.Framework.Assert.AreEqual("Keys don't match: " + "*" + iwritable.Get() + ":"
+ sourceInt + "*", sourceInt, iwritable.Get());
NUnit.Framework.Assert.IsTrue("Vals don't match: " + "*" + dwritable.Get() + ":"
+ sourceDouble + "*", double.Compare(dwritable.Get(), sourceDouble) == 0);
++count;
}
}
finally
{
reader.Close();
}
}
NUnit.Framework.Assert.AreEqual("Some records not found", Records, count);
}
public virtual void ResetBuffers()
{
@out.Reset();
@in.Reset();
}