public virtual void SetUp()
{
path = new Path(Root, outputFile);
fs = path.GetFileSystem(conf);
@out = fs.Create(path);
writer = new TFile.Writer(@out, BlockSize, compression, comparator, conf);
}
/// <exception cref="System.IO.IOException"/>
protected override void SetUp()
{
conf = new Configuration();
path = new Path(Root, outputFile);
fs = path.GetFileSystem(conf);
@out = fs.Create(path);
writer = new TFile.Writer(@out, BlockSize, compression, comparator, conf);
}
// write empty keys and values
/// <exception cref="System.IO.IOException"/>
private void WriteEmptyRecords(TFile.Writer writer, int n)
{
byte[] key = new byte[0];
byte[] value = new byte[0];
for (int i = 0; i < n; i++)
{
writer.Append(key, value);
}
}
/// <exception cref="System.IO.IOException"/>
private void WriteRecords(TFile.Writer writer)
{
WriteEmptyRecords(writer, 10);
int ret = WriteSomeRecords(writer, 0, 100);
ret = WriteLargeRecords(writer, ret, 1);
ret = WritePrepWithKnownLength(writer, ret, 40);
ret = WritePrepWithUnkownLength(writer, ret, 50);
writer.Close();
}
/// <exception cref="System.IO.IOException"/>
private void WriteNumMetablocks(TFile.Writer writer, string compression, int n)
{
for (int i = 0; i < n; i++)
{
DataOutputStream dout = writer.PrepareMetaBlock("TfileMeta" + i, compression);
byte[] b = Runtime.GetBytesForString(("something to test" + i));
dout.Write(b);
dout.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void CloseOutput()
{
if (writer != null)
{
writer.Close();
writer = null;
@out.Close();
@out = null;
}
}
// write some large records
// write them twice
/// <exception cref="System.IO.IOException"/>
private int WriteLargeRecords(TFile.Writer writer, int start, int n)
{
byte[] value = new byte[largeVal];
for (int i = start; i < (start + n); i++)
{
string key = string.Format(localFormatter, i);
writer.Append(Runtime.GetBytesForString(key), value);
writer.Append(Runtime.GetBytesForString(key), value);
}
return(start + n);
}
// bad comparator format
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureBadComparatorNames()
{
try
{
writer = new TFile.Writer(@out, BlockSize, compression, "badcmp", conf);
NUnit.Framework.Assert.Fail("Failed to catch unsupported comparator names");
}
catch (Exception e)
{
// noop, expecting exceptions
Runtime.PrintStackTrace(e);
}
}
/// <summary>test none codecs</summary>
/// <exception cref="System.IO.IOException"/>
internal virtual void BasicWithSomeCodec(string codec)
{
Path ncTFile = new Path(Root, "basic.tfile");
FSDataOutputStream fout = CreateFSOutput(ncTFile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, "memcmp", conf);
WriteRecords(writer);
fout.Close();
FSDataInputStream fin = fs.Open(ncTFile);
TFile.Reader reader = new TFile.Reader(fs.Open(ncTFile), fs.GetFileStatus(ncTFile
).GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.SeekTo(GetSomeKey(50));
Assert.True("location lookup failed", scanner.SeekTo(GetSomeKey
(50)));
// read the key and see if it matches
byte[] readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(GetSomeKey
(50), readKey));
scanner.SeekTo(new byte[0]);
byte[] val1 = ReadValue(scanner);
scanner.SeekTo(new byte[0]);
byte[] val2 = ReadValue(scanner);
Assert.True(Arrays.Equals(val1, val2));
// check for lowerBound
scanner.LowerBound(GetSomeKey(50));
Assert.True("locaton lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(50)));
// check for upper bound
scanner.UpperBound(GetSomeKey(50));
Assert.True("location lookup failed", scanner.currentLocation.CompareTo
(reader.End()) < 0);
readKey = ReadKey(scanner);
Assert.True("seeked key does not match", Arrays.Equals(readKey,
GetSomeKey(51)));
scanner.Close();
// test for a range of scanner
scanner = reader.CreateScannerByKey(GetSomeKey(10), GetSomeKey(60));
ReadAndCheckbytes(scanner, 10, 50);
NUnit.Framework.Assert.IsFalse(scanner.Advance());
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(ncTFile, true);
}
// class exists but not a RawComparator
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureBadJClasses()
{
try
{
writer = new TFile.Writer(@out, BlockSize, compression, "jclass:org.apache.hadoop.io.file.tfile.Chunk"
, conf);
NUnit.Framework.Assert.Fail("Failed to catch unsupported comparator names");
}
catch (Exception e)
{
// noop, expecting exceptions
Runtime.PrintStackTrace(e);
}
}
// write some records into the tfile
// write them twice
/// <exception cref="System.IO.IOException"/>
private int WriteSomeRecords(TFile.Writer writer, int start, int n)
{
string value = "value";
for (int i = start; i < (start + n); i++)
{
string key = string.Format(localFormatter, i);
writer.Append(Runtime.GetBytesForString(key), Runtime.GetBytesForString
((value + key)));
writer.Append(Runtime.GetBytesForString(key), Runtime.GetBytesForString
((value + key)));
}
return(start + n);
}
/// <exception cref="System.IO.IOException"/>
private void CreateTFile()
{
long totalBytes = 0;
FSDataOutputStream fout = CreateFSOutput(path, fs);
try
{
TFile.Writer writer = new TFile.Writer(fout, options.minBlockSize, options.compress
, "memcmp", conf);
try
{
BytesWritable key = new BytesWritable();
BytesWritable val = new BytesWritable();
timer.Start();
for (long i = 0; true; ++i)
{
if (i % 1000 == 0)
{
// test the size for every 1000 rows.
if (fs.GetFileStatus(path).GetLen() >= options.fileSize)
{
break;
}
}
kvGen.Next(key, val, false);
writer.Append(key.Get(), 0, key.GetSize(), val.Get(), 0, val.GetSize());
totalBytes += key.GetSize();
totalBytes += val.GetSize();
}
timer.Stop();
}
finally
{
writer.Close();
}
}
finally
{
fout.Close();
}
double duration = (double)timer.Read() / 1000;
// in us.
long fsize = fs.GetFileStatus(path).GetLen();
System.Console.Out.Printf("time: %s...uncompressed: %.2fMB...raw thrpt: %.2fMB/s\n"
, timer.ToString(), (double)totalBytes / 1024 / 1024, totalBytes / duration);
System.Console.Out.Printf("time: %s...file size: %.2fMB...disk thrpt: %.2fMB/s\n"
, timer.ToString(), (double)fsize / 1024 / 1024, fsize / duration);
}
/// <exception cref="System.IO.IOException"/>
private int WritePrepWithUnkownLength(TFile.Writer writer, int start, int n)
{
for (int i = start; i < (start + n); i++)
{
DataOutputStream @out = writer.PrepareAppendKey(-1);
string localKey = string.Format(localFormatter, i);
@out.Write(Runtime.GetBytesForString(localKey));
@out.Close();
string value = "value" + localKey;
@out = writer.PrepareAppendValue(-1);
@out.Write(Runtime.GetBytesForString(value));
@out.Close();
}
return(start + n);
}
/// <exception cref="System.IO.IOException"/>
internal static long WriteRecords(TFile.Writer writer, int count)
{
long rawDataSize = 0;
int nx;
for (nx = 0; nx < count; nx++)
{
byte[] key = Runtime.GetBytesForString(ComposeSortedKey(Key, nx));
byte[] value = Runtime.GetBytesForString((Value + nx));
writer.Append(key, value);
rawDataSize += WritableUtils.GetVIntSize(key.Length) + key.Length + WritableUtils
.GetVIntSize(value.Length) + value.Length;
}
return(rawDataSize);
}
/// <exception cref="System.IO.IOException"/>
protected override void SetUp()
{
conf = new Configuration();
path = new Path(Root, outputFile);
fs = path.GetFileSystem(conf);
@out = fs.Create(path);
writer = new TFile.Writer(@out, BlockSize, compression, null, conf);
writer.Append(Runtime.GetBytesForString("keyZ"), Runtime.GetBytesForString
("valueZ"));
writer.Append(Runtime.GetBytesForString("keyM"), Runtime.GetBytesForString
("valueM"));
writer.Append(Runtime.GetBytesForString("keyN"), Runtime.GetBytesForString
("valueN"));
writer.Append(Runtime.GetBytesForString("keyA"), Runtime.GetBytesForString
("valueA"));
CloseOutput();
}
/// <exception cref="System.IO.IOException"/>
private void SomeTestingWithMetaBlock(TFile.Writer writer, string compression)
{
DataOutputStream dout = null;
WriteNumMetablocks(writer, compression, 10);
try
{
dout = writer.PrepareMetaBlock("TfileMeta1", compression);
Assert.True(false);
}
catch (MetaBlockAlreadyExists)
{
}
// avoid this exception
dout = writer.PrepareMetaBlock("TFileMeta100", compression);
dout.Close();
}
public virtual void TestFailureBadCompressionCodec()
{
if (skip)
{
return;
}
CloseOutput();
@out = fs.Create(path);
try
{
writer = new TFile.Writer(@out, BlockSize, "BAD", comparator, conf);
NUnit.Framework.Assert.Fail("Error on handling invalid compression codecs.");
}
catch (Exception)
{
}
}
// test meta blocks for tfiles
/// <exception cref="System.IO.IOException"/>
public virtual void TestMetaBlocks()
{
Path mFile = new Path(Root, "meta.tfile");
FSDataOutputStream fout = CreateFSOutput(mFile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, "none", null, conf);
SomeTestingWithMetaBlock(writer, "none");
writer.Close();
fout.Close();
FSDataInputStream fin = fs.Open(mFile);
TFile.Reader reader = new TFile.Reader(fin, fs.GetFileStatus(mFile).GetLen(), conf
);
SomeReadingWithMetaBlock(reader);
fs.Delete(mFile, true);
reader.Close();
fin.Close();
}
/// <exception cref="System.IO.IOException"/>
public HistoryFileWriter(FileSystemApplicationHistoryStore _enclosing, Path historyFile
)
{
this._enclosing = _enclosing;
if (this._enclosing.fs.Exists(historyFile))
{
this.fsdos = this._enclosing.fs.Append(historyFile);
}
else
{
this.fsdos = this._enclosing.fs.Create(historyFile);
}
this._enclosing.fs.SetPermission(historyFile, FileSystemApplicationHistoryStore.HistoryFileUmask
);
this.writer = new TFile.Writer(this.fsdos, FileSystemApplicationHistoryStore.MinBlockSize
, this._enclosing.GetConfig().Get(YarnConfiguration.FsApplicationHistoryStoreCompressionType
, YarnConfiguration.DefaultFsApplicationHistoryStoreCompressionType), null, this
._enclosing.GetConfig());
}
/// <exception cref="System.IO.IOException"/>
internal virtual void CreateFile(int count, string compress)
{
conf = new Configuration();
path = new Path(Root, outputFile + "." + compress);
fs = path.GetFileSystem(conf);
FSDataOutputStream @out = fs.Create(path);
TFile.Writer writer = new TFile.Writer(@out, BlockSize, compress, comparator, conf
);
int nx;
for (nx = 0; nx < count; nx++)
{
byte[] key = Runtime.GetBytesForString(ComposeSortedKey(Key, count, nx));
byte[] value = Runtime.GetBytesForString((Value + nx));
writer.Append(key, value);
}
writer.Close();
@out.Close();
}
/// <exception cref="System.IO.IOException"/>
public LogWriter(Configuration conf, Path remoteAppLogFile, UserGroupInformation
userUgi)
{
try
{
this.fsDataOStream = userUgi.DoAs(new _PrivilegedExceptionAction_379(this, conf,
remoteAppLogFile));
}
catch (Exception e)
{
throw new IOException(e);
}
// Keys are not sorted: null arg
// 256KB minBlockSize : Expected log size for each container too
this.writer = new TFile.Writer(this.fsDataOStream, 256 * 1024, conf.Get(YarnConfiguration
.NmLogAggCompressionType, YarnConfiguration.DefaultNmLogAggCompressionType), null
, conf);
//Write the version string
WriteVersion();
}
// unsorted with some codec
/// <exception cref="System.IO.IOException"/>
internal virtual void UnsortedWithSomeCodec(string codec)
{
Path uTfile = new Path(Root, "unsorted.tfile");
FSDataOutputStream fout = CreateFSOutput(uTfile);
TFile.Writer writer = new TFile.Writer(fout, minBlockSize, codec, null, conf);
WriteRecords(writer);
writer.Close();
fout.Close();
FSDataInputStream fin = fs.Open(uTfile);
TFile.Reader reader = new TFile.Reader(fs.Open(uTfile), fs.GetFileStatus(uTfile).
GetLen(), conf);
TFile.Reader.Scanner scanner = reader.CreateScanner();
ReadAllRecords(scanner);
scanner.Close();
reader.Close();
fin.Close();
fs.Delete(uTfile, true);
}
public virtual void TestFailureFileWriteNotAt0Position()
{
if (skip)
{
return;
}
CloseOutput();
@out = fs.Create(path);
@out.Write(123);
try
{
writer = new TFile.Writer(@out, BlockSize, compression, comparator, conf);
NUnit.Framework.Assert.Fail("Failed to catch file write not at position 0.");
}
catch (Exception)
{
}
// noop, expecting exceptions
CloseOutput();
}
/// <exception cref="System.IO.IOException"/>
private int WritePrepWithKnownLength(TFile.Writer writer, int start, int n)
{
// get the length of the key
string key = string.Format(localFormatter, start);
int keyLen = Runtime.GetBytesForString(key).Length;
string value = "value" + key;
int valueLen = Runtime.GetBytesForString(value).Length;
for (int i = start; i < (start + n); i++)
{
DataOutputStream @out = writer.PrepareAppendKey(keyLen);
string localKey = string.Format(localFormatter, i);
@out.Write(Runtime.GetBytesForString(localKey));
@out.Close();
@out = writer.PrepareAppendValue(valueLen);
string localValue = "value" + localKey;
@out.Write(Runtime.GetBytesForString(localValue));
@out.Close();
}
return(start + n);
}
/// <exception cref="System.IO.IOException"/>
public TFileAppendable(FileSystem fs, Path path, string compress, int minBlkSize,
int osBufferSize, Configuration conf)
{
this.fsdos = fs.Create(path, true, osBufferSize);
this.writer = new TFile.Writer(fsdos, minBlkSize, compress, null, conf);
}
public virtual void TestSortedLongWritable()
{
Configuration conf = new Configuration();
Path path = new Path(Root, name);
FileSystem fs = path.GetFileSystem(conf);
FSDataOutputStream @out = fs.Create(path);
try
{
TFile.Writer writer = new TFile.Writer(@out, BlockSize, "gz", jClassLongWritableComparator
, conf);
try
{
LongWritable key = new LongWritable(0);
for (long i = 0; i < Nentry; ++i)
{
key.Set(Cube(i - Nentry / 2));
DataOutputStream dos = writer.PrepareAppendKey(-1);
try
{
key.Write(dos);
}
finally
{
dos.Close();
}
dos = writer.PrepareAppendValue(-1);
try
{
dos.Write(Runtime.GetBytesForString(BuildValue(i)));
}
finally
{
dos.Close();
}
}
}
finally
{
writer.Close();
}
}
finally
{
@out.Close();
}
FSDataInputStream @in = fs.Open(path);
try
{
TFile.Reader reader = new TFile.Reader(@in, fs.GetFileStatus(path).GetLen(), conf
);
try
{
TFile.Reader.Scanner scanner = reader.CreateScanner();
long i = 0;
BytesWritable value = new BytesWritable();
for (; !scanner.AtEnd(); scanner.Advance())
{
scanner.Entry().GetValue(value);
Assert.Equal(BuildValue(i), Runtime.GetStringForBytes(
value.GetBytes(), 0, value.GetLength()));
++i;
}
}
finally
{
reader.Close();
}
}
finally
{
@in.Close();
}
}
