/// <summary>Test that comparator is defined and configured.</summary>
/// <exception cref="System.Exception"/>
public static void TestGetComparator()
{
Configuration conf = new Configuration();
// Without conf.
WritableComparator frobComparator = WritableComparator.Get(typeof(TestWritable.Frob
));
System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
NUnit.Framework.Assert.IsNull(frobComparator.GetConf().Get(TestConfigParam));
// With conf.
conf.Set(TestConfigParam, TestConfigValue);
frobComparator = WritableComparator.Get(typeof(TestWritable.Frob), conf);
System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
Assert.Equal(conf.Get(TestConfigParam), TestConfigValue);
// Without conf. should reuse configuration.
frobComparator = WritableComparator.Get(typeof(TestWritable.Frob));
System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
Assert.Equal(conf.Get(TestConfigParam), TestConfigValue);
// New conf. should use new configuration.
frobComparator = WritableComparator.Get(typeof(TestWritable.Frob), new Configuration
());
System.Diagnostics.Debug.Assert((frobComparator is TestWritable.FrobComparator));
NUnit.Framework.Assert.IsNotNull(frobComparator.GetConf());
NUnit.Framework.Assert.IsNull(frobComparator.GetConf().Get(TestConfigParam));
}
public Writer(Configuration conf, FileSystem fs, string dirName, WritableComparator
comparator, Type valClass, SequenceFile.CompressionType compress, Progressable
progress)
: this(conf, new Path(dirName), Comparator(comparator), ValueClass(valClass), Compression
(compress), Progressable(progress))
{
}
/// <exception cref="System.Exception"/>
public virtual void TestCoding()
{
string before = "Bad \t encoding \t testcase";
Org.Apache.Hadoop.IO.Text text = new Org.Apache.Hadoop.IO.Text(before);
string after = text.ToString();
Assert.True(before.Equals(after));
for (int i = 0; i < NumIterations; i++)
{
// generate a random string
if (i == 0)
{
before = GetLongString();
}
else
{
before = GetTestString();
}
// test string to utf8
ByteBuffer bb = Org.Apache.Hadoop.IO.Text.Encode(before);
byte[] utf8Text = ((byte[])bb.Array());
byte[] utf8Java = Runtime.GetBytesForString(before, "UTF-8");
Assert.Equal(0, WritableComparator.CompareBytes(utf8Text, 0, bb
.Limit(), utf8Java, 0, utf8Java.Length));
// test utf8 to string
after = Org.Apache.Hadoop.IO.Text.Decode(utf8Java);
Assert.True(before.Equals(after));
}
}
/// <exception cref="System.IO.IOException"/>
private void SortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile,
SequenceFile.Metadata metadata)
{
fs.Delete(sortedFile, true);
Log.Info("sorting: " + unsortedFile + " to: " + sortedFile);
WritableComparator comparator = WritableComparator.Get(typeof(RandomDatum));
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, typeof(RandomDatum
), typeof(RandomDatum), conf, metadata);
sorter.Sort(new Path[] { unsortedFile }, sortedFile, false);
}
/// <summary>
/// test
/// <c>BloomMapFile.Writer</c>
/// constructors
/// </summary>
public virtual void TestBloomMapFileConstructors()
{
BloomMapFile.Writer writer = null;
try
{
FileSystem ts = FileSystem.Get(conf);
string testFileName = TestFile.ToString();
writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
Text), SequenceFile.CompressionType.Block, defaultCodec, defaultProgress);
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
Text), SequenceFile.CompressionType.Block, defaultProgress);
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
Text), SequenceFile.CompressionType.Block);
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
Text), SequenceFile.CompressionType.Record, defaultCodec, defaultProgress);
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
Text), SequenceFile.CompressionType.Record, defaultProgress);
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
writer = new BloomMapFile.Writer(conf, ts, testFileName, typeof(IntWritable), typeof(
Text), SequenceFile.CompressionType.Record);
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
writer = new BloomMapFile.Writer(conf, ts, testFileName, WritableComparator.Get(typeof(
Text)), typeof(Text));
NUnit.Framework.Assert.IsNotNull("testBloomMapFileConstructors error !!!", writer
);
writer.Close();
}
catch (Exception)
{
Fail("testBloomMapFileConstructors error !!!");
}
finally
{
IOUtils.Cleanup(null, writer);
}
}
/// <exception cref="System.IO.IOException"/>
private static void WriteTest(FileSystem fs, RandomDatum[] data, string file, SequenceFile.CompressionType
compress)
{
MapFile.Delete(fs, file);
Log.Info("creating with " + data.Length + " records");
SetFile.Writer writer = new SetFile.Writer(conf, fs, file, WritableComparator.Get
(typeof(RandomDatum)), compress);
for (int i = 0; i < data.Length; i++)
{
writer.Append(data[i]);
}
writer.Close();
}
/// <summary>Test that Writable's are configured by Comparator.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestConfigurableWritableComparator()
{
Configuration conf = new Configuration();
conf.Set(TestWritableConfigParam, TestWritableConfigValue);
WritableComparator wc = WritableComparator.Get(typeof(TestWritable.SimpleWritableComparable
), conf);
TestWritable.SimpleWritableComparable key = ((TestWritable.SimpleWritableComparable
)wc.NewKey());
NUnit.Framework.Assert.IsNotNull(wc.GetConf());
NUnit.Framework.Assert.IsNotNull(key.GetConf());
Assert.Equal(key.GetConf().Get(TestWritableConfigParam), TestWritableConfigValue
);
}
public virtual void TestDeprecatedConstructors()
{
string path = new Path(TestDir, "writes.mapfile").ToString();
MapFile.Writer writer = null;
MapFile.Reader reader = null;
try
{
FileSystem fs = FileSystem.GetLocal(conf);
writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
.Record);
NUnit.Framework.Assert.IsNotNull(writer);
writer.Close();
writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
.Record, defaultProgressable);
NUnit.Framework.Assert.IsNotNull(writer);
writer.Close();
writer = new MapFile.Writer(conf, fs, path, typeof(IntWritable), typeof(Text), SequenceFile.CompressionType
.Record, defaultCodec, defaultProgressable);
NUnit.Framework.Assert.IsNotNull(writer);
writer.Close();
writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
typeof(Text));
NUnit.Framework.Assert.IsNotNull(writer);
writer.Close();
writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
typeof(Text), SequenceFile.CompressionType.Record);
NUnit.Framework.Assert.IsNotNull(writer);
writer.Close();
writer = new MapFile.Writer(conf, fs, path, WritableComparator.Get(typeof(Text)),
typeof(Text), SequenceFile.CompressionType.Record, defaultProgressable);
NUnit.Framework.Assert.IsNotNull(writer);
writer.Close();
reader = new MapFile.Reader(fs, path, WritableComparator.Get(typeof(IntWritable))
, conf);
NUnit.Framework.Assert.IsNotNull(reader);
NUnit.Framework.Assert.IsNotNull("reader key is null !!!", reader.GetKeyClass());
NUnit.Framework.Assert.IsNotNull("reader value in null", reader.GetValueClass());
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/// <summary>
/// Test a user comparator that relies on deserializing both arguments for each
/// compare.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestShortWritableComparator()
{
ShortWritable writable1 = new ShortWritable((short)256);
ShortWritable writable2 = new ShortWritable((short)128);
ShortWritable writable3 = new ShortWritable((short)256);
string ShouldNotMatchWithResultOne = "Result should be 1, should not match the writables";
Assert.True(ShouldNotMatchWithResultOne, writable1.CompareTo(writable2
) == 1);
Assert.True(ShouldNotMatchWithResultOne, WritableComparator.Get
(typeof(ShortWritable)).Compare(writable1, writable2) == 1);
string ShouldNotMatchWithResultMinusOne = "Result should be -1, should not match the writables";
Assert.True(ShouldNotMatchWithResultMinusOne, writable2.CompareTo
(writable1) == -1);
Assert.True(ShouldNotMatchWithResultMinusOne, WritableComparator
.Get(typeof(ShortWritable)).Compare(writable2, writable1) == -1);
string ShouldMatch = "Result should be 0, should match the writables";
Assert.True(ShouldMatch, writable1.CompareTo(writable1) == 0);
Assert.True(ShouldMatch, WritableComparator.Get(typeof(ShortWritable
)).Compare(writable1, writable3) == 0);
}
public Writer(Configuration conf, FileSystem fs, string dirName, WritableComparator
comparator, Type valClass)
: this(conf, new Path(dirName), Comparator(comparator), ValueClass(valClass))
{
}
static MD5Hash()
{
// register this comparator
WritableComparator.Define(typeof(MD5Hash), new MD5Hash.Comparator());
}
/// <summary>Compares this object with the specified object for order.</summary>
public virtual int CompareTo(Org.Apache.Hadoop.IO.MD5Hash that)
{
return(WritableComparator.CompareBytes(this.digest, 0, Md5Len, that.digest, 0, Md5Len
));
}
static ShortWritable()
{
// register this comparator
WritableComparator.Define(typeof(ShortWritable), new ShortWritable.Comparator());
}
static NullWritable()
{
// register this comparator
WritableComparator.Define(typeof(NullWritable), new NullWritable.Comparator());
}
public virtual int CompareTo(Org.Apache.Hadoop.IO.RandomDatum o)
{
return(WritableComparator.CompareBytes(this.data, 0, this.length, o.data, 0, o.length
));
}
/// <exception cref="System.IO.IOException"/>
private SetFile.Reader CreateReader(FileSystem fs)
{
return(new SetFile.Reader(fs, File, WritableComparator.Get(typeof(IntWritable)),
conf));
}
static Frob()
{
// register default comparator
WritableComparator.Define(typeof(TestWritable.Frob), new TestWritable.FrobComparator
());
}
