/// <exception cref="System.Exception"/>
public virtual void TestAppendSort()
{
GenericTestUtils.AssumeInNativeProfile();
Path file = new Path(RootPath, "testseqappendSort.seq");
fs.Delete(file, true);
Path sortedFile = new Path(RootPath, "testseqappendSort.seq.sort");
fs.Delete(sortedFile, true);
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, new JavaSerializationComparator
<long>(), typeof(long), typeof(string), conf);
SequenceFile.Writer.Option compressOption = SequenceFile.Writer.Compression(SequenceFile.CompressionType
.Block, new GzipCodec());
SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.
File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass
(typeof(string)), compressOption);
writer.Append(2L, "two");
writer.Append(1L, "one");
writer.Close();
writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer
.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer
.AppendIfExists(true), compressOption);
writer.Append(4L, "four");
writer.Append(3L, "three");
writer.Close();
// Sort file after append
sorter.Sort(file, sortedFile);
VerifyAll4Values(sortedFile);
fs.DeleteOnExit(file);
fs.DeleteOnExit(sortedFile);
}
/// <exception cref="System.IO.IOException"/>
private void SortTest(FileSystem fs, int count, int megabytes, int factor, bool fast
, Path file)
{
fs.Delete(new Path(file + ".sorted"), true);
SequenceFile.Sorter sorter = NewSorter(fs, fast, megabytes, factor);
Log.Debug("sorting " + count + " records");
sorter.Sort(file, file.Suffix(".sorted"));
Log.Info("done sorting " + count + " debug");
}
private SequenceFile.Sorter NewSorter(FileSystem fs, bool fast, int megabytes, int
factor)
{
SequenceFile.Sorter sorter = fast ? new SequenceFile.Sorter(fs, new RandomDatum.Comparator
(), typeof(RandomDatum), typeof(RandomDatum), conf) : new SequenceFile.Sorter(fs
, typeof(RandomDatum), typeof(RandomDatum), conf);
sorter.SetMemory(megabytes * 1024 * 1024);
sorter.SetFactor(factor);
return(sorter);
}
/// <exception cref="System.IO.IOException"/>
private void SortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile,
SequenceFile.Metadata metadata)
{
fs.Delete(sortedFile, true);
Log.Info("sorting: " + unsortedFile + " to: " + sortedFile);
WritableComparator comparator = WritableComparator.Get(typeof(RandomDatum));
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, typeof(RandomDatum
), typeof(RandomDatum), conf, metadata);
sorter.Sort(new Path[] { unsortedFile }, sortedFile, false);
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunTest(SequenceFile.CompressionType compressionType)
{
JobConf job = new JobConf();
FileSystem fs = FileSystem.GetLocal(job);
Path dir = new Path(Runtime.GetProperty("test.build.data", ".") + "/mapred");
Path file = new Path(dir, "test.seq");
Path tempDir = new Path(dir, "tmp");
fs.Delete(dir, true);
FileInputFormat.SetInputPaths(job, dir);
fs.Mkdirs(tempDir);
LongWritable tkey = new LongWritable();
Text tval = new Text();
SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, job, file, typeof(LongWritable
), typeof(Text), compressionType, new DefaultCodec());
try
{
for (int i = 0; i < Records; ++i)
{
tkey.Set(1234);
tval.Set("valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue");
writer.Append(tkey, tval);
}
}
finally
{
writer.Close();
}
long fileLength = fs.GetFileStatus(file).GetLen();
Log.Info("With compression = " + compressionType + ": " + "compressed length = "
+ fileLength);
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, job.GetOutputKeyComparator
(), job.GetMapOutputKeyClass(), job.GetMapOutputValueClass(), job);
Path[] paths = new Path[] { file };
SequenceFile.Sorter.RawKeyValueIterator rIter = sorter.Merge(paths, tempDir, false
);
int count = 0;
while (rIter.Next())
{
count++;
}
NUnit.Framework.Assert.AreEqual(Records, count);
NUnit.Framework.Assert.AreEqual(1.0f, rIter.GetProgress().Get());
}
