/// <exception cref="System.IO.IOException"/>
public override void Commit()
{
fs.Rename(tmpOutputPath, outputPath);
MergeManagerImpl.CompressAwarePath compressAwarePath = new MergeManagerImpl.CompressAwarePath
(outputPath, GetSize(), this.compressedSize);
merger.CloseOnDiskFile(compressAwarePath);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
public virtual void TestOnDiskMerger()
{
JobConf jobConf = new JobConf();
int SortFactor = 5;
jobConf.SetInt(MRJobConfig.IoSortFactor, SortFactor);
MapOutputFile mapOutputFile = new MROutputFiles();
FileSystem fs = FileSystem.GetLocal(jobConf);
MergeManagerImpl <IntWritable, IntWritable> manager = new MergeManagerImpl <IntWritable
, IntWritable>(null, jobConf, fs, null, null, null, null, null, null, null, null
, null, null, mapOutputFile);
MergeThread <MapOutput <IntWritable, IntWritable>, IntWritable, IntWritable> onDiskMerger
= (MergeThread <MapOutput <IntWritable, IntWritable>, IntWritable, IntWritable>)Whitebox
.GetInternalState(manager, "onDiskMerger");
int mergeFactor = (int)Whitebox.GetInternalState(onDiskMerger, "mergeFactor");
// make sure the io.sort.factor is set properly
NUnit.Framework.Assert.AreEqual(mergeFactor, SortFactor);
// Stop the onDiskMerger thread so that we can intercept the list of files
// waiting to be merged.
onDiskMerger.Suspend();
//Send the list of fake files waiting to be merged
Random rand = new Random();
for (int i = 0; i < 2 * SortFactor; ++i)
{
Path path = new Path("somePath");
MergeManagerImpl.CompressAwarePath cap = new MergeManagerImpl.CompressAwarePath(path
, 1l, rand.Next());
manager.CloseOnDiskFile(cap);
}
//Check that the files pending to be merged are in sorted order.
List <IList <MergeManagerImpl.CompressAwarePath> > pendingToBeMerged = (List <IList <MergeManagerImpl.CompressAwarePath
> >)Whitebox.GetInternalState(onDiskMerger, "pendingToBeMerged");
NUnit.Framework.Assert.IsTrue("No inputs were added to list pending to merge", pendingToBeMerged
.Count > 0);
for (int i_1 = 0; i_1 < pendingToBeMerged.Count; ++i_1)
{
IList <MergeManagerImpl.CompressAwarePath> inputs = pendingToBeMerged[i_1];
for (int j = 1; j < inputs.Count; ++j)
{
NUnit.Framework.Assert.IsTrue("Not enough / too many inputs were going to be merged"
, inputs.Count > 0 && inputs.Count <= SortFactor);
NUnit.Framework.Assert.IsTrue("Inputs to be merged were not sorted according to size: "
, inputs[j].GetCompressedSize() >= inputs[j - 1].GetCompressedSize());
}
}
}