/// <exception cref="System.IO.IOException"/> public override void Commit() { fs.Rename(tmpOutputPath, outputPath); MergeManagerImpl.CompressAwarePath compressAwarePath = new MergeManagerImpl.CompressAwarePath (outputPath, GetSize(), this.compressedSize); merger.CloseOnDiskFile(compressAwarePath); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> public virtual void TestOnDiskMerger() { JobConf jobConf = new JobConf(); int SortFactor = 5; jobConf.SetInt(MRJobConfig.IoSortFactor, SortFactor); MapOutputFile mapOutputFile = new MROutputFiles(); FileSystem fs = FileSystem.GetLocal(jobConf); MergeManagerImpl <IntWritable, IntWritable> manager = new MergeManagerImpl <IntWritable , IntWritable>(null, jobConf, fs, null, null, null, null, null, null, null, null , null, null, mapOutputFile); MergeThread <MapOutput <IntWritable, IntWritable>, IntWritable, IntWritable> onDiskMerger = (MergeThread <MapOutput <IntWritable, IntWritable>, IntWritable, IntWritable>)Whitebox .GetInternalState(manager, "onDiskMerger"); int mergeFactor = (int)Whitebox.GetInternalState(onDiskMerger, "mergeFactor"); // make sure the io.sort.factor is set properly NUnit.Framework.Assert.AreEqual(mergeFactor, SortFactor); // Stop the onDiskMerger thread so that we can intercept the list of files // waiting to be merged. onDiskMerger.Suspend(); //Send the list of fake files waiting to be merged Random rand = new Random(); for (int i = 0; i < 2 * SortFactor; ++i) { Path path = new Path("somePath"); MergeManagerImpl.CompressAwarePath cap = new MergeManagerImpl.CompressAwarePath(path , 1l, rand.Next()); manager.CloseOnDiskFile(cap); } //Check that the files pending to be merged are in sorted order. List <IList <MergeManagerImpl.CompressAwarePath> > pendingToBeMerged = (List <IList <MergeManagerImpl.CompressAwarePath > >)Whitebox.GetInternalState(onDiskMerger, "pendingToBeMerged"); NUnit.Framework.Assert.IsTrue("No inputs were added to list pending to merge", pendingToBeMerged .Count > 0); for (int i_1 = 0; i_1 < pendingToBeMerged.Count; ++i_1) { IList <MergeManagerImpl.CompressAwarePath> inputs = pendingToBeMerged[i_1]; for (int j = 1; j < inputs.Count; ++j) { NUnit.Framework.Assert.IsTrue("Not enough / too many inputs were going to be merged" , inputs.Count > 0 && inputs.Count <= SortFactor); NUnit.Framework.Assert.IsTrue("Inputs to be merged were not sorted according to size: " , inputs[j].GetCompressedSize() >= inputs[j - 1].GetCompressedSize()); } } }