/// <exception cref="System.Exception"/> private void CheckCompression(bool compressMapOutputs, SequenceFile.CompressionType redCompression, bool includeCombine) { JobConf conf = new JobConf(typeof(TestMapRed)); Path testdir = new Path(TestDir.GetAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.Get(conf); fs.Delete(testdir, true); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); conf.SetMapperClass(typeof(TestMapRed.MyMap)); conf.SetReducerClass(typeof(TestMapRed.MyReduce)); conf.SetOutputKeyClass(typeof(Text)); conf.SetOutputValueClass(typeof(Text)); conf.SetOutputFormat(typeof(SequenceFileOutputFormat)); conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); if (includeCombine) { conf.SetCombinerClass(typeof(IdentityReducer)); } conf.SetCompressMapOutput(compressMapOutputs); SequenceFileOutputFormat.SetOutputCompressionType(conf, redCompression); try { if (!fs.Mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.ToString()); } if (!fs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.ToString()); } Path inFile = new Path(inDir, "part0"); DataOutputStream f = fs.Create(inFile); f.WriteBytes("Owen was here\n"); f.WriteBytes("Hadoop is fun\n"); f.WriteBytes("Is this done, yet?\n"); f.Close(); RunningJob rj = JobClient.RunJob(conf); NUnit.Framework.Assert.IsTrue("job was complete", rj.IsComplete()); NUnit.Framework.Assert.IsTrue("job was successful", rj.IsSuccessful()); Path output = new Path(outDir, Task.GetOutputName(0)); NUnit.Framework.Assert.IsTrue("reduce output exists " + output, fs.Exists(output) ); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, output, conf); NUnit.Framework.Assert.AreEqual("is reduce output compressed " + output, redCompression != SequenceFile.CompressionType.None, rdr.IsCompressed()); rdr.Close(); } finally { fs.Delete(testdir, true); } }
/// <exception cref="System.IO.IOException"/> public override RecordWriter <WritableComparable, Writable> GetRecordWriter(FileSystem ignored, JobConf job, string name, Progressable progress) { // get the path of the temporary output file Path file = FileOutputFormat.GetTaskOutputPath(job, name); FileSystem fs = file.GetFileSystem(job); CompressionCodec codec = null; SequenceFile.CompressionType compressionType = SequenceFile.CompressionType.None; if (GetCompressOutput(job)) { // find the kind of compression to do compressionType = SequenceFileOutputFormat.GetOutputCompressionType(job); // find the right codec Type codecClass = GetOutputCompressorClass(job, typeof(DefaultCodec)); codec = ReflectionUtils.NewInstance(codecClass, job); } // ignore the progress parameter, since MapFile is local MapFile.Writer @out = new MapFile.Writer(job, fs, file.ToString(), job.GetOutputKeyClass ().AsSubclass <WritableComparable>(), job.GetOutputValueClass().AsSubclass <Writable >(), compressionType, codec, progress); return(new _RecordWriter_72(@out)); }