/// <exception cref="System.IO.IOException"/> public override RecordWriter <K, V> GetRecordWriter(FileSystem ignored, JobConf job , string name, Progressable progress) { bool isCompressed = GetCompressOutput(job); string keyValueSeparator = job.Get("mapreduce.output.textoutputformat.separator", "\t"); if (!isCompressed) { Path file = FileOutputFormat.GetTaskOutputPath(job, name); FileSystem fs = file.GetFileSystem(job); FSDataOutputStream fileOut = fs.Create(file, progress); return(new TextOutputFormat.LineRecordWriter <K, V>(fileOut, keyValueSeparator)); } else { Type codecClass = GetOutputCompressorClass(job, typeof(GzipCodec)); // create the named codec CompressionCodec codec = ReflectionUtils.NewInstance(codecClass, job); // build the filename including the extension Path file = FileOutputFormat.GetTaskOutputPath(job, name + codec.GetDefaultExtension ()); FileSystem fs = file.GetFileSystem(job); FSDataOutputStream fileOut = fs.Create(file, progress); return(new TextOutputFormat.LineRecordWriter <K, V>(new DataOutputStream(codec.CreateOutputStream (fileOut)), keyValueSeparator)); } }
/// <exception cref="System.IO.IOException"/> public override RecordWriter <BytesWritable, BytesWritable> GetRecordWriter(FileSystem ignored, JobConf job, string name, Progressable progress) { // get the path of the temporary output file Path file = FileOutputFormat.GetTaskOutputPath(job, name); FileSystem fs = file.GetFileSystem(job); CompressionCodec codec = null; SequenceFile.CompressionType compressionType = SequenceFile.CompressionType.None; if (GetCompressOutput(job)) { // find the kind of compression to do compressionType = GetOutputCompressionType(job); // find the right codec Type codecClass = GetOutputCompressorClass(job, typeof(DefaultCodec)); codec = ReflectionUtils.NewInstance(codecClass, job); } SequenceFile.Writer @out = SequenceFile.CreateWriter(fs, job, file, GetSequenceFileOutputKeyClass (job), GetSequenceFileOutputValueClass(job), compressionType, codec, progress); return(new _RecordWriter_138(@out)); }
/// <exception cref="System.IO.IOException"/> public override RecordWriter <WritableComparable, Writable> GetRecordWriter(FileSystem ignored, JobConf job, string name, Progressable progress) { // get the path of the temporary output file Path file = FileOutputFormat.GetTaskOutputPath(job, name); FileSystem fs = file.GetFileSystem(job); CompressionCodec codec = null; SequenceFile.CompressionType compressionType = SequenceFile.CompressionType.None; if (GetCompressOutput(job)) { // find the kind of compression to do compressionType = SequenceFileOutputFormat.GetOutputCompressionType(job); // find the right codec Type codecClass = GetOutputCompressorClass(job, typeof(DefaultCodec)); codec = ReflectionUtils.NewInstance(codecClass, job); } // ignore the progress parameter, since MapFile is local MapFile.Writer @out = new MapFile.Writer(job, fs, file.ToString(), job.GetOutputKeyClass ().AsSubclass <WritableComparable>(), job.GetOutputValueClass().AsSubclass <Writable >(), compressionType, codec, progress); return(new _RecordWriter_72(@out)); }