/// <exception cref="System.IO.IOException"/> public override void Initialize(InputSplit genericSplit, TaskAttemptContext context ) { FileSplit split = (FileSplit)genericSplit; Configuration job = context.GetConfiguration(); this.maxLineLength = job.GetInt(MaxLineLength, int.MaxValue); start = split.GetStart(); end = start + split.GetLength(); Path file = split.GetPath(); // open the file and seek to the start of the split FileSystem fs = file.GetFileSystem(job); fileIn = fs.Open(file); CompressionCodec codec = new CompressionCodecFactory(job).GetCodec(file); if (null != codec) { isCompressedInput = true; decompressor = CodecPool.GetDecompressor(codec); if (codec is SplittableCompressionCodec) { SplitCompressionInputStream cIn = ((SplittableCompressionCodec)codec).CreateInputStream (fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.Byblock); @in = new CompressedSplitLineReader(cIn, job, this.recordDelimiterBytes); start = cIn.GetAdjustedStart(); end = cIn.GetAdjustedEnd(); filePosition = cIn; } else { @in = new SplitLineReader(codec.CreateInputStream(fileIn, decompressor), job, this .recordDelimiterBytes); filePosition = fileIn; } } else { fileIn.Seek(start); @in = new UncompressedSplitLineReader(fileIn, job, this.recordDelimiterBytes, split .GetLength()); filePosition = fileIn; } // If this is not the first split, we always throw away first record // because we always (except the last split) read one extra line in // next() method. if (start != 0) { start += @in.ReadLine(new Text(), 0, MaxBytesToConsume(start)); } this.pos = start; }
// This is also called from the old FixedLengthRecordReader API implementation /// <exception cref="System.IO.IOException"/> public virtual void Initialize(Configuration job, long splitStart, long splitLength , Path file) { start = splitStart; end = start + splitLength; long partialRecordLength = start % recordLength; long numBytesToSkip = 0; if (partialRecordLength != 0) { numBytesToSkip = recordLength - partialRecordLength; } // open the file and seek to the start of the split FileSystem fs = file.GetFileSystem(job); fileIn = fs.Open(file); CompressionCodec codec = new CompressionCodecFactory(job).GetCodec(file); if (null != codec) { isCompressedInput = true; decompressor = CodecPool.GetDecompressor(codec); CompressionInputStream cIn = codec.CreateInputStream(fileIn, decompressor); filePosition = cIn; inputStream = cIn; numRecordsRemainingInSplit = long.MaxValue; Log.Info("Compressed input; cannot compute number of records in the split"); } else { fileIn.Seek(start); filePosition = fileIn; inputStream = fileIn; long splitSize = end - start - numBytesToSkip; numRecordsRemainingInSplit = (splitSize + recordLength - 1) / recordLength; if (numRecordsRemainingInSplit < 0) { numRecordsRemainingInSplit = 0; } Log.Info("Expecting " + numRecordsRemainingInSplit + " records each with a length of " + recordLength + " bytes in the split with an effective size of " + splitSize + " bytes"); } if (numBytesToSkip != 0) { start += inputStream.Skip(numBytesToSkip); } this.pos = start; }