/// <exception cref="System.IO.IOException"/>
private void CopyPartitions(Path mapOutputPath, Path indexPath)
{
FileSystem localFs = FileSystem.GetLocal(jobConf);
FileSystem rfs = ((LocalFileSystem)localFs).GetRaw();
FSDataOutputStream rawOutput = rfs.Create(mapOutputPath, true, BufSize);
SpillRecord spillRecord = new SpillRecord(numberOfPartitions);
IndexRecord indexRecord = new IndexRecord();
for (int i = 0; i < numberOfPartitions; i++)
{
indexRecord.startOffset = rawOutput.GetPos();
byte[] buffer = outStreams[i].ToByteArray();
IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput);
checksumOutput.Write(buffer);
// Write checksum.
checksumOutput.Finish();
// Write index record
indexRecord.rawLength = (long)buffer.Length;
indexRecord.partLength = rawOutput.GetPos() - indexRecord.startOffset;
spillRecord.PutIndex(indexRecord, i);
reporter.Progress();
}
rawOutput.Close();
spillRecord.WriteToFile(indexPath, jobConf);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close()
{
// When IFile writer is created by BackupStore, we do not have
// Key and Value classes set. So, check before closing the
// serializers
if (keyClass != null)
{
keySerializer.Close();
valueSerializer.Close();
}
// Write EOF_MARKER for key/value length
WritableUtils.WriteVInt(@out, EofMarker);
WritableUtils.WriteVInt(@out, EofMarker);
decompressedBytesWritten += 2 * WritableUtils.GetVIntSize(EofMarker);
//Flush the stream
@out.Flush();
if (compressOutput)
{
// Flush
compressedOut.Finish();
compressedOut.ResetState();
}
// Close the underlying stream iff we own it...
if (ownOutputStream)
{
@out.Close();
}
else
{
// Write the checksum
checksumOut.Finish();
}
compressedBytesWritten = rawOut.GetPos() - start;
if (compressOutput)
{
// Return back the compressor
CodecPool.ReturnCompressor(compressor);
compressor = null;
}
@out = null;
if (writtenRecordsCounter != null)
{
writtenRecordsCounter.Increment(numRecordsWritten);
}
}
