/// <exception cref="System.IO.IOException"/> private void CopyPartitions(Path mapOutputPath, Path indexPath) { FileSystem localFs = FileSystem.GetLocal(jobConf); FileSystem rfs = ((LocalFileSystem)localFs).GetRaw(); FSDataOutputStream rawOutput = rfs.Create(mapOutputPath, true, BufSize); SpillRecord spillRecord = new SpillRecord(numberOfPartitions); IndexRecord indexRecord = new IndexRecord(); for (int i = 0; i < numberOfPartitions; i++) { indexRecord.startOffset = rawOutput.GetPos(); byte[] buffer = outStreams[i].ToByteArray(); IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput); checksumOutput.Write(buffer); // Write checksum. checksumOutput.Finish(); // Write index record indexRecord.rawLength = (long)buffer.Length; indexRecord.partLength = rawOutput.GetPos() - indexRecord.startOffset; spillRecord.PutIndex(indexRecord, i); reporter.Progress(); } rawOutput.Close(); spillRecord.WriteToFile(indexPath, jobConf); }
/// <exception cref="System.IO.IOException"/> public virtual void Close() { // When IFile writer is created by BackupStore, we do not have // Key and Value classes set. So, check before closing the // serializers if (keyClass != null) { keySerializer.Close(); valueSerializer.Close(); } // Write EOF_MARKER for key/value length WritableUtils.WriteVInt(@out, EofMarker); WritableUtils.WriteVInt(@out, EofMarker); decompressedBytesWritten += 2 * WritableUtils.GetVIntSize(EofMarker); //Flush the stream @out.Flush(); if (compressOutput) { // Flush compressedOut.Finish(); compressedOut.ResetState(); } // Close the underlying stream iff we own it... if (ownOutputStream) { @out.Close(); } else { // Write the checksum checksumOut.Finish(); } compressedBytesWritten = rawOut.GetPos() - start; if (compressOutput) { // Return back the compressor CodecPool.ReturnCompressor(compressor); compressor = null; } @out = null; if (writtenRecordsCounter != null) { writtenRecordsCounter.Increment(numRecordsWritten); } }