/// <exception cref="System.IO.IOException"/> public override void Shuffle(MapHost host, InputStream input, long compressedLength , long decompressedLength, ShuffleClientMetrics metrics, Reporter reporter) { input = new IFileInputStream(input, compressedLength, conf); // Copy data to local-disk long bytesLeft = compressedLength; try { int BytesToRead = 64 * 1024; byte[] buf = new byte[BytesToRead]; while (bytesLeft > 0) { int n = ((IFileInputStream)input).ReadWithChecksum(buf, 0, (int)Math.Min(bytesLeft , BytesToRead)); if (n < 0) { throw new IOException("read past end of stream reading " + GetMapId()); } disk.Write(buf, 0, n); bytesLeft -= n; metrics.InputBytes(n); reporter.Progress(); } Log.Info("Read " + (compressedLength - bytesLeft) + " bytes from map-output for " + GetMapId()); disk.Close(); } catch (IOException ioe) { // Close the streams IOUtils.Cleanup(Log, input, disk); // Re-throw throw; } // Sanity check if (bytesLeft != 0) { throw new IOException("Incomplete map output received for " + GetMapId() + " from " + host.GetHostName() + " (" + bytesLeft + " bytes missing of " + compressedLength + ")"); } this.compressedSize = compressedLength; }
/// <exception cref="System.IO.IOException"/> public override void Shuffle(MapHost host, InputStream input, long compressedLength , long decompressedLength, ShuffleClientMetrics metrics, Reporter reporter) { IFileInputStream checksumIn = new IFileInputStream(input, compressedLength, conf); input = checksumIn; // Are map-outputs compressed? if (codec != null) { decompressor.Reset(); input = codec.CreateInputStream(input, decompressor); } try { IOUtils.ReadFully(input, memory, 0, memory.Length); metrics.InputBytes(memory.Length); reporter.Progress(); Log.Info("Read " + memory.Length + " bytes from map-output for " + GetMapId()); if (input.Read() >= 0) { throw new IOException("Unexpected extra bytes from input stream for " + GetMapId( )); } } catch (IOException ioe) { // Close the streams IOUtils.Cleanup(Log, input); // Re-throw throw; } finally { CodecPool.ReturnDecompressor(decompressor); } }