/// <exception cref="System.IO.IOException"/>
public override void Shuffle(MapHost host, InputStream input, long compressedLength
, long decompressedLength, ShuffleClientMetrics metrics, Reporter reporter)
{
input = new IFileInputStream(input, compressedLength, conf);
// Copy data to local-disk
long bytesLeft = compressedLength;
try
{
int BytesToRead = 64 * 1024;
byte[] buf = new byte[BytesToRead];
while (bytesLeft > 0)
{
int n = ((IFileInputStream)input).ReadWithChecksum(buf, 0, (int)Math.Min(bytesLeft
, BytesToRead));
if (n < 0)
{
throw new IOException("read past end of stream reading " + GetMapId());
}
disk.Write(buf, 0, n);
bytesLeft -= n;
metrics.InputBytes(n);
reporter.Progress();
}
Log.Info("Read " + (compressedLength - bytesLeft) + " bytes from map-output for "
+ GetMapId());
disk.Close();
}
catch (IOException ioe)
{
// Close the streams
IOUtils.Cleanup(Log, input, disk);
// Re-throw
throw;
}
// Sanity check
if (bytesLeft != 0)
{
throw new IOException("Incomplete map output received for " + GetMapId() + " from "
+ host.GetHostName() + " (" + bytesLeft + " bytes missing of " + compressedLength
+ ")");
}
this.compressedSize = compressedLength;
}
/// <exception cref="System.IO.IOException"/>
public override void Shuffle(MapHost host, InputStream input, long compressedLength
, long decompressedLength, ShuffleClientMetrics metrics, Reporter reporter)
{
IFileInputStream checksumIn = new IFileInputStream(input, compressedLength, conf);
input = checksumIn;
// Are map-outputs compressed?
if (codec != null)
{
decompressor.Reset();
input = codec.CreateInputStream(input, decompressor);
}
try
{
IOUtils.ReadFully(input, memory, 0, memory.Length);
metrics.InputBytes(memory.Length);
reporter.Progress();
Log.Info("Read " + memory.Length + " bytes from map-output for " + GetMapId());
if (input.Read() >= 0)
{
throw new IOException("Unexpected extra bytes from input stream for " + GetMapId(
));
}
}
catch (IOException ioe)
{
// Close the streams
IOUtils.Cleanup(Log, input);
// Re-throw
throw;
}
finally
{
CodecPool.ReturnDecompressor(decompressor);
}
}