/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostCompressFailure()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any <
MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(),
Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host
, true, false);
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetryUnreserve()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
// Verify that unreserve occurs if an exception happens after shuffle
// buffer is reserved.
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(immo).Abort();
}
public virtual void TestReduceOutOfDiskSpace()
{
Log.Info("testReduceOutOfDiskSpace");
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenThrow(new DiskChecker.DiskErrorException("No disk space available"
));
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss).ReportLocalError(Matchers.Any <IOException>());
}
public virtual void TestCopyFromHostExtraBytes()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bout);
IFileOutputStream ios = new IFileOutputStream(dos);
header.Write(dos);
ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123"));
ios.Finish();
ShuffleHeader header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1);
IFileOutputStream ios2 = new IFileOutputStream(dos);
header2.Write(dos);
ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456"));
ios2.Finish();
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
// 8 < 10 therefore there appear to be extra bytes in the IFileInputStream
InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID
, mm, 8, null, true);
InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID
, mm, 10, null, true);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt
())).ThenReturn(mapOut);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt
())).ThenReturn(mapOut2);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(allErrs).Increment(1);
Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
/// <exception cref="System.Exception"/>
public virtual void TestInterruptOnDisk()
{
int Fetcher = 7;
Path p = new Path("file:///tmp/foo");
Path pTmp = OnDiskMapOutput.GetTempPath(p, Fetcher);
FileSystem mFs = Org.Mockito.Mockito.Mock <FileSystem>(ReturnsDeepStubs);
MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Mockito.Mockito.When(mof.GetInputFileForWrite(Matchers.Any <TaskID>(), Matchers.AnyLong
())).ThenReturn(p);
OnDiskMapOutput <Text, Text> odmo = Org.Mockito.Mockito.Spy(new OnDiskMapOutput <Text
, Text>(map1ID, id, mm, 100L, job, mof, Fetcher, true, mFs, p));
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(odmo);
Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
(bout.ToByteArray()));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.DoAnswer(new _Answer_610(@in)).When(connection).Disconnect();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection, Fetcher);
underTest.Start();
// wait for read in inputstream
@in.WaitForFetcher();
underTest.ShutDown();
underTest.Join();
// rely on test timeout to kill if stuck
NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
Org.Mockito.Mockito.Verify(mFs).Create(Matchers.Eq(pTmp));
Org.Mockito.Mockito.Verify(mFs).Delete(Matchers.Eq(pTmp), Matchers.Eq(false));
Org.Mockito.Mockito.Verify(odmo).Abort();
}
/// <exception cref="System.Exception"/>
public virtual void TestInterruptInMemory()
{
int Fetcher = 2;
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Spy(new InMemoryMapOutput
<Text, Text>(job, id, mm, 100, null, true));
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
(bout.ToByteArray()));
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.DoAnswer(new _Answer_562(@in)).When(connection).Disconnect();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection, Fetcher);
underTest.Start();
// wait for read in inputstream
@in.WaitForFetcher();
underTest.ShutDown();
underTest.Join();
// rely on test timeout to kill if stuck
NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
Org.Mockito.Mockito.Verify(immo).Abort();
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFromHostWithRetry()
{
InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
);
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
, id, ss, mm, r, metrics, except, key, connection, true);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(immo);
long retryTime = Time.MonotonicNow();
Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any
<MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
, Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
// Emulate host down for 3 seconds.
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any
<TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean
());
}
public virtual void TestCopyFromHostWait()
{
Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
ss, mm, r, metrics, except, key, connection);
string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
(encHash), key);
Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
)).ThenReturn(replyHash);
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
header.Write(new DataOutputStream(bout));
ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());
Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
.ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
)).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
//Defaults to null, which is what we want to test
Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
(), Matchers.AnyInt())).ThenReturn(null);
underTest.CopyFromHost(host);
Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
, encHash);
Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Never()).Increment(1);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map1ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
, true, false);
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map1ID));
Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
(map2ID));
}
/// <exception cref="System.IO.IOException"/>
private TaskAttemptID[] CopyMapOutput(MapHost host, DataInputStream input, ICollection
<TaskAttemptID> remaining, bool canRetry)
{
MapOutput <K, V> mapOutput = null;
TaskAttemptID mapId = null;
long decompressedLength = -1;
long compressedLength = -1;
try
{
long startTime = Time.MonotonicNow();
int forReduce = -1;
//Read the shuffle header
try
{
ShuffleHeader header = new ShuffleHeader();
header.ReadFields(input);
mapId = TaskAttemptID.ForName(header.mapId);
compressedLength = header.compressedLength;
decompressedLength = header.uncompressedLength;
forReduce = header.forReduce;
}
catch (ArgumentException e)
{
badIdErrs.Increment(1);
Log.Warn("Invalid map id ", e);
//Don't know which one was bad, so consider all of them as bad
return(Sharpen.Collections.ToArray(remaining, new TaskAttemptID[remaining.Count]));
}
InputStream @is = input;
@is = CryptoUtils.WrapIfNecessary(jobConf, @is, compressedLength);
compressedLength -= CryptoUtils.CryptoPadding(jobConf);
decompressedLength -= CryptoUtils.CryptoPadding(jobConf);
// Do some basic sanity verification
if (!VerifySanity(compressedLength, decompressedLength, forReduce, remaining, mapId
))
{
return(new TaskAttemptID[] { mapId });
}
if (Log.IsDebugEnabled())
{
Log.Debug("header: " + mapId + ", len: " + compressedLength + ", decomp len: " +
decompressedLength);
}
// Get the location for the map output - either in-memory or on-disk
try
{
mapOutput = merger.Reserve(mapId, decompressedLength, id);
}
catch (IOException ioe)
{
// kill this reduce attempt
ioErrs.Increment(1);
scheduler.ReportLocalError(ioe);
return(EmptyAttemptIdArray);
}
// Check if we can shuffle *now* ...
if (mapOutput == null)
{
Log.Info("fetcher#" + id + " - MergeManager returned status WAIT ...");
//Not an error but wait to process data.
return(EmptyAttemptIdArray);
}
// The codec for lz0,lz4,snappy,bz2,etc. throw java.lang.InternalError
// on decompression failures. Catching and re-throwing as IOException
// to allow fetch failure logic to be processed
try
{
// Go!
Log.Info("fetcher#" + id + " about to shuffle output of map " + mapOutput.GetMapId
() + " decomp: " + decompressedLength + " len: " + compressedLength + " to " + mapOutput
.GetDescription());
mapOutput.Shuffle(host, @is, compressedLength, decompressedLength, metrics, reporter
);
}
catch (InternalError e)
{
Log.Warn("Failed to shuffle for fetcher#" + id, e);
throw new IOException(e);
}
// Inform the shuffle scheduler
long endTime = Time.MonotonicNow();
// Reset retryStartTime as map task make progress if retried before.
retryStartTime = 0;
scheduler.CopySucceeded(mapId, host, compressedLength, startTime, endTime, mapOutput
);
// Note successful shuffle
remaining.Remove(mapId);
metrics.SuccessFetch();
return(null);
}
catch (IOException ioe)
{
if (mapOutput != null)
{
mapOutput.Abort();
}
if (canRetry)
{
CheckTimeoutOrRetry(host, ioe);
}
ioErrs.Increment(1);
if (mapId == null || mapOutput == null)
{
Log.Warn("fetcher#" + id + " failed to read map header" + mapId + " decomp: " + decompressedLength
+ ", " + compressedLength, ioe);
if (mapId == null)
{
return(Sharpen.Collections.ToArray(remaining, new TaskAttemptID[remaining.Count]));
}
else
{
return(new TaskAttemptID[] { mapId });
}
}
Log.Warn("Failed to shuffle output of " + mapId + " from " + host.GetHostName(),
ioe);
// Inform the shuffle-scheduler
metrics.FailedFetch();
return(new TaskAttemptID[] { mapId });
}
}
public virtual void TestCorruptedIFile()
{
int fetcher = 7;
Path onDiskMapOutputPath = new Path(name.GetMethodName() + "/foo");
Path shuffledToDisk = OnDiskMapOutput.GetTempPath(onDiskMapOutputPath, fetcher);
fs = FileSystem.GetLocal(job).GetRaw();
MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>();
OnDiskMapOutput <Text, Text> odmo = new OnDiskMapOutput <Text, Text>(map1ID, id, mm
, 100L, job, mof, fetcher, true, fs, onDiskMapOutputPath);
string mapData = "MAPDATA12345678901234567890";
ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bout);
IFileOutputStream ios = new IFileOutputStream(dos);
header.Write(dos);
int headerSize = dos.Size();
try
{
ios.Write(Sharpen.Runtime.GetBytesForString(mapData));
}
finally
{
ios.Close();
}
int dataSize = bout.Size() - headerSize;
// Ensure that the OnDiskMapOutput shuffler can successfully read the data.
MapHost host = new MapHost("TestHost", "http://test/url");
ByteArrayInputStream bin = new ByteArrayInputStream(bout.ToByteArray());
try
{
// Read past the shuffle header.
bin.Read(new byte[headerSize], 0, headerSize);
odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null);
}
finally
{
bin.Close();
}
// Now corrupt the IFile data.
byte[] corrupted = bout.ToByteArray();
corrupted[headerSize + (dataSize / 2)] = unchecked ((int)(0x0));
try
{
bin = new ByteArrayInputStream(corrupted);
// Read past the shuffle header.
bin.Read(new byte[headerSize], 0, headerSize);
odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null);
NUnit.Framework.Assert.Fail("OnDiskMapOutput.shuffle didn't detect the corrupted map partition file"
);
}
catch (ChecksumException e)
{
Log.Info("The expected checksum exception was thrown.", e);
}
finally
{
bin.Close();
}
// Ensure that the shuffled file can be read.
IFileInputStream iFin = new IFileInputStream(fs.Open(shuffledToDisk), dataSize, job
);
try
{
iFin.Read(new byte[dataSize], 0, dataSize);
}
finally
{
iFin.Close();
}
}
