/// <exception cref="System.Exception"/> public virtual void TestCopyFromHostCompressFailure() { InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>( ); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(immo); Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any < MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>()); underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash , encHash); Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host , true, false); }
/// <exception cref="System.Exception"/> public virtual void TestCopyFromHostWithRetryUnreserve() { InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>( ); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry , id, ss, mm, r, metrics, except, key, connection); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); // Verify that unreserve occurs if an exception happens after shuffle // buffer is reserved. Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(immo); Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong() , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>()); underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(immo).Abort(); }
public virtual void TestReduceOutOfDiskSpace() { Log.Info("testReduceOutOfDiskSpace"); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenThrow(new DiskChecker.DiskErrorException("No disk space available" )); underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(ss).ReportLocalError(Matchers.Any <IOException>()); }
public virtual void TestCopyFromHostExtraBytes() { Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bout); IFileOutputStream ios = new IFileOutputStream(dos); header.Write(dos); ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123")); ios.Finish(); ShuffleHeader header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1); IFileOutputStream ios2 = new IFileOutputStream(dos); header2.Write(dos); ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456")); ios2.Finish(); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); // 8 < 10 therefore there appear to be extra bytes in the IFileInputStream InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID , mm, 8, null, true); InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID , mm, 10, null, true); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt ())).ThenReturn(mapOut); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt ())).ThenReturn(mapOut2); underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(allErrs).Increment(1); Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false); Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host , true, false); Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq (map1ID)); Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq (map2ID)); }
/// <exception cref="System.Exception"/> public virtual void TestInterruptOnDisk() { int Fetcher = 7; Path p = new Path("file:///tmp/foo"); Path pTmp = OnDiskMapOutput.GetTempPath(p, Fetcher); FileSystem mFs = Org.Mockito.Mockito.Mock <FileSystem>(ReturnsDeepStubs); MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>(); Org.Mockito.Mockito.When(mof.GetInputFileForWrite(Matchers.Any <TaskID>(), Matchers.AnyLong ())).ThenReturn(p); OnDiskMapOutput <Text, Text> odmo = Org.Mockito.Mockito.Spy(new OnDiskMapOutput <Text , Text>(map1ID, id, mm, 100L, job, mof, Fetcher, true, mFs, p)); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(odmo); Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource(); Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream (bout.ToByteArray())); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.DoAnswer(new _Answer_610(@in)).When(connection).Disconnect(); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection, Fetcher); underTest.Start(); // wait for read in inputstream @in.WaitForFetcher(); underTest.ShutDown(); underTest.Join(); // rely on test timeout to kill if stuck NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly()); Org.Mockito.Mockito.Verify(mFs).Create(Matchers.Eq(pTmp)); Org.Mockito.Mockito.Verify(mFs).Delete(Matchers.Eq(pTmp), Matchers.Eq(false)); Org.Mockito.Mockito.Verify(odmo).Abort(); }
/// <exception cref="System.Exception"/> public virtual void TestInterruptInMemory() { int Fetcher = 2; InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Spy(new InMemoryMapOutput <Text, Text>(job, id, mm, 100, null, true)); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(immo); Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource(); Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream (bout.ToByteArray())); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.DoAnswer(new _Answer_562(@in)).When(connection).Disconnect(); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection, Fetcher); underTest.Start(); // wait for read in inputstream @in.WaitForFetcher(); underTest.ShutDown(); underTest.Join(); // rely on test timeout to kill if stuck NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly()); Org.Mockito.Mockito.Verify(immo).Abort(); }
/// <exception cref="System.Exception"/> public virtual void TestCopyFromHostWithRetry() { InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>( ); ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>(); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry , id, ss, mm, r, metrics, except, key, connection, true); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(immo); long retryTime = Time.MonotonicNow(); Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong() , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>()); // Emulate host down for 3 seconds. underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any <TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean ()); }
public virtual void TestCopyFromHostWait() { Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray()); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); //Defaults to null, which is what we want to test Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(null); underTest.CopyFromHost(host); Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash , encHash); Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Never()).Increment(1); Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map1ID, host , true, false); Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host , true, false); Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq (map1ID)); Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq (map2ID)); }
/// <exception cref="System.IO.IOException"/> private TaskAttemptID[] CopyMapOutput(MapHost host, DataInputStream input, ICollection <TaskAttemptID> remaining, bool canRetry) { MapOutput <K, V> mapOutput = null; TaskAttemptID mapId = null; long decompressedLength = -1; long compressedLength = -1; try { long startTime = Time.MonotonicNow(); int forReduce = -1; //Read the shuffle header try { ShuffleHeader header = new ShuffleHeader(); header.ReadFields(input); mapId = TaskAttemptID.ForName(header.mapId); compressedLength = header.compressedLength; decompressedLength = header.uncompressedLength; forReduce = header.forReduce; } catch (ArgumentException e) { badIdErrs.Increment(1); Log.Warn("Invalid map id ", e); //Don't know which one was bad, so consider all of them as bad return(Sharpen.Collections.ToArray(remaining, new TaskAttemptID[remaining.Count])); } InputStream @is = input; @is = CryptoUtils.WrapIfNecessary(jobConf, @is, compressedLength); compressedLength -= CryptoUtils.CryptoPadding(jobConf); decompressedLength -= CryptoUtils.CryptoPadding(jobConf); // Do some basic sanity verification if (!VerifySanity(compressedLength, decompressedLength, forReduce, remaining, mapId )) { return(new TaskAttemptID[] { mapId }); } if (Log.IsDebugEnabled()) { Log.Debug("header: " + mapId + ", len: " + compressedLength + ", decomp len: " + decompressedLength); } // Get the location for the map output - either in-memory or on-disk try { mapOutput = merger.Reserve(mapId, decompressedLength, id); } catch (IOException ioe) { // kill this reduce attempt ioErrs.Increment(1); scheduler.ReportLocalError(ioe); return(EmptyAttemptIdArray); } // Check if we can shuffle *now* ... if (mapOutput == null) { Log.Info("fetcher#" + id + " - MergeManager returned status WAIT ..."); //Not an error but wait to process data. return(EmptyAttemptIdArray); } // The codec for lz0,lz4,snappy,bz2,etc. throw java.lang.InternalError // on decompression failures. Catching and re-throwing as IOException // to allow fetch failure logic to be processed try { // Go! Log.Info("fetcher#" + id + " about to shuffle output of map " + mapOutput.GetMapId () + " decomp: " + decompressedLength + " len: " + compressedLength + " to " + mapOutput .GetDescription()); mapOutput.Shuffle(host, @is, compressedLength, decompressedLength, metrics, reporter ); } catch (InternalError e) { Log.Warn("Failed to shuffle for fetcher#" + id, e); throw new IOException(e); } // Inform the shuffle scheduler long endTime = Time.MonotonicNow(); // Reset retryStartTime as map task make progress if retried before. retryStartTime = 0; scheduler.CopySucceeded(mapId, host, compressedLength, startTime, endTime, mapOutput ); // Note successful shuffle remaining.Remove(mapId); metrics.SuccessFetch(); return(null); } catch (IOException ioe) { if (mapOutput != null) { mapOutput.Abort(); } if (canRetry) { CheckTimeoutOrRetry(host, ioe); } ioErrs.Increment(1); if (mapId == null || mapOutput == null) { Log.Warn("fetcher#" + id + " failed to read map header" + mapId + " decomp: " + decompressedLength + ", " + compressedLength, ioe); if (mapId == null) { return(Sharpen.Collections.ToArray(remaining, new TaskAttemptID[remaining.Count])); } else { return(new TaskAttemptID[] { mapId }); } } Log.Warn("Failed to shuffle output of " + mapId + " from " + host.GetHostName(), ioe); // Inform the shuffle-scheduler metrics.FailedFetch(); return(new TaskAttemptID[] { mapId }); } }
public virtual void TestCorruptedIFile() { int fetcher = 7; Path onDiskMapOutputPath = new Path(name.GetMethodName() + "/foo"); Path shuffledToDisk = OnDiskMapOutput.GetTempPath(onDiskMapOutputPath, fetcher); fs = FileSystem.GetLocal(job).GetRaw(); MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>(); OnDiskMapOutput <Text, Text> odmo = new OnDiskMapOutput <Text, Text>(map1ID, id, mm , 100L, job, mof, fetcher, true, fs, onDiskMapOutputPath); string mapData = "MAPDATA12345678901234567890"; ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bout); IFileOutputStream ios = new IFileOutputStream(dos); header.Write(dos); int headerSize = dos.Size(); try { ios.Write(Sharpen.Runtime.GetBytesForString(mapData)); } finally { ios.Close(); } int dataSize = bout.Size() - headerSize; // Ensure that the OnDiskMapOutput shuffler can successfully read the data. MapHost host = new MapHost("TestHost", "http://test/url"); ByteArrayInputStream bin = new ByteArrayInputStream(bout.ToByteArray()); try { // Read past the shuffle header. bin.Read(new byte[headerSize], 0, headerSize); odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null); } finally { bin.Close(); } // Now corrupt the IFile data. byte[] corrupted = bout.ToByteArray(); corrupted[headerSize + (dataSize / 2)] = unchecked ((int)(0x0)); try { bin = new ByteArrayInputStream(corrupted); // Read past the shuffle header. bin.Read(new byte[headerSize], 0, headerSize); odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null); NUnit.Framework.Assert.Fail("OnDiskMapOutput.shuffle didn't detect the corrupted map partition file" ); } catch (ChecksumException e) { Log.Info("The expected checksum exception was thrown.", e); } finally { bin.Close(); } // Ensure that the shuffled file can be read. IFileInputStream iFin = new IFileInputStream(fs.Open(shuffledToDisk), dataSize, job ); try { iFin.Read(new byte[dataSize], 0, dataSize); } finally { iFin.Close(); } }