/// <exception cref="System.Exception"/> public virtual void TestInterruptOnDisk() { int Fetcher = 7; Path p = new Path("file:///tmp/foo"); Path pTmp = OnDiskMapOutput.GetTempPath(p, Fetcher); FileSystem mFs = Org.Mockito.Mockito.Mock <FileSystem>(ReturnsDeepStubs); MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>(); Org.Mockito.Mockito.When(mof.GetInputFileForWrite(Matchers.Any <TaskID>(), Matchers.AnyLong ())).ThenReturn(p); OnDiskMapOutput <Text, Text> odmo = Org.Mockito.Mockito.Spy(new OnDiskMapOutput <Text , Text>(map1ID, id, mm, 100L, job, mof, Fetcher, true, mFs, p)); Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong (), Matchers.AnyInt())).ThenReturn(odmo); Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource(); Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host); string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString (encHash), key); Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200); Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash )).ThenReturn(replyHash); ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); header.Write(new DataOutputStream(bout)); TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream (bout.ToByteArray())); Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName)) .ThenReturn(ShuffleHeader.DefaultHttpHeaderName); Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion); Org.Mockito.Mockito.DoAnswer(new _Answer_610(@in)).When(connection).Disconnect(); Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id, ss, mm, r, metrics, except, key, connection, Fetcher); underTest.Start(); // wait for read in inputstream @in.WaitForFetcher(); underTest.ShutDown(); underTest.Join(); // rely on test timeout to kill if stuck NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly()); Org.Mockito.Mockito.Verify(mFs).Create(Matchers.Eq(pTmp)); Org.Mockito.Mockito.Verify(mFs).Delete(Matchers.Eq(pTmp), Matchers.Eq(false)); Org.Mockito.Mockito.Verify(odmo).Abort(); }
public virtual void TestCorruptedIFile() { int fetcher = 7; Path onDiskMapOutputPath = new Path(name.GetMethodName() + "/foo"); Path shuffledToDisk = OnDiskMapOutput.GetTempPath(onDiskMapOutputPath, fetcher); fs = FileSystem.GetLocal(job).GetRaw(); MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>(); OnDiskMapOutput <Text, Text> odmo = new OnDiskMapOutput <Text, Text>(map1ID, id, mm , 100L, job, mof, fetcher, true, fs, onDiskMapOutputPath); string mapData = "MAPDATA12345678901234567890"; ShuffleHeader header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1); ByteArrayOutputStream bout = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(bout); IFileOutputStream ios = new IFileOutputStream(dos); header.Write(dos); int headerSize = dos.Size(); try { ios.Write(Sharpen.Runtime.GetBytesForString(mapData)); } finally { ios.Close(); } int dataSize = bout.Size() - headerSize; // Ensure that the OnDiskMapOutput shuffler can successfully read the data. MapHost host = new MapHost("TestHost", "http://test/url"); ByteArrayInputStream bin = new ByteArrayInputStream(bout.ToByteArray()); try { // Read past the shuffle header. bin.Read(new byte[headerSize], 0, headerSize); odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null); } finally { bin.Close(); } // Now corrupt the IFile data. byte[] corrupted = bout.ToByteArray(); corrupted[headerSize + (dataSize / 2)] = unchecked ((int)(0x0)); try { bin = new ByteArrayInputStream(corrupted); // Read past the shuffle header. bin.Read(new byte[headerSize], 0, headerSize); odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null); NUnit.Framework.Assert.Fail("OnDiskMapOutput.shuffle didn't detect the corrupted map partition file" ); } catch (ChecksumException e) { Log.Info("The expected checksum exception was thrown.", e); } finally { bin.Close(); } // Ensure that the shuffled file can be read. IFileInputStream iFin = new IFileInputStream(fs.Open(shuffledToDisk), dataSize, job ); try { iFin.Read(new byte[dataSize], 0, dataSize); } finally { iFin.Close(); } }