示例#1
0
        public virtual void TestCopyFromHostExtraBytes()
        {
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();
            DataOutputStream      dos    = new DataOutputStream(bout);
            IFileOutputStream     ios    = new IFileOutputStream(dos);

            header.Write(dos);
            ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123"));
            ios.Finish();
            ShuffleHeader     header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1);
            IFileOutputStream ios2    = new IFileOutputStream(dos);

            header2.Write(dos);
            ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456"));
            ios2.Finish();
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            // 8 < 10 therefore there appear to be extra bytes in the IFileInputStream
            InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID
                                                                                       , mm, 8, null, true);
            InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID
                                                                                        , mm, 10, null, true);

            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt
                                                    ())).ThenReturn(mapOut);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt
                                                    ())).ThenReturn(mapOut2);
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(allErrs).Increment(1);
            Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
                                                                                   , true, false);
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map1ID));
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map2ID));
        }
示例#2
0
        public virtual void TestCorruptedIFile()
        {
            int  fetcher             = 7;
            Path onDiskMapOutputPath = new Path(name.GetMethodName() + "/foo");
            Path shuffledToDisk      = OnDiskMapOutput.GetTempPath(onDiskMapOutputPath, fetcher);

            fs = FileSystem.GetLocal(job).GetRaw();
            MapOutputFile mof = Org.Mockito.Mockito.Mock <MapOutputFile>();
            OnDiskMapOutput <Text, Text> odmo = new OnDiskMapOutput <Text, Text>(map1ID, id, mm
                                                                                 , 100L, job, mof, fetcher, true, fs, onDiskMapOutputPath);
            string                mapData = "MAPDATA12345678901234567890";
            ShuffleHeader         header  = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
            ByteArrayOutputStream bout    = new ByteArrayOutputStream();
            DataOutputStream      dos     = new DataOutputStream(bout);
            IFileOutputStream     ios     = new IFileOutputStream(dos);

            header.Write(dos);
            int headerSize = dos.Size();

            try
            {
                ios.Write(Sharpen.Runtime.GetBytesForString(mapData));
            }
            finally
            {
                ios.Close();
            }
            int dataSize = bout.Size() - headerSize;
            // Ensure that the OnDiskMapOutput shuffler can successfully read the data.
            MapHost host             = new MapHost("TestHost", "http://test/url");
            ByteArrayInputStream bin = new ByteArrayInputStream(bout.ToByteArray());

            try
            {
                // Read past the shuffle header.
                bin.Read(new byte[headerSize], 0, headerSize);
                odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null);
            }
            finally
            {
                bin.Close();
            }
            // Now corrupt the IFile data.
            byte[] corrupted = bout.ToByteArray();
            corrupted[headerSize + (dataSize / 2)] = unchecked ((int)(0x0));
            try
            {
                bin = new ByteArrayInputStream(corrupted);
                // Read past the shuffle header.
                bin.Read(new byte[headerSize], 0, headerSize);
                odmo.Shuffle(host, bin, dataSize, dataSize, metrics, Reporter.Null);
                NUnit.Framework.Assert.Fail("OnDiskMapOutput.shuffle didn't detect the corrupted map partition file"
                                            );
            }
            catch (ChecksumException e)
            {
                Log.Info("The expected checksum exception was thrown.", e);
            }
            finally
            {
                bin.Close();
            }
            // Ensure that the shuffled file can be read.
            IFileInputStream iFin = new IFileInputStream(fs.Open(shuffledToDisk), dataSize, job
                                                         );

            try
            {
                iFin.Read(new byte[dataSize], 0, dataSize);
            }
            finally
            {
                iFin.Close();
            }
        }