Esempio n. 1
0
        /// <exception cref="System.IO.IOException"/>
        private void VerifyConnection(Uri url, string msgToEncode, string encHash)
        {
            // Validate response code
            int rc = connection.GetResponseCode();

            if (rc != HttpURLConnection.HttpOk)
            {
                throw new IOException("Got invalid response code " + rc + " from " + url + ": " +
                                      connection.GetResponseMessage());
            }
            // get the shuffle version
            if (!ShuffleHeader.DefaultHttpHeaderName.Equals(connection.GetHeaderField(ShuffleHeader
                                                                                      .HttpHeaderName)) || !ShuffleHeader.DefaultHttpHeaderVersion.Equals(connection.GetHeaderField
                                                                                                                                                              (ShuffleHeader.HttpHeaderVersion)))
            {
                throw new IOException("Incompatible shuffle response version");
            }
            // get the replyHash which is HMac of the encHash we sent to the server
            string replyHash = connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                         );

            if (replyHash == null)
            {
                throw new IOException("security validation of TT Map output failed");
            }
            Log.Debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash=" + replyHash
                      );
            // verify that replyHash is HMac of encHash
            SecureShuffleUtils.VerifyReply(replyHash, encHash, shuffleSecretKey);
            Log.Info("for url=" + msgToEncode + " sent hash and received reply");
        }
Esempio n. 2
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCopyFromHostWithRetryUnreserve()
        {
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
                );
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
                                                                                      , id, ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            // Verify that unreserve occurs if an exception happens after shuffle
            // buffer is reserved.
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            Org.Mockito.Mockito.DoThrow(new IOException("forced error")).When(immo).Shuffle(Matchers.Any
                                                                                            <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
                                                                                            , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(immo).Abort();
        }
Esempio n. 3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCopyFromHostCompressFailure()
        {
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
                );
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            Org.Mockito.Mockito.DoThrow(new InternalError()).When(immo).Shuffle(Matchers.Any <
                                                                                    MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong(),
                                                                                Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
                                                                      , encHash);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(1)).CopyFailed(map1ID, host
                                                                                    , true, false);
        }
Esempio n. 4
0
        public virtual void TestCopyFromHostIncompatibleShuffleVersionWithRetry()
        {
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn("mapreduce").ThenReturn("other").ThenReturn("other");
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn("1.0.1").ThenReturn("1.0.0").ThenReturn("1.0.1");
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ByteArrayInputStream @in = new ByteArrayInputStream(new byte[0]);

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            for (int i = 0; i < 3; ++i)
            {
                Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
                                                                                          , id, ss, mm, r, metrics, except, key, connection);
                underTest.CopyFromHost(host);
            }
            Org.Mockito.Mockito.Verify(connection, Org.Mockito.Mockito.Times(3)).AddRequestProperty
                (SecureShuffleUtils.HttpHeaderUrlHash, encHash);
            Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Times(3)).Increment(1);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).CopyFailed(map1ID, host
                                                                                    , false, false);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).CopyFailed(map2ID, host
                                                                                    , false, false);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).PutBackKnownMapOutput
                (Matchers.Any <MapHost>(), Matchers.Eq(map1ID));
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Times(3)).PutBackKnownMapOutput
                (Matchers.Any <MapHost>(), Matchers.Eq(map2ID));
        }
Esempio n. 5
0
        public virtual void TestCopyFromHostBogusHeader()
        {
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ByteArrayInputStream @in = new ByteArrayInputStream(Sharpen.Runtime.GetBytesForString
                                                                    ("\u00010 BOGUS DATA\nBOGUS DATA\nBOGUS DATA\n"));

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
                                                                      , encHash);
            Org.Mockito.Mockito.Verify(allErrs).Increment(1);
            Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
            Org.Mockito.Mockito.Verify(ss).CopyFailed(map2ID, host, true, false);
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map1ID));
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map2ID));
        }
Esempio n. 6
0
        public virtual void TestReduceOutOfDiskSpace()
        {
            Log.Info("testReduceOutOfDiskSpace");
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenThrow(new DiskChecker.DiskErrorException("No disk space available"
                                                                                                                         ));
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(ss).ReportLocalError(Matchers.Any <IOException>());
        }
Esempio n. 7
0
        public virtual void TestCopyFromHostExtraBytes()
        {
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 14, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();
            DataOutputStream      dos    = new DataOutputStream(bout);
            IFileOutputStream     ios    = new IFileOutputStream(dos);

            header.Write(dos);
            ios.Write(Sharpen.Runtime.GetBytesForString("MAPDATA123"));
            ios.Finish();
            ShuffleHeader     header2 = new ShuffleHeader(map2ID.ToString(), 14, 10, 1);
            IFileOutputStream ios2    = new IFileOutputStream(dos);

            header2.Write(dos);
            ios2.Write(Sharpen.Runtime.GetBytesForString("MAPDATA456"));
            ios2.Finish();
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            // 8 < 10 therefore there appear to be extra bytes in the IFileInputStream
            InMemoryMapOutput <Text, Text> mapOut = new InMemoryMapOutput <Text, Text>(job, map1ID
                                                                                       , mm, 8, null, true);
            InMemoryMapOutput <Text, Text> mapOut2 = new InMemoryMapOutput <Text, Text>(job, map2ID
                                                                                        , mm, 10, null, true);

            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map1ID), Matchers.AnyLong(), Matchers.AnyInt
                                                    ())).ThenReturn(mapOut);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Eq(map2ID), Matchers.AnyLong(), Matchers.AnyInt
                                                    ())).ThenReturn(mapOut2);
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(allErrs).Increment(1);
            Org.Mockito.Mockito.Verify(ss).CopyFailed(map1ID, host, true, false);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
                                                                                   , true, false);
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map1ID));
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map2ID));
        }
Esempio n. 8
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInterruptOnDisk()
        {
            int           Fetcher = 7;
            Path          p       = new Path("file:///tmp/foo");
            Path          pTmp    = OnDiskMapOutput.GetTempPath(p, Fetcher);
            FileSystem    mFs     = Org.Mockito.Mockito.Mock <FileSystem>(ReturnsDeepStubs);
            MapOutputFile mof     = Org.Mockito.Mockito.Mock <MapOutputFile>();

            Org.Mockito.Mockito.When(mof.GetInputFileForWrite(Matchers.Any <TaskID>(), Matchers.AnyLong
                                                                  ())).ThenReturn(p);
            OnDiskMapOutput <Text, Text> odmo = Org.Mockito.Mockito.Spy(new OnDiskMapOutput <Text
                                                                                             , Text>(map1ID, id, mm, 100L, job, mof, Fetcher, true, mFs, p));

            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(odmo);
            Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
            Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
                                                                                    (bout.ToByteArray()));
            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.DoAnswer(new _Answer_610(@in)).When(connection).Disconnect();
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection, Fetcher);

            underTest.Start();
            // wait for read in inputstream
            @in.WaitForFetcher();
            underTest.ShutDown();
            underTest.Join();
            // rely on test timeout to kill if stuck
            NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
            Org.Mockito.Mockito.Verify(mFs).Create(Matchers.Eq(pTmp));
            Org.Mockito.Mockito.Verify(mFs).Delete(Matchers.Eq(pTmp), Matchers.Eq(false));
            Org.Mockito.Mockito.Verify(odmo).Abort();
        }
Esempio n. 9
0
			/// <exception cref="System.IO.IOException"/>
			protected internal virtual void VerifyRequest(string appid, ChannelHandlerContext
				 ctx, HttpRequest request, HttpResponse response, Uri requestUri)
			{
				SecretKey tokenSecret = this._enclosing.secretManager.RetrieveTokenSecret(appid);
				if (null == tokenSecret)
				{
					ShuffleHandler.Log.Info("Request for unknown token " + appid);
					throw new IOException("could not find jobid");
				}
				// string to encrypt
				string enc_str = SecureShuffleUtils.BuildMsgFrom(requestUri);
				// hash from the fetcher
				string urlHashStr = request.GetHeader(SecureShuffleUtils.HttpHeaderUrlHash);
				if (urlHashStr == null)
				{
					ShuffleHandler.Log.Info("Missing header hash for " + appid);
					throw new IOException("fetcher cannot be authenticated");
				}
				if (ShuffleHandler.Log.IsDebugEnabled())
				{
					int len = urlHashStr.Length;
					ShuffleHandler.Log.Debug("verifying request. enc_str=" + enc_str + "; hash=..." +
						 Sharpen.Runtime.Substring(urlHashStr, len - len / 2, len - 1));
				}
				// verify - throws exception
				SecureShuffleUtils.VerifyReply(urlHashStr, enc_str, tokenSecret);
				// verification passed - encode the reply
				string reply = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString(
					urlHashStr, Charsets.Utf8), tokenSecret);
				response.SetHeader(SecureShuffleUtils.HttpHeaderReplyUrlHash, reply);
				// Put shuffle version into http header
				response.SetHeader(ShuffleHeader.HttpHeaderName, ShuffleHeader.DefaultHttpHeaderName
					);
				response.SetHeader(ShuffleHeader.HttpHeaderVersion, ShuffleHeader.DefaultHttpHeaderVersion
					);
				if (ShuffleHandler.Log.IsDebugEnabled())
				{
					int len = reply.Length;
					ShuffleHandler.Log.Debug("Fetcher request verfied. enc_str=" + enc_str + ";reply="
						 + Sharpen.Runtime.Substring(reply, len - len / 2, len - 1));
				}
			}
Esempio n. 10
0
        /// <exception cref="System.IO.IOException"/>
        private void SetupConnectionsWithRetry(MapHost host, ICollection <TaskAttemptID> remaining
                                               , Uri url)
        {
            OpenConnectionWithRetry(host, remaining, url);
            if (stopped)
            {
                return;
            }
            // generate hash of the url
            string msgToEncode = SecureShuffleUtils.BuildMsgFrom(url);
            string encHash     = SecureShuffleUtils.HashFromString(msgToEncode, shuffleSecretKey);

            SetupShuffleConnection(encHash);
            Connect(connection, connectionTimeout);
            // verify that the thread wasn't stopped during calls to connect
            if (stopped)
            {
                return;
            }
            VerifyConnection(url, msgToEncode, encHash);
        }
Esempio n. 11
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInterruptInMemory()
        {
            int Fetcher = 2;
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Spy(new InMemoryMapOutput
                                                                          <Text, Text>(job, id, mm, 100, null, true));

            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            Org.Mockito.Mockito.DoNothing().When(mm).WaitForResource();
            Org.Mockito.Mockito.When(ss.GetHost()).ThenReturn(host);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            TestFetcher.StuckInputStream @in = new TestFetcher.StuckInputStream(new ByteArrayInputStream
                                                                                    (bout.ToByteArray()));
            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.DoAnswer(new _Answer_562(@in)).When(connection).Disconnect();
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection, Fetcher);

            underTest.Start();
            // wait for read in inputstream
            @in.WaitForFetcher();
            underTest.ShutDown();
            underTest.Join();
            // rely on test timeout to kill if stuck
            NUnit.Framework.Assert.IsTrue(@in.WasClosedProperly());
            Org.Mockito.Mockito.Verify(immo).Abort();
        }
Esempio n. 12
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCopyFromHostWithRetry()
        {
            InMemoryMapOutput <Text, Text> immo = Org.Mockito.Mockito.Mock <InMemoryMapOutput>(
                );

            ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(jobWithRetry
                                                                                      , id, ss, mm, r, metrics, except, key, connection, true);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(immo);
            long retryTime = Time.MonotonicNow();

            Org.Mockito.Mockito.DoAnswer(new _Answer_375(retryTime)).When(immo).Shuffle(Matchers.Any
                                                                                        <MapHost>(), Matchers.Any <InputStream>(), Matchers.AnyLong(), Matchers.AnyLong()
                                                                                        , Matchers.Any <ShuffleClientMetrics>(), Matchers.Any <Reporter>());
            // Emulate host down for 3 seconds.
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(Matchers.Any
                                                                                   <TaskAttemptID>(), Matchers.Any <MapHost>(), Matchers.AnyBoolean(), Matchers.AnyBoolean
                                                                                       ());
        }
Esempio n. 13
0
        public virtual void TestCopyFromHostWait()
        {
            Fetcher <Text, Text> underTest = new TestFetcher.FakeFetcher <Text, Text>(job, id,
                                                                                      ss, mm, r, metrics, except, key, connection);
            string replyHash = SecureShuffleUtils.GenerateHash(Sharpen.Runtime.GetBytesForString
                                                                   (encHash), key);

            Org.Mockito.Mockito.When(connection.GetResponseCode()).ThenReturn(200);
            Org.Mockito.Mockito.When(connection.GetHeaderField(SecureShuffleUtils.HttpHeaderReplyUrlHash
                                                               )).ThenReturn(replyHash);
            ShuffleHeader         header = new ShuffleHeader(map1ID.ToString(), 10, 10, 1);
            ByteArrayOutputStream bout   = new ByteArrayOutputStream();

            header.Write(new DataOutputStream(bout));
            ByteArrayInputStream @in = new ByteArrayInputStream(bout.ToByteArray());

            Org.Mockito.Mockito.When(connection.GetInputStream()).ThenReturn(@in);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderName))
            .ThenReturn(ShuffleHeader.DefaultHttpHeaderName);
            Org.Mockito.Mockito.When(connection.GetHeaderField(ShuffleHeader.HttpHeaderVersion
                                                               )).ThenReturn(ShuffleHeader.DefaultHttpHeaderVersion);
            //Defaults to null, which is what we want to test
            Org.Mockito.Mockito.When(mm.Reserve(Matchers.Any <TaskAttemptID>(), Matchers.AnyLong
                                                    (), Matchers.AnyInt())).ThenReturn(null);
            underTest.CopyFromHost(host);
            Org.Mockito.Mockito.Verify(connection).AddRequestProperty(SecureShuffleUtils.HttpHeaderUrlHash
                                                                      , encHash);
            Org.Mockito.Mockito.Verify(allErrs, Org.Mockito.Mockito.Never()).Increment(1);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map1ID, host
                                                                                   , true, false);
            Org.Mockito.Mockito.Verify(ss, Org.Mockito.Mockito.Never()).CopyFailed(map2ID, host
                                                                                   , true, false);
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map1ID));
            Org.Mockito.Mockito.Verify(ss).PutBackKnownMapOutput(Matchers.Any <MapHost>(), Matchers.Eq
                                                                     (map2ID));
        }
Esempio n. 14
0
        /// <exception cref="System.IO.IOException"/>
        public static string CreateDigest(byte[] password, string data)
        {
            SecretKey key = JobTokenSecretManager.CreateSecretKey(password);

            return(SecureShuffleUtils.HashFromString(data, key));
        }
Esempio n. 15
0
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual string CreateDigest(byte[] password, string data)
        {
            SecretKey key = JobTokenSecretManager.CreateSecretKey(password);

            return(SecureShuffleUtils.HashFromString(data, key));
        }