/// <exception cref="System.IO.IOException"/> public virtual void TestRPCInterruptedSimple() { Configuration conf = new Configuration(); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(null).Build(); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, addr, conf); // Connect to the server proxy.Ping(); // Interrupt self, try another call Thread.CurrentThread().Interrupt(); try { proxy.Ping(); NUnit.Framework.Assert.Fail("Interruption did not cause IPC to fail"); } catch (IOException ioe) { if (!ioe.ToString().Contains("InterruptedException")) { throw; } // clear interrupt status for future tests Thread.Interrupted(); } finally { server.Stop(); } }
public virtual void TestStopsAllThreads() { int threadsBefore = CountThreads("Server$Listener$Reader"); Assert.Equal("Expect no Reader threads running before test", 0 , threadsBefore); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).Build(); server.Start(); try { // Wait for at least one reader thread to start int threadsRunning = 0; long totalSleepTime = 0; do { totalSleepTime += 10; Thread.Sleep(10); threadsRunning = CountThreads("Server$Listener$Reader"); }while (threadsRunning == 0 && totalSleepTime < 5000); // Validate that at least one thread started (we didn't timeout) threadsRunning = CountThreads("Server$Listener$Reader"); Assert.True(threadsRunning > 0); } finally { server.Stop(); } int threadsAfter = CountThreads("Server$Listener$Reader"); Assert.Equal("Expect no Reader threads left running after test" , 0, threadsAfter); }
public virtual void TestRealUserGroupNotSpecified() { Configuration conf = new Configuration(); ConfigureSuperUserIPAddresses(conf, RealUserShortName); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol )).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address) .SetPort(0).SetNumHandlers(2).SetVerbose(false).Build(); try { server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); UserGroupInformation realUserUgi = UserGroupInformation.CreateRemoteUser(RealUserName ); UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting (ProxyUserName, realUserUgi, GroupNames); string retVal = proxyUserUgi.DoAs(new _PrivilegedExceptionAction_359(this, addr, conf)); NUnit.Framework.Assert.Fail("The RPC must have failed " + retVal); } catch (Exception e) { Runtime.PrintStackTrace(e); } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
public virtual void TestConnectionPing() { Configuration conf = new Configuration(); int pingInterval = 50; conf.SetBoolean(CommonConfigurationKeys.IpcClientPingKey, true); conf.SetInt(CommonConfigurationKeys.IpcPingIntervalKey, pingInterval); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).Build(); server.Start(); TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, server.GetListenerAddress(), conf); try { // this call will throw exception if server couldn't decode the ping proxy.Sleep(pingInterval * 4); } finally { if (proxy != null) { RPC.StopProxy(proxy); } server.Stop(); } }
/// <summary>Verify the RPC server can shutdown properly when callQueue is full.</summary> /// <exception cref="System.Exception"/> public virtual void TestRPCServerShutdown() { int numClients = 3; IList <Future <Void> > res = new AList <Future <Void> >(); ExecutorService executorService = Executors.NewFixedThreadPool(numClients); Configuration conf = new Configuration(); conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 0); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetQueueSizePerHandler (1).SetNumHandlers(1).SetVerbose(true).Build(); server.Start(); TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, NetUtils.GetConnectAddress(server), conf); try { // start a sleep RPC call to consume the only handler thread. // Start another sleep RPC call to make callQueue full. // Start another sleep RPC call to make reader thread block on CallQueue. for (int i = 0; i < numClients; i++) { res.AddItem(executorService.Submit(new _Callable_1046(proxy))); } while (server.GetCallQueueLen() != 1 && CountThreads(typeof(CallQueueManager).FullName ) != 1 && CountThreads(typeof(TestRPC.TestProtocol).FullName) != 1) { Thread.Sleep(100); } } finally { try { server.Stop(); Assert.Equal("Not enough clients", numClients, res.Count); foreach (Future <Void> f in res) { try { f.Get(); NUnit.Framework.Assert.Fail("Future get should not return"); } catch (ExecutionException e) { Assert.True("Unexpected exception: " + e, e.InnerException is IOException ); Log.Info("Expected exception", e.InnerException); } } } finally { RPC.StopProxy(proxy); executorService.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> private void DoRPCs(Configuration conf, bool expectFailure) { Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).Build(); server.RefreshServiceAcl(conf, new TestRPC.TestPolicyProvider()); TestRPC.TestProtocol proxy = null; server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); try { proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr, conf); proxy.Ping(); if (expectFailure) { NUnit.Framework.Assert.Fail("Expect RPC.getProxy to fail with AuthorizationException!" ); } } catch (RemoteException e) { if (expectFailure) { Assert.True(e.UnwrapRemoteException() is AuthorizationException ); } else { throw; } } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(server.rpcMetrics.Name()); if (expectFailure) { MetricsAsserts.AssertCounter("RpcAuthorizationFailures", 1L, rb); } else { MetricsAsserts.AssertCounter("RpcAuthorizationSuccesses", 1L, rb); } //since we don't have authentication turned ON, we should see // 0 for the authentication successes and 0 for failure MetricsAsserts.AssertCounter("RpcAuthenticationFailures", 0L, rb); MetricsAsserts.AssertCounter("RpcAuthenticationSuccesses", 0L, rb); } }
public virtual void TestSlowRpc() { System.Console.Out.WriteLine("Testing Slow RPC"); // create a server with two handlers Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(2).SetVerbose (false).Build(); TestRPC.TestProtocol proxy = null; try { server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); // create a client proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr, conf); TestRPC.SlowRPC slowrpc = new TestRPC.SlowRPC(proxy); Thread thread = new Thread(slowrpc, "SlowRPC"); thread.Start(); // send a slow RPC, which won't return until two fast pings Assert.True("Slow RPC should not have finished1.", !slowrpc.IsDone ()); proxy.SlowPing(false); // first fast ping // verify that the first RPC is still stuck Assert.True("Slow RPC should not have finished2.", !slowrpc.IsDone ()); proxy.SlowPing(false); // second fast ping // Now the slow ping should be able to be executed while (!slowrpc.IsDone()) { System.Console.Out.WriteLine("Waiting for slow RPC to get done."); try { Thread.Sleep(1000); } catch (Exception) { } } } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } System.Console.Out.WriteLine("Down slow rpc testing"); } }
public virtual void TestServerAddress() { Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).Build(); IPEndPoint bindAddr = null; try { bindAddr = NetUtils.GetConnectAddress(server); } finally { server.Stop(); } Assert.Equal(Runtime.GetLocalHost(), bindAddr.Address); }
public virtual void TestNMAuditLoggerWithIP() { Configuration conf = new Configuration(); // start the IPC server Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol )).SetInstance(new TestNMAuditLogger.MyTestRPCServer(this)).SetBindAddress("0.0.0.0" ).SetPort(0).SetNumHandlers(5).SetVerbose(true).Build(); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); // Make a client connection and test the audit log TestRPC.TestProtocol proxy = (TestRPC.TestProtocol)RPC.GetProxy <TestRPC.TestProtocol >(TestRPC.TestProtocol.versionID, addr, conf); // Start the testcase proxy.Ping(); server.Stop(); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestRPCInterrupted() { Configuration conf = new Configuration(); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(null).Build(); server.Start(); int numConcurrentRPC = 200; IPEndPoint addr = NetUtils.GetConnectAddress(server); CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC); CountDownLatch latch = new CountDownLatch(numConcurrentRPC); AtomicBoolean leaderRunning = new AtomicBoolean(true); AtomicReference <Exception> error = new AtomicReference <Exception>(); Thread leaderThread = null; for (int i = 0; i < numConcurrentRPC; i++) { int num = i; TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, addr, conf); Thread rpcThread = new Thread(new _Runnable_915(barrier, num, leaderRunning , proxy, error, latch)); rpcThread.Start(); if (leaderThread == null) { leaderThread = rpcThread; } } // let threads get past the barrier Thread.Sleep(1000); // stop a single thread while (leaderRunning.Get()) { leaderThread.Interrupt(); } latch.Await(); // should not cause any other thread to get an error Assert.True("rpc got exception " + error.Get(), error.Get() == null); server.Stop(); }
public virtual void TestRpcMetrics() { Configuration configuration = new Configuration(); int interval = 1; configuration.SetBoolean(CommonConfigurationKeys.RpcMetricsQuantileEnable, true); configuration.Set(CommonConfigurationKeys.RpcMetricsPercentilesIntervalsKey, string.Empty + interval); Server server = new RPC.Builder(configuration).SetProtocol(typeof(TestRPC.TestProtocol )).SetInstance(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers (5).SetVerbose(true).Build(); server.Start(); TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, server.GetListenerAddress(), configuration); try { for (int i = 0; i < 1000; i++) { proxy.Ping(); proxy.Echo(string.Empty + i); } MetricsRecordBuilder rpcMetrics = MetricsAsserts.GetMetrics(server.GetRpcMetrics( ).Name()); Assert.True("Expected non-zero rpc queue time", MetricsAsserts.GetLongCounter ("RpcQueueTimeNumOps", rpcMetrics) > 0); Assert.True("Expected non-zero rpc processing time", MetricsAsserts.GetLongCounter ("RpcProcessingTimeNumOps", rpcMetrics) > 0); MetricsAsserts.AssertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics); MetricsAsserts.AssertQuantileGauges("RpcProcessingTime" + interval + "s", rpcMetrics ); } finally { if (proxy != null) { RPC.StopProxy(proxy); } server.Stop(); } }
public virtual void TestConfRpc() { Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(1).SetVerbose (false).Build(); // Just one handler int confQ = conf.GetInt(CommonConfigurationKeys.IpcServerHandlerQueueSizeKey, CommonConfigurationKeys .IpcServerHandlerQueueSizeDefault); Assert.Equal(confQ, server.GetMaxQueueSize()); int confReaders = conf.GetInt(CommonConfigurationKeys.IpcServerRpcReadThreadsKey, CommonConfigurationKeys.IpcServerRpcReadThreadsDefault); Assert.Equal(confReaders, server.GetNumReaders()); server.Stop(); server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(1).SetnumReaders (3).SetQueueSizePerHandler(200).SetVerbose(false).Build(); Assert.Equal(3, server.GetNumReaders()); Assert.Equal(200, server.GetMaxQueueSize()); server.Stop(); }
public virtual void TestRealUserIPAuthorizationFailure() { Configuration conf = new Configuration(); conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (RealUserShortName), "20.20.20.20"); //Authorized IP address conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (RealUserShortName), "group1"); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol )).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address) .SetPort(0).SetNumHandlers(2).SetVerbose(false).Build(); RefreshConf(conf); try { server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); UserGroupInformation realUserUgi = UserGroupInformation.CreateRemoteUser(RealUserName ); UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting (ProxyUserName, realUserUgi, GroupNames); string retVal = proxyUserUgi.DoAs(new _PrivilegedExceptionAction_276(this, addr, conf)); NUnit.Framework.Assert.Fail("The RPC must have failed " + retVal); } catch (Exception e) { Runtime.PrintStackTrace(e); } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
/// <exception cref="System.IO.IOException"/> public virtual void TestRealUserAuthorizationSuccess() { Configuration conf = new Configuration(); ConfigureSuperUserIPAddresses(conf, RealUserShortName); conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (RealUserShortName), "group1"); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol )).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address) .SetPort(0).SetNumHandlers(2).SetVerbose(false).Build(); RefreshConf(conf); try { server.Start(); UserGroupInformation realUserUgi = UserGroupInformation.CreateRemoteUser(RealUserName ); CheckRemoteUgi(server, realUserUgi, conf); UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting (ProxyUserName, realUserUgi, GroupNames); CheckRemoteUgi(server, proxyUserUgi, conf); } catch (Exception e) { Runtime.PrintStackTrace(e); NUnit.Framework.Assert.Fail(); } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
public virtual void TestProxyAddress() { Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).Build(); TestRPC.TestProtocol proxy = null; try { server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); // create a client proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr, conf); Assert.Equal(addr, RPC.GetServerAddress(proxy)); } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
/// <exception cref="System.IO.IOException"/> private void TestCallsInternal(Configuration conf) { Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).Build(); TestRPC.TestProtocol proxy = null; try { server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr, conf); proxy.Ping(); string stringResult = proxy.Echo("foo"); Assert.Equal(stringResult, "foo"); stringResult = proxy.Echo((string)null); Assert.Equal(stringResult, null); // Check rpcMetrics MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(server.rpcMetrics.Name()); MetricsAsserts.AssertCounter("RpcProcessingTimeNumOps", 3L, rb); MetricsAsserts.AssertCounterGt("SentBytes", 0L, rb); MetricsAsserts.AssertCounterGt("ReceivedBytes", 0L, rb); // Number of calls to echo method should be 2 rb = MetricsAsserts.GetMetrics(server.rpcDetailedMetrics.Name()); MetricsAsserts.AssertCounter("EchoNumOps", 2L, rb); // Number of calls to ping method should be 1 MetricsAsserts.AssertCounter("PingNumOps", 1L, rb); string[] stringResults = proxy.Echo(new string[] { "foo", "bar" }); Assert.True(Arrays.Equals(stringResults, new string[] { "foo", "bar" })); stringResults = proxy.Echo((string[])null); Assert.True(Arrays.Equals(stringResults, null)); UTF8 utf8Result = (UTF8)proxy.Echo(new UTF8("hello world")); Assert.Equal(new UTF8("hello world"), utf8Result); utf8Result = (UTF8)proxy.Echo((UTF8)null); Assert.Equal(null, utf8Result); int intResult = proxy.Add(1, 2); Assert.Equal(intResult, 3); intResult = proxy.Add(new int[] { 1, 2 }); Assert.Equal(intResult, 3); // Test protobufs DescriptorProtos.EnumDescriptorProto sendProto = ((DescriptorProtos.EnumDescriptorProto )DescriptorProtos.EnumDescriptorProto.NewBuilder().SetName("test").Build()); DescriptorProtos.EnumDescriptorProto retProto = proxy.ExchangeProto(sendProto); Assert.Equal(sendProto, retProto); NUnit.Framework.Assert.AreNotSame(sendProto, retProto); bool caught = false; try { proxy.Error(); } catch (IOException e) { if (Log.IsDebugEnabled()) { Log.Debug("Caught " + e); } caught = true; } Assert.True(caught); rb = MetricsAsserts.GetMetrics(server.rpcDetailedMetrics.Name()); MetricsAsserts.AssertCounter("IOExceptionNumOps", 1L, rb); proxy.TestServerGet(); // create multiple threads and make them do large data transfers System.Console.Out.WriteLine("Starting multi-threaded RPC test..."); server.SetSocketSendBufSize(1024); Thread[] threadId = new Thread[numThreads]; for (int i = 0; i < numThreads; i++) { TestRPC.Transactions trans = new TestRPC.Transactions(proxy, datasize); threadId[i] = new Thread(trans, "TransactionThread-" + i); threadId[i].Start(); } // wait for all transactions to get over System.Console.Out.WriteLine("Waiting for all threads to finish RPCs..."); for (int i_1 = 0; i_1 < numThreads; i_1++) { try { threadId[i_1].Join(); } catch (Exception) { i_1--; } } } finally { // retry server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
public virtual void TestErrorMsgForInsecureClient() { Configuration serverConf = new Configuration(conf); SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos , serverConf); UserGroupInformation.SetConfiguration(serverConf); Server server = new RPC.Builder(serverConf).SetProtocol(typeof(TestRPC.TestProtocol )).SetInstance(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers (5).SetVerbose(true).Build(); server.Start(); UserGroupInformation.SetConfiguration(conf); bool succeeded = false; IPEndPoint addr = NetUtils.GetConnectAddress(server); TestRPC.TestProtocol proxy = null; try { proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr, conf); proxy.Echo(string.Empty); } catch (RemoteException e) { Log.Info("LOGGING MESSAGE: " + e.GetLocalizedMessage()); Assert.True(e.UnwrapRemoteException() is AccessControlException ); succeeded = true; } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } Assert.True(succeeded); conf.SetInt(CommonConfigurationKeys.IpcServerRpcReadThreadsKey, 2); UserGroupInformation.SetConfiguration(serverConf); Server multiServer = new RPC.Builder(serverConf).SetProtocol(typeof(TestRPC.TestProtocol )).SetInstance(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers (5).SetVerbose(true).Build(); multiServer.Start(); succeeded = false; IPEndPoint mulitServerAddr = NetUtils.GetConnectAddress(multiServer); proxy = null; try { UserGroupInformation.SetConfiguration(conf); proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, mulitServerAddr , conf); proxy.Echo(string.Empty); } catch (RemoteException e) { Log.Info("LOGGING MESSAGE: " + e.GetLocalizedMessage()); Assert.True(e.UnwrapRemoteException() is AccessControlException ); succeeded = true; } finally { multiServer.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } Assert.True(succeeded); }