/// <summary>Verify the RPC server can shutdown properly when callQueue is full.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRPCServerShutdown()
{
int numClients = 3;
IList <Future <Void> > res = new AList <Future <Void> >();
ExecutorService executorService = Executors.NewFixedThreadPool(numClients);
Configuration conf = new Configuration();
conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 0);
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetQueueSizePerHandler
(1).SetNumHandlers(1).SetVerbose(true).Build();
server.Start();
TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol
.versionID, NetUtils.GetConnectAddress(server), conf);
try
{
// start a sleep RPC call to consume the only handler thread.
// Start another sleep RPC call to make callQueue full.
// Start another sleep RPC call to make reader thread block on CallQueue.
for (int i = 0; i < numClients; i++)
{
res.AddItem(executorService.Submit(new _Callable_1046(proxy)));
}
while (server.GetCallQueueLen() != 1 && CountThreads(typeof(CallQueueManager).FullName
) != 1 && CountThreads(typeof(TestRPC.TestProtocol).FullName) != 1)
{
Thread.Sleep(100);
}
}
finally
{
try
{
server.Stop();
Assert.Equal("Not enough clients", numClients, res.Count);
foreach (Future <Void> f in res)
{
try
{
f.Get();
NUnit.Framework.Assert.Fail("Future get should not return");
}
catch (ExecutionException e)
{
Assert.True("Unexpected exception: " + e, e.InnerException is IOException
);
Log.Info("Expected exception", e.InnerException);
}
}
}
finally
{
RPC.StopProxy(proxy);
executorService.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void DoRPCs(Configuration conf, bool expectFailure)
{
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).Build();
server.RefreshServiceAcl(conf, new TestRPC.TestPolicyProvider());
TestRPC.TestProtocol proxy = null;
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
try
{
proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr,
conf);
proxy.Ping();
if (expectFailure)
{
NUnit.Framework.Assert.Fail("Expect RPC.getProxy to fail with AuthorizationException!"
);
}
}
catch (RemoteException e)
{
if (expectFailure)
{
Assert.True(e.UnwrapRemoteException() is AuthorizationException
);
}
else
{
throw;
}
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(server.rpcMetrics.Name());
if (expectFailure)
{
MetricsAsserts.AssertCounter("RpcAuthorizationFailures", 1L, rb);
}
else
{
MetricsAsserts.AssertCounter("RpcAuthorizationSuccesses", 1L, rb);
}
//since we don't have authentication turned ON, we should see
// 0 for the authentication successes and 0 for failure
MetricsAsserts.AssertCounter("RpcAuthenticationFailures", 0L, rb);
MetricsAsserts.AssertCounter("RpcAuthenticationSuccesses", 0L, rb);
}
}
public virtual void TestSlowRpc()
{
System.Console.Out.WriteLine("Testing Slow RPC");
// create a server with two handlers
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(2).SetVerbose
(false).Build();
TestRPC.TestProtocol proxy = null;
try
{
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
// create a client
proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr,
conf);
TestRPC.SlowRPC slowrpc = new TestRPC.SlowRPC(proxy);
Thread thread = new Thread(slowrpc, "SlowRPC");
thread.Start();
// send a slow RPC, which won't return until two fast pings
Assert.True("Slow RPC should not have finished1.", !slowrpc.IsDone
());
proxy.SlowPing(false);
// first fast ping
// verify that the first RPC is still stuck
Assert.True("Slow RPC should not have finished2.", !slowrpc.IsDone
());
proxy.SlowPing(false);
// second fast ping
// Now the slow ping should be able to be executed
while (!slowrpc.IsDone())
{
System.Console.Out.WriteLine("Waiting for slow RPC to get done.");
try
{
Thread.Sleep(1000);
}
catch (Exception)
{
}
}
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
System.Console.Out.WriteLine("Down slow rpc testing");
}
}
public virtual void TestNMAuditLoggerWithIP()
{
Configuration conf = new Configuration();
// start the IPC server
Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol
)).SetInstance(new TestNMAuditLogger.MyTestRPCServer(this)).SetBindAddress("0.0.0.0"
).SetPort(0).SetNumHandlers(5).SetVerbose(true).Build();
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
// Make a client connection and test the audit log
TestRPC.TestProtocol proxy = (TestRPC.TestProtocol)RPC.GetProxy <TestRPC.TestProtocol
>(TestRPC.TestProtocol.versionID, addr, conf);
// Start the testcase
proxy.Ping();
server.Stop();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
Org.Apache.Hadoop.Ipc.Server server;
try
{
secretMgr = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey);
server = new RPC.Builder(conf).SetProtocol(typeof(TestClientToAMTokens.CustomProtocol
)).SetNumHandlers(1).SetSecretManager(secretMgr).SetInstance(this).Build();
}
catch (Exception e)
{
throw new YarnRuntimeException(e);
}
server.Start();
this.address = NetUtils.GetConnectAddress(server);
base.ServiceStart();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestRPCInterrupted()
{
Configuration conf = new Configuration();
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(null).Build();
server.Start();
int numConcurrentRPC = 200;
IPEndPoint addr = NetUtils.GetConnectAddress(server);
CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
AtomicBoolean leaderRunning = new AtomicBoolean(true);
AtomicReference <Exception> error = new AtomicReference <Exception>();
Thread leaderThread = null;
for (int i = 0; i < numConcurrentRPC; i++)
{
int num = i;
TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol
.versionID, addr, conf);
Thread rpcThread = new Thread(new _Runnable_915(barrier, num, leaderRunning
, proxy, error, latch));
rpcThread.Start();
if (leaderThread == null)
{
leaderThread = rpcThread;
}
}
// let threads get past the barrier
Thread.Sleep(1000);
// stop a single thread
while (leaderRunning.Get())
{
leaderThread.Interrupt();
}
latch.Await();
// should not cause any other thread to get an error
Assert.True("rpc got exception " + error.Get(), error.Get() ==
null);
server.Stop();
}
public virtual void TestRpcMetrics()
{
Configuration configuration = new Configuration();
int interval = 1;
configuration.SetBoolean(CommonConfigurationKeys.RpcMetricsQuantileEnable, true);
configuration.Set(CommonConfigurationKeys.RpcMetricsPercentilesIntervalsKey, string.Empty
+ interval);
Server server = new RPC.Builder(configuration).SetProtocol(typeof(TestRPC.TestProtocol
)).SetInstance(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers
(5).SetVerbose(true).Build();
server.Start();
TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol
.versionID, server.GetListenerAddress(), configuration);
try
{
for (int i = 0; i < 1000; i++)
{
proxy.Ping();
proxy.Echo(string.Empty + i);
}
MetricsRecordBuilder rpcMetrics = MetricsAsserts.GetMetrics(server.GetRpcMetrics(
).Name());
Assert.True("Expected non-zero rpc queue time", MetricsAsserts.GetLongCounter
("RpcQueueTimeNumOps", rpcMetrics) > 0);
Assert.True("Expected non-zero rpc processing time", MetricsAsserts.GetLongCounter
("RpcProcessingTimeNumOps", rpcMetrics) > 0);
MetricsAsserts.AssertQuantileGauges("RpcQueueTime" + interval + "s", rpcMetrics);
MetricsAsserts.AssertQuantileGauges("RpcProcessingTime" + interval + "s", rpcMetrics
);
}
finally
{
if (proxy != null)
{
RPC.StopProxy(proxy);
}
server.Stop();
}
}
public virtual void TestRealUserIPAuthorizationFailure()
{
Configuration conf = new Configuration();
conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(RealUserShortName), "20.20.20.20");
//Authorized IP address
conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(RealUserShortName), "group1");
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol
)).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address)
.SetPort(0).SetNumHandlers(2).SetVerbose(false).Build();
RefreshConf(conf);
try
{
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
UserGroupInformation realUserUgi = UserGroupInformation.CreateRemoteUser(RealUserName
);
UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting
(ProxyUserName, realUserUgi, GroupNames);
string retVal = proxyUserUgi.DoAs(new _PrivilegedExceptionAction_276(this, addr,
conf));
NUnit.Framework.Assert.Fail("The RPC must have failed " + retVal);
}
catch (Exception e)
{
Runtime.PrintStackTrace(e);
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}
private IPEndPoint StartAndGetRPCServerAddress(IPEndPoint serverAddress)
{
Configuration conf = new Configuration();
try
{
RPC.SetProtocolEngine(conf, typeof(HAServiceProtocolPB), typeof(ProtobufRpcEngine
));
HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator = new HAServiceProtocolServerSideTranslatorPB
(new DummyHAService.MockHAProtocolImpl(this));
BlockingService haPbService = HAServiceProtocolProtos.HAServiceProtocolService.NewReflectiveBlockingService
(haServiceProtocolXlator);
Server server = new RPC.Builder(conf).SetProtocol(typeof(HAServiceProtocolPB)).SetInstance
(haPbService).SetBindAddress(serverAddress.GetHostName()).SetPort(serverAddress.
Port).Build();
server.Start();
return(NetUtils.GetConnectAddress(server));
}
catch (IOException)
{
return(null);
}
}
public virtual void TestProxyWithToken()
{
Configuration conf = new Configuration(masterConf);
TestSaslRPC.TestTokenSecretManager sm = new TestSaslRPC.TestTokenSecretManager();
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, conf);
UserGroupInformation.SetConfiguration(conf);
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol
)).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address)
.SetPort(0).SetNumHandlers(5).SetVerbose(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.CreateRemoteUser(RealUserName
);
IPEndPoint addr = NetUtils.GetConnectAddress(server);
TestSaslRPC.TestTokenIdentifier tokenId = new TestSaslRPC.TestTokenIdentifier(new
Org.Apache.Hadoop.IO.Text(current.GetUserName()), new Org.Apache.Hadoop.IO.Text(
"SomeSuperUser"));
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier> token = new
Org.Apache.Hadoop.Security.Token.Token <TestSaslRPC.TestTokenIdentifier>(tokenId,
sm);
SecurityUtil.SetTokenService(token, addr);
UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting
(ProxyUserName, current, GroupNames);
proxyUserUgi.AddToken(token);
RefreshConf(conf);
string retVal = proxyUserUgi.DoAs(new _PrivilegedExceptionAction_457(this, addr,
conf, server));
//The user returned by server must be the one in the token.
Assert.Equal(RealUserName + " (auth:TOKEN) via SomeSuperUser (auth:SIMPLE)"
, retVal);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestRealUserAuthorizationSuccess()
{
Configuration conf = new Configuration();
ConfigureSuperUserIPAddresses(conf, RealUserShortName);
conf.SetStrings(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(RealUserShortName), "group1");
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestDoAsEffectiveUser.TestProtocol
)).SetInstance(new TestDoAsEffectiveUser.TestImpl(this)).SetBindAddress(Address)
.SetPort(0).SetNumHandlers(2).SetVerbose(false).Build();
RefreshConf(conf);
try
{
server.Start();
UserGroupInformation realUserUgi = UserGroupInformation.CreateRemoteUser(RealUserName
);
CheckRemoteUgi(server, realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUserForTesting
(ProxyUserName, realUserUgi, GroupNames);
CheckRemoteUgi(server, proxyUserUgi, conf);
}
catch (Exception e)
{
Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail();
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}
public virtual void TestProxyAddress()
{
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).Build();
TestRPC.TestProtocol proxy = null;
try
{
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
// create a client
proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr,
conf);
Assert.Equal(addr, RPC.GetServerAddress(proxy));
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}
public virtual void TestErrorMsgForInsecureClient()
{
Configuration serverConf = new Configuration(conf);
SecurityUtil.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos
, serverConf);
UserGroupInformation.SetConfiguration(serverConf);
Server server = new RPC.Builder(serverConf).SetProtocol(typeof(TestRPC.TestProtocol
)).SetInstance(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers
(5).SetVerbose(true).Build();
server.Start();
UserGroupInformation.SetConfiguration(conf);
bool succeeded = false;
IPEndPoint addr = NetUtils.GetConnectAddress(server);
TestRPC.TestProtocol proxy = null;
try
{
proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr,
conf);
proxy.Echo(string.Empty);
}
catch (RemoteException e)
{
Log.Info("LOGGING MESSAGE: " + e.GetLocalizedMessage());
Assert.True(e.UnwrapRemoteException() is AccessControlException
);
succeeded = true;
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
Assert.True(succeeded);
conf.SetInt(CommonConfigurationKeys.IpcServerRpcReadThreadsKey, 2);
UserGroupInformation.SetConfiguration(serverConf);
Server multiServer = new RPC.Builder(serverConf).SetProtocol(typeof(TestRPC.TestProtocol
)).SetInstance(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetNumHandlers
(5).SetVerbose(true).Build();
multiServer.Start();
succeeded = false;
IPEndPoint mulitServerAddr = NetUtils.GetConnectAddress(multiServer);
proxy = null;
try
{
UserGroupInformation.SetConfiguration(conf);
proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, mulitServerAddr
, conf);
proxy.Echo(string.Empty);
}
catch (RemoteException e)
{
Log.Info("LOGGING MESSAGE: " + e.GetLocalizedMessage());
Assert.True(e.UnwrapRemoteException() is AccessControlException
);
succeeded = true;
}
finally
{
multiServer.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
Assert.True(succeeded);
}
/// <exception cref="System.IO.IOException"/>
private void TestCallsInternal(Configuration conf)
{
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).Build();
TestRPC.TestProtocol proxy = null;
try
{
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol.versionID, addr,
conf);
proxy.Ping();
string stringResult = proxy.Echo("foo");
Assert.Equal(stringResult, "foo");
stringResult = proxy.Echo((string)null);
Assert.Equal(stringResult, null);
// Check rpcMetrics
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(server.rpcMetrics.Name());
MetricsAsserts.AssertCounter("RpcProcessingTimeNumOps", 3L, rb);
MetricsAsserts.AssertCounterGt("SentBytes", 0L, rb);
MetricsAsserts.AssertCounterGt("ReceivedBytes", 0L, rb);
// Number of calls to echo method should be 2
rb = MetricsAsserts.GetMetrics(server.rpcDetailedMetrics.Name());
MetricsAsserts.AssertCounter("EchoNumOps", 2L, rb);
// Number of calls to ping method should be 1
MetricsAsserts.AssertCounter("PingNumOps", 1L, rb);
string[] stringResults = proxy.Echo(new string[] { "foo", "bar" });
Assert.True(Arrays.Equals(stringResults, new string[] { "foo",
"bar" }));
stringResults = proxy.Echo((string[])null);
Assert.True(Arrays.Equals(stringResults, null));
UTF8 utf8Result = (UTF8)proxy.Echo(new UTF8("hello world"));
Assert.Equal(new UTF8("hello world"), utf8Result);
utf8Result = (UTF8)proxy.Echo((UTF8)null);
Assert.Equal(null, utf8Result);
int intResult = proxy.Add(1, 2);
Assert.Equal(intResult, 3);
intResult = proxy.Add(new int[] { 1, 2 });
Assert.Equal(intResult, 3);
// Test protobufs
DescriptorProtos.EnumDescriptorProto sendProto = ((DescriptorProtos.EnumDescriptorProto
)DescriptorProtos.EnumDescriptorProto.NewBuilder().SetName("test").Build());
DescriptorProtos.EnumDescriptorProto retProto = proxy.ExchangeProto(sendProto);
Assert.Equal(sendProto, retProto);
NUnit.Framework.Assert.AreNotSame(sendProto, retProto);
bool caught = false;
try
{
proxy.Error();
}
catch (IOException e)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Caught " + e);
}
caught = true;
}
Assert.True(caught);
rb = MetricsAsserts.GetMetrics(server.rpcDetailedMetrics.Name());
MetricsAsserts.AssertCounter("IOExceptionNumOps", 1L, rb);
proxy.TestServerGet();
// create multiple threads and make them do large data transfers
System.Console.Out.WriteLine("Starting multi-threaded RPC test...");
server.SetSocketSendBufSize(1024);
Thread[] threadId = new Thread[numThreads];
for (int i = 0; i < numThreads; i++)
{
TestRPC.Transactions trans = new TestRPC.Transactions(proxy, datasize);
threadId[i] = new Thread(trans, "TransactionThread-" + i);
threadId[i].Start();
}
// wait for all transactions to get over
System.Console.Out.WriteLine("Waiting for all threads to finish RPCs...");
for (int i_1 = 0; i_1 < numThreads; i_1++)
{
try
{
threadId[i_1].Join();
}
catch (Exception)
{
i_1--;
}
}
}
finally
{
// retry
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}