public virtual void TestUnknownCall()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
// Any unrelated protocol would do
ApplicationClientProtocol proxy = (ApplicationClientProtocol)rpc.GetProxy(typeof(
ApplicationClientProtocol), NetUtils.GetConnectAddress(server), conf);
try
{
proxy.GetNewApplication(Records.NewRecord <GetNewApplicationRequest>());
NUnit.Framework.Assert.Fail("Excepted RPC call to fail with unknown method.");
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Matches("Unknown method getNewApplication called on.*"
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."
));
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
internal virtual LocalizationProtocol GetProxy(IPEndPoint nmAddr)
{
YarnRPC rpc = YarnRPC.Create(conf);
return((LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol), nmAddr, conf
));
}
/// <exception cref="System.Exception"/>
public MRClientProtocol Run()
{
YarnRPC rpc = YarnRPC.Create(conf);
return((MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService
.GetBindAddress(), conf));
}
protected internal virtual ClientSCMProtocol CreateClientProxy()
{
YarnRPC rpc = YarnRPC.Create(GetConfig());
return((ClientSCMProtocol)rpc.GetProxy(typeof(ClientSCMProtocol), this.scmAddress
, GetConfig()));
}
protected internal override YarnRPC GetYarnRPC()
{
YarnRPC mockRpc = Org.Mockito.Mockito.Mock <YarnRPC>();
Org.Mockito.Mockito.When(mockRpc.GetProxy(Matchers.Any <Type>(), Matchers.Any <IPEndPoint
>(), Matchers.Any <Configuration>())).ThenReturn(mockProxy);
return(mockRpc);
}
private HSClientProtocol InstantiateHistoryProxy()
{
string serviceAddr = mrCluster.GetConfig().Get(JHAdminConfig.MrHistoryAddress);
YarnRPC rpc = YarnRPC.Create(conf);
HSClientProtocol historyClient = (HSClientProtocol)rpc.GetProxy(typeof(HSClientProtocol
), NetUtils.CreateSocketAddr(serviceAddr), mrCluster.GetConfig());
return(historyClient);
}
private SCMUploaderProtocol CreateSCMClient(Configuration conf)
{
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmUploaderServerAddress
, YarnConfiguration.DefaultScmUploaderServerAddress, YarnConfiguration.DefaultScmUploaderServerPort
);
return((SCMUploaderProtocol)rpc.GetProxy(typeof(SCMUploaderProtocol), scmAddress,
conf));
}
public ContainerManagementProtocol Run()
{
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
return((ContainerManagementProtocol)rpc.GetProxy(typeof(ContainerManagementProtocol
), containerManagerBindAddress, conf));
}
/// <exception cref="System.IO.IOException"/>
internal virtual MRClientProtocol InstantiateAMProxy(IPEndPoint serviceAddr)
{
Log.Trace("Connecting to ApplicationMaster at: " + serviceAddr);
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
serviceAddr, conf);
usingAMProxy.Set(true);
Log.Trace("Connected to ApplicationMaster at: " + serviceAddr);
return(proxy);
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
/// <exception cref="System.IO.IOException"/>
protected override GetUserMappingsProtocol GetUgmProtocol()
{
Configuration conf = GetConf();
IPEndPoint addr = conf.GetSocketAddr(YarnConfiguration.RmAdminAddress, YarnConfiguration
.DefaultRmAdminAddress, YarnConfiguration.DefaultRmAdminPort);
YarnRPC rpc = YarnRPC.Create(conf);
ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol
)rpc.GetProxy(typeof(ResourceManagerAdministrationProtocol), addr, GetConf());
return(adminProtocol);
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual SCMAdminProtocol CreateSCMAdminProtocol()
{
// Get the current configuration
YarnConfiguration conf = new YarnConfiguration(GetConf());
// Create the admin client
IPEndPoint addr = conf.GetSocketAddr(YarnConfiguration.ScmAdminAddress, YarnConfiguration
.DefaultScmAdminAddress, YarnConfiguration.DefaultScmAdminPort);
YarnRPC rpc = YarnRPC.Create(conf);
SCMAdminProtocol scmAdminProtocol = (SCMAdminProtocol)rpc.GetProxy(typeof(SCMAdminProtocol
), addr, conf);
return(scmAdminProtocol);
}
public virtual void StartUp()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.ScmStoreClass, typeof(InMemorySCMStore).FullName);
cleaner = Org.Mockito.Mockito.Mock <CleanerService>();
service = Org.Mockito.Mockito.Spy(new SCMAdminProtocolService(cleaner));
service.Init(conf);
service.Start();
YarnRPC rpc = YarnRPC.Create(new Configuration());
IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmAdminAddress, YarnConfiguration
.DefaultScmAdminAddress, YarnConfiguration.DefaultScmAdminPort);
SCMAdminProxy = (SCMAdminProtocol)rpc.GetProxy(typeof(SCMAdminProtocol), scmAddress
, conf);
mockAdmin = Org.Mockito.Mockito.Mock <SCMAdminProtocol>();
adminCLI = new _SCMAdmin_90(new Configuration());
}
public virtual void StartUp()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.ScmStoreClass, typeof(InMemorySCMStore).FullName);
conf.Set(YarnConfiguration.SharedCacheRoot, testDir.GetPath());
AppChecker appChecker = Org.Mockito.Mockito.Spy(new DummyAppChecker());
store = new InMemorySCMStore(appChecker);
store.Init(conf);
store.Start();
service = new ClientProtocolService(store);
service.Init(conf);
service.Start();
YarnRPC rpc = YarnRPC.Create(new Configuration());
IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmClientServerAddress
, YarnConfiguration.DefaultScmClientServerAddress, YarnConfiguration.DefaultScmClientServerPort
);
clientSCMProxy = (ClientSCMProtocol)rpc.GetProxy(typeof(ClientSCMProtocol), scmAddress
, conf);
}
public virtual void TestLocalizerRPC()
{
IPEndPoint locAddr = new IPEndPoint("0.0.0.0", 8040);
TestPBLocalizerRPC.LocalizerService server = new TestPBLocalizerRPC.LocalizerService
(locAddr);
try
{
server.Start();
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.Create(conf);
LocalizationProtocol client = (LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol
), locAddr, conf);
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId("localizer0");
LocalizerHeartbeatResponse response = client.Heartbeat(status);
NUnit.Framework.Assert.AreEqual(DieHBResponse(), response);
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.IsTrue(true);
}
public virtual void Test()
{
TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService
(this, 1, 0, false);
Configuration conf = new Configuration();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task task = it.Next();
app.WaitForState(task, TaskState.Running);
TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next();
app.WaitForState(attempt, TaskAttemptState.Running);
// send the diagnostic
string diagnostic1 = "Diagnostic1";
string diagnostic2 = "Diagnostic2";
app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt
.GetID(), diagnostic1));
// send the status update
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = attempt.GetID();
taskAttemptStatus.progress = 0.5f;
taskAttemptStatus.stateString = "RUNNING";
taskAttemptStatus.taskState = TaskAttemptState.Running;
taskAttemptStatus.phase = Phase.Map;
// send the status update
app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt
.GetID(), taskAttemptStatus));
//verify that all object are fully populated by invoking RPCs.
YarnRPC rpc = YarnRPC.Create(conf);
MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol),
app.clientService.GetBindAddress(), conf);
GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest
>();
gcRequest.SetJobId(job.GetID());
NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest)
.GetCounters());
GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest
>();
gjrRequest.SetJobId(job.GetID());
JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport();
VerifyJobReport(jr);
GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance
<GetTaskAttemptCompletionEventsRequest>();
gtaceRequest.SetJobId(job.GetID());
gtaceRequest.SetFromEventId(0);
gtaceRequest.SetMaxEvents(10);
NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents
(gtaceRequest).GetCompletionEventList());
GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest
>();
gdRequest.SetTaskAttemptId(attempt.GetID());
NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest
).GetDiagnosticsList());
GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
>();
gtarRequest.SetTaskAttemptId(attempt.GetID());
TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport
();
VerifyTaskAttemptReport(tar);
GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest
>();
gtrRequest.SetTaskId(task.GetID());
NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest
).GetTaskReport());
GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest
>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Map);
NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(job.GetID());
gtreportsRequest.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports
(gtreportsRequest).GetTaskReportList());
IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count);
NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString
());
TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport();
NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount
());
//send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts()
.Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone));
app.WaitForState(job, JobState.Succeeded);
// For invalid jobid, throw IOException
gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>();
gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001"
)));
gtreportsRequest.SetTaskType(TaskType.Reduce);
try
{
proxy.GetTaskReports(gtreportsRequest);
NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id");
}
catch (IOException)
{
}
}
/// <exception cref="System.Exception"/>
private void Test(string rpcClass)
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
RPC.SetProtocolEngine(conf, typeof(ContainerManagementProtocolPB), typeof(ProtobufRpcEngine
));
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), NetUtils.GetConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
proxy.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(containerId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
GetContainerStatusesResponse response = proxy.GetContainerStatuses(gcsRequest);
IList <ContainerStatus> statuses = response.GetContainerStatuses();
//test remote exception
bool exception = false;
try
{
StopContainersRequest stopRequest = recordFactory.NewRecordInstance <StopContainersRequest
>();
stopRequest.SetContainerIds(containerIds);
proxy.StopContainers(stopRequest);
}
catch (YarnException e)
{
exception = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionMsg));
NUnit.Framework.Assert.IsTrue(e.Message.Contains(ExceptionCause));
System.Console.Out.WriteLine("Test Exception is " + e.Message);
}
catch (Exception ex)
{
Sharpen.Runtime.PrintStackTrace(ex);
}
NUnit.Framework.Assert.IsTrue(exception);
server.Stop();
NUnit.Framework.Assert.IsNotNull(statuses[0]);
NUnit.Framework.Assert.AreEqual(ContainerState.Running, statuses[0].GetState());
}