/// <exception cref="System.IO.IOException"/>
private int RunAndBlock(ContainerId cId, IDictionary <string, string> launchCtxEnv
, params string[] cmd)
{
string appId = "APP_" + Runtime.CurrentTimeMillis();
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(launchCtxEnv);
string script = WriteScriptFile(launchCtxEnv, cmd);
Path scriptPath = new Path(script);
Path tokensPath = new Path("/dev/null");
Path workDir = new Path(workSpace.GetAbsolutePath());
Path pidFile = new Path(workDir, "pid.txt");
exec.ActivateContainer(cId, pidFile);
return(exec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter, appId
, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLaunchNullImage()
{
string appSubmitter = "nobody";
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
string testImage = string.Empty;
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>(Org.Mockito.Mockito.ReturnsDeepStubs);
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
);
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
()).ThenReturn(appId);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName
, testImage);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path pidFile = new Path(workDir, "pid.txt");
dockerContainerExecutor.ActivateContainer(cId, pidFile);
dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
}
public virtual void TestAMStandardEnv()
{
string AdminLibPath = "foo";
string UserLibPath = "bar";
string UserShell = "shell";
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath);
jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath);
jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell);
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf);
// make sure PWD is first in the lib path
ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec();
IDictionary <string, string> env = clc.GetEnvironment();
string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()];
NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath);
string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig
.DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator
: FilePath.pathSeparator;
NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv
(conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath
, libPath);
// make sure SHELL is set
string shell = env[ApplicationConstants.Environment.Shell.ToString()];
NUnit.Framework.Assert.IsNotNull("SHELL not set", shell);
NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Launch()
{
Connect();
ContainerId masterContainerID = masterContainer.GetId();
ApplicationSubmissionContext applicationContext = application.GetSubmissionContext
();
Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId
());
ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext
, masterContainerID);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, masterContainer.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
StartContainersResponse response = containerMgrProxy.StartContainers(allRequests);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(masterContainerID))
{
Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize();
ParseAndThrowException(t);
}
else
{
Log.Info("Done launching container " + masterContainer + " for AM " + application
.GetAppAttemptId());
}
}
/// <exception cref="System.IO.IOException"/>
private void SetupTokens(ContainerLaunchContext container, ContainerId containerID
)
{
IDictionary <string, string> environment = container.GetEnvironment();
environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase
();
// Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId();
environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId
].GetSubmitTime().ToString();
environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId
].GetMaxAppAttempts().ToString();
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
if (container.GetTokens() != null)
{
// TODO: Don't do this kind of checks everywhere.
dibb.Reset(container.GetTokens());
credentials.ReadTokenStorageStream(dibb);
}
// Add AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken
();
if (amrmToken != null)
{
credentials.AddToken(amrmToken.GetService(), amrmToken);
}
DataOutputBuffer dob = new DataOutputBuffer();
credentials.WriteTokenStorageToStream(dob);
container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()));
}
/// <exception cref="System.Exception"/>
private void StartContainer(YarnRPC rpc, Token nmToken, Token containerToken, NodeId
nodeId, string user)
{
ContainerLaunchContext context = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext
>();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(context, containerToken
);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
ContainerManagementProtocol proxy = null;
try
{
proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
StartContainersResponse response = proxy.StartContainers(allRequests);
foreach (SerializedException ex in response.GetFailedRequests().Values)
{
ParseAndThrowException(ex.DeSerialize());
}
}
finally
{
if (proxy != null)
{
rpc.StopProxy(proxy, conf);
}
}
}
public _Thread_434(TestNMClientAsync _enclosing, Container container, ContainerLaunchContext
clc)
{
this._enclosing = _enclosing;
this.container = container;
this.clc = clc;
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool
setupACLs)
{
GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance();
ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter);
ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId
();
Resource resource = BuilderUtils.NewResource(1024, 1);
IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs
);
ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null,
null, null, null, null, acls);
ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.
NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec,
false, true, 1, resource, "applicationType");
appSubmissionContext.SetApplicationId(applicationId);
appSubmissionContext.SetQueue(queueName);
SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext
);
submitterClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
/// <exception cref="System.Exception"/>
private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList
modifyACL)
{
SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance
<GetNewApplicationRequest>()).GetApplicationId();
context.SetApplicationId(applicationId);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString();
acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString();
ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
Resource resource = BuilderUtils.NewResource(1024, 1);
context.SetResource(resource);
amContainer.SetApplicationACLs(acls);
context.SetAMContainerSpec(amContainer);
submitRequest.SetApplicationSubmissionContext(context);
rmClient.SubmitApplication(submitRequest);
resourceManager.WaitForState(applicationId, RMAppState.Accepted);
return(applicationId);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void Submit()
{
lock (this)
{
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
context.SetApplicationId(this.applicationId);
context.SetQueue(this.queue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
context.SetAMContainerSpec(amContainer);
context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
));
SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
request.SetApplicationSubmissionContext(context);
ResourceScheduler scheduler = resourceManager.GetResourceScheduler();
resourceManager.GetClientRMService().SubmitApplication(request);
// Notify scheduler
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId
, this.queue, "user");
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent
(this.applicationAttemptId, false);
scheduler.Handle(addAttemptEvent);
}
}
private ContainerLaunchContext CreateCLC()
{
ContainerLaunchContext clc = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
return(clc);
}
public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
public virtual void TestContainerLaunch()
{
string appSubmitter = "nobody";
string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
);
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
NUnit.Framework.Assert.AreEqual(0, ret);
NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
, pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
.Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
}
public StartContainerEvent(Container container, ContainerLaunchContext containerLaunchContext
)
: base(container.GetId(), container.GetNodeId(), container.GetContainerToken(), NMClientAsyncImpl.ContainerEventType
.StartContainer)
{
this.container = container;
this.containerLaunchContext = containerLaunchContext;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
public override void SetAMContainerSpec(ContainerLaunchContext amContainer)
{
MaybeInitBuilder();
if (amContainer == null)
{
builder.ClearAmContainerSpec();
}
this.amContainer = amContainer;
}
public SharedCacheUploadEvent(IDictionary <LocalResourceRequest, Path> resources,
ContainerLaunchContext context, string user, SharedCacheUploadEventType eventType
)
: base(eventType)
{
this.resources = resources;
this.context = context;
this.user = user;
}
public virtual void TestContainerLaunchError()
{
// reinitialize executer
FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error"
);
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo");
conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null");
mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor());
Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any
<string>());
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
mockExec.SetConf(conf);
string appSubmitter = "nobody";
string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
);
string appId = "APP_ID";
string containerId = "CONTAINER_ID";
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
();
Dictionary <string, string> env = new Dictionary <string, string>();
Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any
<ContainerDiagnosticsUpdateEvent>());
Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
Path scriptPath = new Path("file:///bin/echo");
Path tokensPath = new Path("file:///dev/null");
Path workDir = new Path("/tmp");
Path pidFile = new Path(workDir, "pid.txt");
mockExec.ActivateContainer(cId, pidFile);
int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
, appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
NUnit.Framework.Assert.AreNotSame(0, ret);
NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
, appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
, pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
.Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
}
private static ContainerLaunchContext MockContainerLaunchContext(RecordFactory recordFactory
)
{
ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
amContainer.SetApplicationACLs(new Dictionary <ApplicationAccessType, string>());
return(amContainer);
}
/// <exception cref="System.Exception"/>
public virtual void TestAMAdminCommandOpts()
{
JobConf jobConf = new JobConf();
jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true");
jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m");
YARNRunner yarnRunner = new YARNRunner(jobConf);
ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf
);
ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec();
IList <string> commands = containerSpec.GetCommands();
int index = 0;
int adminIndex = 0;
int adminPos = -1;
int userIndex = 0;
int userPos = -1;
int tmpDirPos = -1;
foreach (string command in commands)
{
if (command != null)
{
NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command.
Contains(ProfileParams));
adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true");
if (adminPos >= 0)
{
adminIndex = index;
}
userPos = command.IndexOf("-Xmx1024m");
if (userPos >= 0)
{
userIndex = index;
}
tmpDirPos = command.IndexOf("-Djava.io.tmpdir=");
}
index++;
}
// Check java.io.tmpdir opts are set in the commands
NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0);
// Check both admin java opts and user java opts are in the commands
NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos
> 0);
NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos
> 0);
// Check the admin java opts is before user java opts in the commands
if (adminIndex == userIndex)
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminPos < userPos);
}
else
{
NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts."
, adminIndex < userIndex);
}
}
public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID, ContainerLaunchContext
containerLaunchContext, Container allocatedContainer, Task remoteTask)
: base(taskAttemptID, allocatedContainer.GetId(), StringInterner.WeakIntern(allocatedContainer
.GetNodeId().ToString()), allocatedContainer.GetContainerToken(), ContainerLauncher.EventType
.ContainerRemoteLaunch)
{
this.allocatedContainer = allocatedContainer;
this.containerLaunchContext = containerLaunchContext;
this.task = remoteTask;
}
public override void SetContainerLaunchContext(ContainerLaunchContext containerLaunchContext
)
{
MaybeInitBuilder();
if (containerLaunchContext == null)
{
builder.ClearContainerLaunchContext();
}
this.containerLaunchContext = containerLaunchContext;
}
public static StartContainerRequest NewInstance(ContainerLaunchContext context, Token
container)
{
StartContainerRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StartContainerRequest
>();
request.SetContainerLaunchContext(context);
request.SetContainerToken(container);
return(request);
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
// TODO: Don't create again and again.
/// <exception cref="System.IO.IOException"/>
private ContainerLaunchContext CreateAMContainerLaunchContext(ApplicationSubmissionContext
applicationMasterContext, ContainerId containerID)
{
// Construct the actual Container
ContainerLaunchContext container = applicationMasterContext.GetAMContainerSpec();
Log.Info("Command to launch container " + containerID + " : " + StringUtils.ArrayToString
(Sharpen.Collections.ToArray(container.GetCommands(), new string[0])));
// Finalize the container
SetupTokens(container, containerID);
return(container);
}
public virtual void TestShuffleProviders()
{
ApplicationId appId = ApplicationId.NewInstance(1, 1);
JobId jobId = MRBuilderUtils.NewJobId(appId, 1);
TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
Path jobFile = Org.Mockito.Mockito.Mock <Path>();
EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
, 0));
JobConf jobConf = new JobConf();
jobConf.SetClass("fs.file.impl", typeof(TestShuffleProvider.StubbedFS), typeof(FileSystem
));
jobConf.SetBoolean("fs.file.impl.disable.cache", true);
jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
jobConf.Set(YarnConfiguration.NmAuxServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
string serviceName = TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid;
string serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler1).FullName);
serviceName = TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid;
serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler2).FullName);
jobConf.Set(MRJobConfig.MapreduceJobShuffleProviderServices, TestShuffleProvider.TestShuffleHandler1
.MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
);
Credentials credentials = new Credentials();
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
(("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
jobToken, credentials, new SystemClock(), null);
jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(null
, jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
<WrappedJvmID>(), taListener, credentials);
IDictionary <string, ByteBuffer> serviceDataMap = launchCtx.GetServiceData();
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler1 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsNotNull("TestShuffleHandler2 is missing", serviceDataMap
[TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid]);
NUnit.Framework.Assert.IsTrue("mismatch number of services in map", serviceDataMap
.Count == 3);
}
public override void Handle(ContainerLauncherEvent @event)
{
if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteLaunch)
{
ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent)@event;
ContainerLaunchContext launchContext = launchEvent.GetContainerLaunchContext();
string cmdString = launchContext.GetCommands().ToString();
TestMapReduceChildJVM.Log.Info("launchContext " + cmdString);
this._enclosing.myCommandLine = cmdString;
this._enclosing.cmdEnvironment = launchContext.GetEnvironment();
}
base.Handle(@event);
}
/// <exception cref="System.IO.IOException"/>
public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration
conf, string user, ApplicationId appId, int uniqId)
{
this.user = user;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
this.id = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId, uniqId);
this.launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext>();
long currentTime = Runtime.CurrentTimeMillis();
this.containerTokenIdentifier = BuilderUtils.NewContainerTokenIdentifier(BuilderUtils
.NewContainerToken(id, "127.0.0.1", 1234, user, BuilderUtils.NewResource(1024, 1
), currentTime + 10000, 123, Sharpen.Runtime.GetBytesForString("password"), currentTime
));
this.state = ContainerState.New;
}
public override ContainerLaunchContext GetAMContainerSpec()
{
YarnProtos.ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
if (this.amContainer != null)
{
return(amContainer);
}
// Else via proto
if (!p.HasAmContainerSpec())
{
return(null);
}
amContainer = ConvertFromProtoFormat(p.GetAmContainerSpec());
return(amContainer);
}
public override ContainerLaunchContext GetContainerLaunchContext()
{
YarnServiceProtos.StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerLaunchContext != null)
{
return(this.containerLaunchContext);
}
if (!p.HasContainerLaunchContext())
{
return(null);
}
this.containerLaunchContext = ConvertFromProtoFormat(p.GetContainerLaunchContext(
));
return(this.containerLaunchContext);
}
