/// <exception cref="System.IO.IOException"/> private int RunAndBlock(ContainerId cId, IDictionary <string, string> launchCtxEnv , params string[] cmd) { string appId = "APP_" + Runtime.CurrentTimeMillis(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString ()).ThenReturn(appId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(launchCtxEnv); string script = WriteScriptFile(launchCtxEnv, cmd); Path scriptPath = new Path(script); Path tokensPath = new Path("/dev/null"); Path workDir = new Path(workSpace.GetAbsolutePath()); Path pidFile = new Path(workDir, "pid.txt"); exec.ActivateContainer(cId, pidFile); return(exec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter, appId , workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs())); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLaunchNullImage() { string appSubmitter = "nobody"; string appId = "APP_ID"; string containerId = "CONTAINER_ID"; string testImage = string.Empty; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(Org.Mockito.Mockito.ReturnsDeepStubs); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs ); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString ()).ThenReturn(appId); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage; dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName , testImage); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path pidFile = new Path(workDir, "pid.txt"); dockerContainerExecutor.ActivateContainer(cId, pidFile); dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); }
public virtual void TestAMStandardEnv() { string AdminLibPath = "foo"; string UserLibPath = "bar"; string UserShell = "shell"; JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminUserEnv, "LD_LIBRARY_PATH=" + AdminLibPath); jobConf.Set(MRJobConfig.MrAmEnv, "LD_LIBRARY_PATH=" + UserLibPath); jobConf.Set(MRJobConfig.MapredAdminUserShell, UserShell); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext appSubCtx = BuildSubmitContext(yarnRunner, jobConf); // make sure PWD is first in the lib path ContainerLaunchContext clc = appSubCtx.GetAMContainerSpec(); IDictionary <string, string> env = clc.GetEnvironment(); string libPath = env[ApplicationConstants.Environment.LdLibraryPath.ToString()]; NUnit.Framework.Assert.IsNotNull("LD_LIBRARY_PATH not set", libPath); string cps = jobConf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, MRConfig .DefaultMapreduceAppSubmissionCrossPlatform) ? ApplicationConstants.ClassPathSeparator : FilePath.pathSeparator; NUnit.Framework.Assert.AreEqual("Bad AM LD_LIBRARY_PATH setting", MRApps.CrossPlatformifyMREnv (conf, ApplicationConstants.Environment.Pwd) + cps + AdminLibPath + cps + UserLibPath , libPath); // make sure SHELL is set string shell = env[ApplicationConstants.Environment.Shell.ToString()]; NUnit.Framework.Assert.IsNotNull("SHELL not set", shell); NUnit.Framework.Assert.AreEqual("Bad SHELL setting", UserShell, shell); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void Launch() { Connect(); ContainerId masterContainerID = masterContainer.GetId(); ApplicationSubmissionContext applicationContext = application.GetSubmissionContext (); Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId ()); ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext , masterContainerID); StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext , masterContainer.GetContainerToken()); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); StartContainersResponse response = containerMgrProxy.StartContainers(allRequests); if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains (masterContainerID)) { Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize(); ParseAndThrowException(t); } else { Log.Info("Done launching container " + masterContainer + " for AM " + application .GetAppAttemptId()); } }
/// <exception cref="System.IO.IOException"/> private void SetupTokens(ContainerLaunchContext container, ContainerId containerID ) { IDictionary <string, string> environment = container.GetEnvironment(); environment[ApplicationConstants.ApplicationWebProxyBaseEnv] = application.GetWebProxyBase (); // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM. ApplicationId applicationId = application.GetAppAttemptId().GetApplicationId(); environment[ApplicationConstants.AppSubmitTimeEnv] = rmContext.GetRMApps()[applicationId ].GetSubmitTime().ToString(); environment[ApplicationConstants.MaxAppAttemptsEnv] = rmContext.GetRMApps()[applicationId ].GetMaxAppAttempts().ToString(); Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); if (container.GetTokens() != null) { // TODO: Don't do this kind of checks everywhere. dibb.Reset(container.GetTokens()); credentials.ReadTokenStorageStream(dibb); } // Add AMRMToken Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = CreateAndSetAMRMToken (); if (amrmToken != null) { credentials.AddToken(amrmToken.GetService(), amrmToken); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.WriteTokenStorageToStream(dob); container.SetTokens(ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength())); }
/// <exception cref="System.Exception"/> private void StartContainer(YarnRPC rpc, Token nmToken, Token containerToken, NodeId nodeId, string user) { ContainerLaunchContext context = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext >(); StartContainerRequest scRequest = StartContainerRequest.NewInstance(context, containerToken ); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); ContainerManagementProtocol proxy = null; try { proxy = GetContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); StartContainersResponse response = proxy.StartContainers(allRequests); foreach (SerializedException ex in response.GetFailedRequests().Values) { ParseAndThrowException(ex.DeSerialize()); } } finally { if (proxy != null) { rpc.StopProxy(proxy, conf); } } }
public _Thread_434(TestNMClientAsync _enclosing, Container container, ContainerLaunchContext clc) { this._enclosing = _enclosing; this.container = container; this.clc = clc; }
/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(string submitter, string queueName, bool setupACLs) { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.NewInstance(); ApplicationClientProtocol submitterClient = GetRMClientForUser(submitter); ApplicationId applicationId = submitterClient.GetNewApplication(newAppRequest).GetApplicationId (); Resource resource = BuilderUtils.NewResource(1024, 1); IDictionary <ApplicationAccessType, string> acls = CreateACLs(submitter, setupACLs ); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.NewInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext. NewInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.SetApplicationId(applicationId); appSubmissionContext.SetQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.NewInstance(appSubmissionContext ); submitterClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
/// <exception cref="System.Exception"/> private ApplicationId SubmitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) { SubmitApplicationRequest submitRequest = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); ApplicationId applicationId = rmClient.GetNewApplication(recordFactory.NewRecordInstance <GetNewApplicationRequest>()).GetApplicationId(); context.SetApplicationId(applicationId); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); acls[ApplicationAccessType.ViewApp] = viewACL.GetAclString(); acls[ApplicationAccessType.ModifyApp] = modifyACL.GetAclString(); ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext >(); Resource resource = BuilderUtils.NewResource(1024, 1); context.SetResource(resource); amContainer.SetApplicationACLs(acls); context.SetAMContainerSpec(amContainer); submitRequest.SetApplicationSubmissionContext(context); rmClient.SubmitApplication(submitRequest); resourceManager.WaitForState(applicationId, RMAppState.Accepted); return(applicationId); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void Submit() { lock (this) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(this.applicationId); context.SetQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); context.SetAMContainerSpec(amContainer); context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb )); SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); request.SetApplicationSubmissionContext(context); ResourceScheduler scheduler = resourceManager.GetResourceScheduler(); resourceManager.GetClientRMService().SubmitApplication(request); // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId , this.queue, "user"); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent (this.applicationAttemptId, false); scheduler.Handle(addAttemptEvent); } }
private ContainerLaunchContext CreateCLC() { ContainerLaunchContext clc = recordFactory.NewRecordInstance <ContainerLaunchContext >(); return(clc); }
public virtual void TestNMWebApp() { Context nmContext = new NodeManager.NMContext(null, null, null, null, null); ResourceView resourceView = new _ResourceView_147(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager (conf), dirsHandler); server.Init(conf); server.Start(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); Dispatcher dispatcher = new AsyncDispatcher(); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); nmContext.GetApplications()[appId] = app; ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 1); NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>(); NMStateStoreService stateStore = new NMNullStateStoreService(); foreach (ContainerId containerId in new ContainerId[] { container1, container2 }) { // TODO: Use builder utils ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); long currentTime = Runtime.CurrentTimeMillis(); Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234 , user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString ("password"), currentTime); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics , BuilderUtils.NewContainerTokenIdentifier(containerToken)); nmContext.GetContainers()[containerId] = container; //TODO: Gross hack. Fix in code. ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId (); nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container; WriteContainerLogs(nmContext, containerId, dirsHandler); } }
public virtual void TestContainerLaunch() { string appSubmitter = "nobody"; string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString( ); string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); NUnit.Framework.Assert.AreEqual(0, ret); NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null" , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams()); }
public StartContainerEvent(Container container, ContainerLaunchContext containerLaunchContext ) : base(container.GetId(), container.GetNodeId(), container.GetContainerToken(), NMClientAsyncImpl.ContainerEventType .StartContainer) { this.container = container; this.containerLaunchContext = containerLaunchContext; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
public override void SetAMContainerSpec(ContainerLaunchContext amContainer) { MaybeInitBuilder(); if (amContainer == null) { builder.ClearAmContainerSpec(); } this.amContainer = amContainer; }
public SharedCacheUploadEvent(IDictionary <LocalResourceRequest, Path> resources, ContainerLaunchContext context, string user, SharedCacheUploadEventType eventType ) : base(eventType) { this.resources = resources; this.context = context; this.user = user; }
public virtual void TestContainerLaunchError() { // reinitialize executer FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error" ); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo"); conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null"); mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor()); Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any <string>()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); mockExec.SetConf(conf); string appSubmitter = "nobody"; string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString( ); string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any <ContainerDiagnosticsUpdateEvent>()); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); NUnit.Framework.Assert.AreNotSame(0, ret); NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null" , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams()); }
private static ContainerLaunchContext MockContainerLaunchContext(RecordFactory recordFactory ) { ContainerLaunchContext amContainer = recordFactory.NewRecordInstance <ContainerLaunchContext >(); amContainer.SetApplicationACLs(new Dictionary <ApplicationAccessType, string>()); return(amContainer); }
/// <exception cref="System.Exception"/> public virtual void TestAMAdminCommandOpts() { JobConf jobConf = new JobConf(); jobConf.Set(MRJobConfig.MrAmAdminCommandOpts, "-Djava.net.preferIPv4Stack=true"); jobConf.Set(MRJobConfig.MrAmCommandOpts, "-Xmx1024m"); YARNRunner yarnRunner = new YARNRunner(jobConf); ApplicationSubmissionContext submissionContext = BuildSubmitContext(yarnRunner, jobConf ); ContainerLaunchContext containerSpec = submissionContext.GetAMContainerSpec(); IList <string> commands = containerSpec.GetCommands(); int index = 0; int adminIndex = 0; int adminPos = -1; int userIndex = 0; int userPos = -1; int tmpDirPos = -1; foreach (string command in commands) { if (command != null) { NUnit.Framework.Assert.IsFalse("Profiler should be disabled by default", command. Contains(ProfileParams)); adminPos = command.IndexOf("-Djava.net.preferIPv4Stack=true"); if (adminPos >= 0) { adminIndex = index; } userPos = command.IndexOf("-Xmx1024m"); if (userPos >= 0) { userIndex = index; } tmpDirPos = command.IndexOf("-Djava.io.tmpdir="); } index++; } // Check java.io.tmpdir opts are set in the commands NUnit.Framework.Assert.IsTrue("java.io.tmpdir is not set for AM", tmpDirPos > 0); // Check both admin java opts and user java opts are in the commands NUnit.Framework.Assert.IsTrue("AM admin command opts not in the commands.", adminPos > 0); NUnit.Framework.Assert.IsTrue("AM user command opts not in the commands.", userPos > 0); // Check the admin java opts is before user java opts in the commands if (adminIndex == userIndex) { NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts." , adminPos < userPos); } else { NUnit.Framework.Assert.IsTrue("AM admin command opts is after user command opts." , adminIndex < userIndex); } }
public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID, ContainerLaunchContext containerLaunchContext, Container allocatedContainer, Task remoteTask) : base(taskAttemptID, allocatedContainer.GetId(), StringInterner.WeakIntern(allocatedContainer .GetNodeId().ToString()), allocatedContainer.GetContainerToken(), ContainerLauncher.EventType .ContainerRemoteLaunch) { this.allocatedContainer = allocatedContainer; this.containerLaunchContext = containerLaunchContext; this.task = remoteTask; }
public override void SetContainerLaunchContext(ContainerLaunchContext containerLaunchContext ) { MaybeInitBuilder(); if (containerLaunchContext == null) { builder.ClearContainerLaunchContext(); } this.containerLaunchContext = containerLaunchContext; }
public static StartContainerRequest NewInstance(ContainerLaunchContext context, Token container) { StartContainerRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <StartContainerRequest >(); request.SetContainerLaunchContext(context); request.SetContainerToken(container); return(request); }
/// <exception cref="System.Exception"/> private void TestRPCTimeout(string rpcClass) { Configuration conf = new Configuration(); // set timeout low for the test conf.SetInt("yarn.rpc.nm-command-timeout", 3000); conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); try { ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof( ContainerManagementProtocol), server.GetListenerAddress(), conf); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100); NodeId nodeId = NodeId.NewInstance("localhost", 1234); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000, 42, 42, Priority.NewInstance(0), 0); Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); try { proxy.StartContainers(allRequests); } catch (Exception e) { Log.Info(StringUtils.StringifyException(e)); NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException ).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName); return; } } finally { server.Stop(); } NUnit.Framework.Assert.Fail("timeout exception should have occurred!"); }
// TODO: Don't create again and again. /// <exception cref="System.IO.IOException"/> private ContainerLaunchContext CreateAMContainerLaunchContext(ApplicationSubmissionContext applicationMasterContext, ContainerId containerID) { // Construct the actual Container ContainerLaunchContext container = applicationMasterContext.GetAMContainerSpec(); Log.Info("Command to launch container " + containerID + " : " + StringUtils.ArrayToString (Sharpen.Collections.ToArray(container.GetCommands(), new string[0]))); // Finalize the container SetupTokens(container, containerID); return(container); }
public virtual void TestShuffleProviders() { ApplicationId appId = ApplicationId.NewInstance(1, 1); JobId jobId = MRBuilderUtils.NewJobId(appId, 1); TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map); Path jobFile = Org.Mockito.Mockito.Mock <Path>(); EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>(); Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost" , 0)); JobConf jobConf = new JobConf(); jobConf.SetClass("fs.file.impl", typeof(TestShuffleProvider.StubbedFS), typeof(FileSystem )); jobConf.SetBoolean("fs.file.impl.disable.cache", true); jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty); jobConf.Set(YarnConfiguration.NmAuxServices, TestShuffleProvider.TestShuffleHandler1 .MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid ); string serviceName = TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid; string serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName); jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler1).FullName); serviceName = TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid; serviceStr = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName); jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler2).FullName); jobConf.Set(MRJobConfig.MapreduceJobShuffleProviderServices, TestShuffleProvider.TestShuffleHandler1 .MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid ); Credentials credentials = new Credentials(); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString (("tokenpw")), new Text("tokenkind"), new Text("tokenservice")); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener, jobToken, credentials, new SystemClock(), null); jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString()); ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(null , jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock <WrappedJvmID>(), taListener, credentials); IDictionary <string, ByteBuffer> serviceDataMap = launchCtx.GetServiceData(); NUnit.Framework.Assert.IsNotNull("TestShuffleHandler1 is missing", serviceDataMap [TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid]); NUnit.Framework.Assert.IsNotNull("TestShuffleHandler2 is missing", serviceDataMap [TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid]); NUnit.Framework.Assert.IsTrue("mismatch number of services in map", serviceDataMap .Count == 3); }
public override void Handle(ContainerLauncherEvent @event) { if (@event.GetType() == ContainerLauncher.EventType.ContainerRemoteLaunch) { ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent)@event; ContainerLaunchContext launchContext = launchEvent.GetContainerLaunchContext(); string cmdString = launchContext.GetCommands().ToString(); TestMapReduceChildJVM.Log.Info("launchContext " + cmdString); this._enclosing.myCommandLine = cmdString; this._enclosing.cmdEnvironment = launchContext.GetEnvironment(); } base.Handle(@event); }
/// <exception cref="System.IO.IOException"/> public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration conf, string user, ApplicationId appId, int uniqId) { this.user = user; this.recordFactory = RecordFactoryProvider.GetRecordFactory(conf); this.id = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId, uniqId); this.launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext>(); long currentTime = Runtime.CurrentTimeMillis(); this.containerTokenIdentifier = BuilderUtils.NewContainerTokenIdentifier(BuilderUtils .NewContainerToken(id, "127.0.0.1", 1234, user, BuilderUtils.NewResource(1024, 1 ), currentTime + 10000, 123, Sharpen.Runtime.GetBytesForString("password"), currentTime )); this.state = ContainerState.New; }
public override ContainerLaunchContext GetAMContainerSpec() { YarnProtos.ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; if (this.amContainer != null) { return(amContainer); } // Else via proto if (!p.HasAmContainerSpec()) { return(null); } amContainer = ConvertFromProtoFormat(p.GetAmContainerSpec()); return(amContainer); }
public override ContainerLaunchContext GetContainerLaunchContext() { YarnServiceProtos.StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.containerLaunchContext != null) { return(this.containerLaunchContext); } if (!p.HasContainerLaunchContext()) { return(null); } this.containerLaunchContext = ConvertFromProtoFormat(p.GetContainerLaunchContext( )); return(this.containerLaunchContext); }