/// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private void TestContainerTokenWithEpoch(Configuration conf) { Log.Info("Running test for serializing/deserializing containerIds"); NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager ().GetRMContext().GetNMTokenSecretManager(); ApplicationId appId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 0); ContainerId cId = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L); NodeManager nm = yarnCluster.GetNodeManager(0); NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager (); string user = "******"; WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm); NodeId nodeId = nm.GetNMContext().GetNodeId(); // Both id should be equal. NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId (), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId()); // Creating a normal Container Token RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager ().GetRMContext().GetContainerTokenSecretManager(); Resource r = Resource.NewInstance(1230, 2); Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager .CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( ); byte[] tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array()); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length); containerTokenIdentifier.ReadFields(dib); NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID()); NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID ().ToString()); Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken (appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.Create(conf); TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false); IList <ContainerId> containerIds = new List <ContainerId>(); containerIds.AddItem(cId); ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken , nodeId, user); GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest .NewInstance(containerIds)); NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]); NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId ()); NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId ().ToString()); }
private void ClearLimits(ContainerId containerId) { if (IsCpuWeightEnabled()) { DeleteCgroup(PathForCgroup(ControllerCpu, containerId.ToString())); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public StartContainersResponse StartContainers(StartContainersRequest requests) { StartContainerRequest request = requests.GetStartContainerRequests()[0]; Log.Info("Container started by MyContainerManager: " + request); launched = true; IDictionary <string, string> env = request.GetContainerLaunchContext().GetEnvironment (); Token containerToken = request.GetContainerToken(); ContainerTokenIdentifier tokenId = null; try { tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken); } catch (IOException e) { throw RPCUtil.GetRemoteException(e); } ContainerId containerId = tokenId.GetContainerID(); containerIdAtContainerManager = containerId.ToString(); attemptIdAtContainerManager = containerId.GetApplicationAttemptId().ToString(); nmHostAtContainerManager = tokenId.GetNmHostAddress(); submitTimeAtContainerManager = long.Parse(env[ApplicationConstants.AppSubmitTimeEnv ]); maxAppAttempts = System.Convert.ToInt32(env[ApplicationConstants.MaxAppAttemptsEnv ]); return(StartContainersResponse.NewInstance(new Dictionary <string, ByteBuffer>(), new AList <ContainerId>(), new Dictionary <ContainerId, SerializedException>())); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLaunchNullImage() { string appSubmitter = "nobody"; string appId = "APP_ID"; string containerId = "CONTAINER_ID"; string testImage = string.Empty; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(Org.Mockito.Mockito.ReturnsDeepStubs); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs ); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString ()).ThenReturn(appId); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage; dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName , testImage); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path pidFile = new Path(workDir, "pid.txt"); dockerContainerExecutor.ActivateContainer(cId, pidFile); dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); }
public AMAttemptInfo(AMInfo amInfo, string jobId, string user) { this.nodeHttpAddress = string.Empty; this.nodeId = string.Empty; string nmHost = amInfo.GetNodeManagerHost(); int nmHttpPort = amInfo.GetNodeManagerHttpPort(); int nmPort = amInfo.GetNodeManagerPort(); if (nmHost != null) { this.nodeHttpAddress = nmHost + ":" + nmHttpPort; NodeId nodeId = NodeId.NewInstance(nmHost, nmPort); this.nodeId = nodeId.ToString(); } this.id = amInfo.GetAppAttemptId().GetAttemptId(); this.startTime = amInfo.GetStartTime(); this.containerId = string.Empty; this.logsLink = string.Empty; ContainerId containerId = amInfo.GetContainerId(); if (containerId != null) { this.containerId = containerId.ToString(); this.logsLink = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress , StringHelper.Ujoin("node", "containerlogs", this.containerId, user)); } }
public virtual void TestContainerLaunch() { string appSubmitter = "nobody"; string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString( ); string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); NUnit.Framework.Assert.AreEqual(0, ret); NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null" , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams()); }
public virtual void TestContainerIdWithEpoch() { ContainerId id = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(0, 0, 0, 25645811); string cid = ConverterUtils.ToString(id); NUnit.Framework.Assert.AreEqual("container_0_0000_00_25645811", cid); ContainerId gen = ConverterUtils.ToContainerId(cid); NUnit.Framework.Assert.AreEqual(gen.ToString(), id.ToString()); long ts = Runtime.CurrentTimeMillis(); ContainerId id2 = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(36473 , 4365472, ts, 4298334883325L); string cid2 = ConverterUtils.ToString(id2); NUnit.Framework.Assert.AreEqual("container_e03_" + ts + "_36473_4365472_999799999997" , cid2); ContainerId gen2 = ConverterUtils.ToContainerId(cid2); NUnit.Framework.Assert.AreEqual(gen2.ToString(), id2.ToString()); ContainerId id3 = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(36473 , 4365472, ts, 844424930131965L); string cid3 = ConverterUtils.ToString(id3); NUnit.Framework.Assert.AreEqual("container_e767_" + ts + "_36473_4365472_1099511627773" , cid3); ContainerId gen3 = ConverterUtils.ToContainerId(cid3); NUnit.Framework.Assert.AreEqual(gen3.ToString(), id3.ToString()); }
/// <exception cref="System.IO.IOException"/> public override void RemoveContainer(ContainerId containerId) { string keyPrefix = ContainersKeyPrefix + containerId.ToString(); try { WriteBatch batch = db.CreateWriteBatch(); try { batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerRequestKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerDiagsKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerLaunchedKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerKilledKeySuffix)); batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerExitCodeKeySuffix)); db.Write(batch); } finally { batch.Close(); } } catch (DBException e) { throw new IOException(e); } }
public AMAttemptInfo(AMInfo amInfo, string jobId, string user, string host, string pathPrefix) { this.nodeHttpAddress = string.Empty; this.nodeId = string.Empty; string nmHost = amInfo.GetNodeManagerHost(); int nmHttpPort = amInfo.GetNodeManagerHttpPort(); int nmPort = amInfo.GetNodeManagerPort(); if (nmHost != null) { this.nodeHttpAddress = nmHost + ":" + nmHttpPort; NodeId nodeId = NodeId.NewInstance(nmHost, nmPort); this.nodeId = nodeId.ToString(); } this.id = amInfo.GetAppAttemptId().GetAttemptId(); this.startTime = amInfo.GetStartTime(); this.containerId = string.Empty; this.logsLink = string.Empty; this.shortLogsLink = string.Empty; ContainerId containerId = amInfo.GetContainerId(); if (containerId != null) { this.containerId = containerId.ToString(); this.logsLink = StringHelper.Join(host, pathPrefix, StringHelper.Ujoin("logs", this .nodeId, this.containerId, jobId, user)); this.shortLogsLink = StringHelper.Ujoin("logs", this.nodeId, this.containerId, jobId , user); } }
public virtual void TestForCorruptedAggregatedLogs() { Configuration conf = new Configuration(); FilePath workDir = new FilePath(testWorkDir, "testReadAcontainerLogs1"); Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile"); Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles"); ContainerId testContainerId = TestContainerId.NewContainerId(1, 1, 1, 1); Path t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId ().ToString()); Path srcFilePath = new Path(t, testContainerId.ToString()); long numChars = 950000; WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot , testContainerId); AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf, remoteAppLogFile); AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey(); DataInputStream dis = logReader.Next(rLogKey); TextWriter writer = new StringWriter(); try { AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer); } catch (Exception e) { if (e.ToString().Contains("NumberFormatException")) { NUnit.Framework.Assert.Fail("Aggregated logs are corrupted."); } } }
/// <summary>A helper api for creating an audit log for a failure event.</summary> internal static string CreateFailureLog(string user, string operation, string perm , string target, string description, ApplicationId appId, ApplicationAttemptId attemptId , ContainerId containerId) { StringBuilder b = new StringBuilder(); Start(RMAuditLogger.Keys.User, user, b); AddRemoteIP(b); Add(RMAuditLogger.Keys.Operation, operation, b); Add(RMAuditLogger.Keys.Target, target, b); Add(RMAuditLogger.Keys.Result, RMAuditLogger.AuditConstants.Failure, b); Add(RMAuditLogger.Keys.Description, description, b); Add(RMAuditLogger.Keys.Permissions, perm, b); if (appId != null) { Add(RMAuditLogger.Keys.Appid, appId.ToString(), b); } if (attemptId != null) { Add(RMAuditLogger.Keys.Appattemptid, attemptId.ToString(), b); } if (containerId != null) { Add(RMAuditLogger.Keys.Containerid, containerId.ToString(), b); } return(b.ToString()); }
private ContainerId GetNextContainerId() { ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); string id = "CONTAINER_" + GetNextId(); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(id); return(cId); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
/// <exception cref="System.Exception"/> public virtual void TestInvalidContainerReleaseRequest() { MockRM rm = new MockRM(conf); try { rm.Start(); // Register node1 MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); // Submit an application RMApp app1 = rm.SubmitApp(1024); // kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0); RMApp app2 = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); am2.RegisterAppAttempt(); // Now trying to release container allocated for app1 -> appAttempt1. ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId(); am2.AddContainerToBeReleased(cId); try { am2.Schedule(); NUnit.Framework.Assert.Fail("Exception was expected!!"); } catch (InvalidContainerReleaseException e) { StringBuilder sb = new StringBuilder("Cannot release container : "); sb.Append(cId.ToString()); sb.Append(" not belonging to this application attempt : "); sb.Append(attempt2.GetAppAttemptId().ToString()); NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString())); } } finally { if (rm != null) { rm.Stop(); } } }
/// <summary>Print all the information about this AM.</summary> public virtual void PrintAll() { System.Console.Out.WriteLine("APPLICATION_ATTEMPT_ID:" + appAttemptId.ToString()); System.Console.Out.WriteLine("START_TIME: " + startTime); System.Console.Out.WriteLine("CONTAINER_ID: " + containerId.ToString()); System.Console.Out.WriteLine("NODE_MANAGER_HOST: " + nodeManagerHost); System.Console.Out.WriteLine("NODE_MANAGER_PORT: " + nodeManagerPort); System.Console.Out.WriteLine("NODE_MANAGER_HTTP_PORT: " + nodeManagerHttpPort); }
private ContainerId GetNextContainerId() { ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs ); string id = "CONTAINER_" + Runtime.CurrentTimeMillis(); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(id); return(cId); }
/// <exception cref="System.IO.IOException"/> private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt ) { // read application history data NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count); for (int i = 1; i <= num; ++i) { ApplicationId appId = ApplicationId.NewInstance(0, i); ApplicationHistoryData appData = store.GetApplication(appId); NUnit.Framework.Assert.IsNotNull(appData); NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName()); NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo()); // read application attempt history data NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count); for (int j = 1; j <= num; ++j) { ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j); ApplicationAttemptHistoryData attemptData = store.GetApplicationAttempt(appAttemptId ); NUnit.Framework.Assert.IsNotNull(attemptData); NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost()); if (missingApplicationAttempt && j == num) { NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo()); continue; } else { NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo ()); } // read container history data NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count); for (int k = 1; k <= num; ++k) { ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k); ContainerHistoryData containerData = store.GetContainer(containerId); NUnit.Framework.Assert.IsNotNull(containerData); NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData .GetPriority()); if (missingContainer && k == num) { NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo()); } else { NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo ()); } } ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId); NUnit.Framework.Assert.IsNotNull(masterContainer); NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer .GetContainerId()); } } }
public virtual void TestContainerLaunchError() { // reinitialize executer FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error" ); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo"); conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null"); mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor()); Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any <string>()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); mockExec.SetConf(conf); string appSubmitter = "nobody"; string cmd = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString( ); string appId = "APP_ID"; string containerId = "CONTAINER_ID"; Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(); ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext> (); Dictionary <string, string> env = new Dictionary <string, string>(); Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId); Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context); Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any <ContainerDiagnosticsUpdateEvent>()); Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId); Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path workDir = new Path("/tmp"); Path pidFile = new Path(workDir, "pid.txt"); mockExec.ActivateContainer(cId, pidFile); int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs()); NUnit.Framework.Assert.AreNotSame(0, ret); NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null" , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams()); }
/// <exception cref="System.IO.IOException"/> public virtual ContainerHistoryData GetContainer(ContainerId containerId) { FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader (containerId.GetApplicationAttemptId().GetApplicationId()); try { bool readStartData = false; bool readFinishData = false; ContainerHistoryData historyData = ContainerHistoryData.NewInstance(containerId, null, null, null, long.MinValue, long.MaxValue, null, int.MaxValue, null); while ((!readStartData || !readFinishData) && hfReader.HasNext()) { FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next(); if (entry.key.id.Equals(containerId.ToString())) { if (entry.key.suffix.Equals(StartDataSuffix)) { ContainerStartData startData = ParseContainerStartData(entry.value); MergeContainerHistoryData(historyData, startData); readStartData = true; } else { if (entry.key.suffix.Equals(FinishDataSuffix)) { ContainerFinishData finishData = ParseContainerFinishData(entry.value); MergeContainerHistoryData(historyData, finishData); readFinishData = true; } } } } if (!readStartData && !readFinishData) { return(null); } if (!readStartData) { Log.Warn("Start information is missing for container " + containerId); } if (!readFinishData) { Log.Warn("Finish information is missing for container " + containerId); } Log.Info("Completed reading history information of container " + containerId); return(historyData); } catch (IOException e) { Log.Error("Error when reading history file of container " + containerId, e); throw; } finally { hfReader.Close(); } }
private static TimelineEntity CreateContainerEntity(ContainerId containerId) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(ContainerMetricsConstants.EntityType); entity.SetEntityId(containerId.ToString()); entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId .GetApplicationAttemptId().ToString()); return(entity); }
/// <summary>Create an event to record the start of an MR AppMaster</summary> /// <param name="appAttemptId">the application attempt id.</param> /// <param name="startTime">the start time of the AM.</param> /// <param name="containerId">the containerId of the AM.</param> /// <param name="nodeManagerHost">the node on which the AM is running.</param> /// <param name="nodeManagerPort">the port on which the AM is running.</param> /// <param name="nodeManagerHttpPort">the httpPort for the node running the AM.</param> /// <param name="forcedJobStateOnShutDown">the state to force the job into</param> public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime, ContainerId containerId, string nodeManagerHost, int nodeManagerPort, int nodeManagerHttpPort , string forcedJobStateOnShutDown, long submitTime) { datum.applicationAttemptId = new Utf8(appAttemptId.ToString()); datum.startTime = startTime; datum.containerId = new Utf8(containerId.ToString()); datum.nodeManagerHost = new Utf8(nodeManagerHost); datum.nodeManagerPort = nodeManagerPort; datum.nodeManagerHttpPort = nodeManagerHttpPort; this.forcedJobStateOnShutDown = forcedJobStateOnShutDown; this.submitTime = submitTime; }
/// <exception cref="System.IO.IOException"/> public override void StoreContainerKilled(ContainerId containerId) { string key = ContainersKeyPrefix + containerId.ToString() + ContainerKilledKeySuffix; try { db.Put(JniDBFactory.Bytes(key), EmptyValue); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreContainerDiagnostics(ContainerId containerId, StringBuilder diagnostics) { string key = ContainersKeyPrefix + containerId.ToString() + ContainerDiagsKeySuffix; try { db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(diagnostics.ToString())); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogFile() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir (appId.ToString(), containerId.ToString())); containerLogDir.Mkdirs(); string fileName = "fileName"; FilePath containerLogFile = new FilePath(containerLogDir, fileName); containerLogFile.CreateNewFile(); FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user , nmContext); NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI() .ToString()); FileUtil.FullyDelete(absLogDir); }
/// <exception cref="System.IO.IOException"/> public override void StoreContainerCompleted(ContainerId containerId, int exitCode ) { string key = ContainersKeyPrefix + containerId.ToString() + ContainerExitCodeKeySuffix; try { db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(Sharpen.Extensions.ToString(exitCode ))); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> public override void StoreContainer(ContainerId containerId, StartContainerRequest startRequest) { string key = ContainersKeyPrefix + containerId.ToString() + ContainerRequestKeySuffix; try { db.Put(JniDBFactory.Bytes(key), ((StartContainerRequestPBImpl)startRequest).GetProto ().ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.Exception"/> private static void CreateContainerLogInLocalDir(Path appLogsDir, ContainerId containerId , FileSystem fs) { Path containerLogsDir = new Path(appLogsDir, containerId.ToString()); if (fs.Exists(containerLogsDir)) { fs.Delete(containerLogsDir, true); } NUnit.Framework.Assert.IsTrue(fs.Mkdirs(containerLogsDir)); TextWriter writer = new FileWriter(new FilePath(containerLogsDir.ToString(), "sysout" )); writer.Write("Hello " + containerId + "!"); writer.Close(); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> internal static IList <FilePath> GetContainerLogDirs(ContainerId containerId, LocalDirsHandlerService dirsHandler) { IList <string> logDirs = dirsHandler.GetLogDirsForRead(); IList <FilePath> containerLogDirs = new AList <FilePath>(logDirs.Count); foreach (string logDir in logDirs) { logDir = new FilePath(logDir).ToURI().GetPath(); string appIdStr = ConverterUtils.ToString(containerId.GetApplicationAttemptId().GetApplicationId ()); FilePath appLogDir = new FilePath(logDir, appIdStr); containerLogDirs.AddItem(new FilePath(appLogDir, containerId.ToString())); } return(containerLogDirs); }
public virtual string GetResourcesOption(ContainerId containerId) { string containerName = containerId.ToString(); StringBuilder sb = new StringBuilder("cgroups="); if (IsCpuWeightEnabled()) { sb.Append(PathForCgroup(ControllerCpu, containerName) + "/tasks"); sb.Append(","); } if (sb[sb.Length - 1] == ',') { Sharpen.Runtime.DeleteCharAt(sb, sb.Length - 1); } return(sb.ToString()); }
public virtual void TestLogFileWithDriveLetter() { ContainerImpl container = Org.Mockito.Mockito.Mock <ContainerImpl>(); ApplicationIdPBImpl appId = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>(); Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId"); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl >(); Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); applications[appId] = app; ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>(); Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId"); Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId ); ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); containers[containerId] = container; LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId" + Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName" )); NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>(); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs); Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName" , null, context); NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString ().IndexOf("F:" + FilePath.separator + "nmlogs") > -1); }