public virtual void TestNodeSingleContainerXML() { WebResource r = Resource(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = new MockApp(1); nmContext.GetApplications()[app.GetAppId()] = app; Dictionary <string, string> hash = AddAppContainers(app); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app2 = new MockApp(2); nmContext.GetApplications()[app2.GetAppId()] = app2; AddAppContainers(app2); foreach (string id in hash.Keys) { ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containers") .Path(id).Accept(MediaType.ApplicationXml).Get <ClientResponse>(); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationXmlType, response.GetType()); string xml = response.GetEntity <string>(); DocumentBuilderFactory dbf = DocumentBuilderFactory.NewInstance(); DocumentBuilder db = dbf.NewDocumentBuilder(); InputSource @is = new InputSource(); @is.SetCharacterStream(new StringReader(xml)); Document dom = db.Parse(@is); NodeList nodes = dom.GetElementsByTagName("container"); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.GetLength ()); VerifyContainersInfoXML(nodes, nmContext.GetContainers()[ConverterUtils.ToContainerId (id)]); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestMRAppMasterMaxAppAttempts() { // No matter what's the maxAppAttempt or attempt id, the isLastRetry always // equals to false bool[] expectedBools = new bool[] { false, false, false }; string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); FilePath stagingDir = new FilePath(MRApps.GetStagingAreaDir(conf, userName).ToString ()); stagingDir.Mkdirs(); for (int i = 0; i < expectedBools.Length; ++i) { MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis(), false, true); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); NUnit.Framework.Assert.AreEqual("isLastAMRetry is correctly computed.", expectedBools [i], appMaster.IsLastAMRetry()); } }
/// <exception cref="System.IO.IOException"/> public static FileInputStream OpenLogFileForRead(string containerIdStr, FilePath logFile, Context context) { ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId (); string user = context.GetApplications()[applicationId].GetUser(); try { return(SecureIOUtils.OpenForRead(logFile, user, null)); } catch (IOException e) { if (e.Message.Contains("did not match expected owner '" + user + "'")) { Log.Error("Exception reading log file " + logFile.GetAbsolutePath(), e); throw new IOException("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : " + logFile.GetName(), e); } else { throw new IOException("Exception reading log file. It might be because log " + "file was aggregated : " + logFile.GetName(), e); } } }
/// <exception cref="System.Exception"/> protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier > appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher dispatcher) { RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics >(); Container container = new ContainerPBImpl(); container.SetId(ConverterUtils.ToContainerId(containerIdStr)); RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>(); Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId); Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container); Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken); Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey ); Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics ); Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()). ThenReturn(new AggregateAppResourceUsage(0, 0)); dispatcher.attemptId = attemptId; store.StoreNewApplicationAttempt(mockAttempt); WaitNotify(dispatcher); return(container.GetId()); }
protected override void Render(HtmlBlock.Block html) { ContainerId containerID; try { containerID = ConverterUtils.ToContainerId($(ContainerId)); } catch (ArgumentException) { html.P().("Invalid containerId " + $(ContainerId)).(); return; } Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> div = html.Div("#content" ); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = this.nmContext.GetContainers()[containerID]; if (container == null) { div.H1("Unknown Container. Container might have completed, " + "please go back to the previous page and retry." ).(); return; } ContainerInfo info = new ContainerInfo(this.nmContext, container); Info("Container information").("ContainerID", info.GetId()).("ContainerState", info .GetState()).("ExitStatus", info.GetExitStatus()).("Diagnostics", info.GetDiagnostics ()).("User", info.GetUser()).("TotalMemoryNeeded", info.GetMemoryNeeded()).("TotalVCoresNeeded" , info.GetVCoresNeeded()).("logs", info.GetShortLogLink(), "Link to logs"); html.(typeof(InfoBlock)); }
/// <exception cref="System.IO.IOException"/> public virtual IDictionary <ContainerId, ContainerHistoryData> GetContainers(ApplicationAttemptId appAttemptId) { IDictionary <ContainerId, ContainerHistoryData> historyDataMap = new Dictionary <ContainerId , ContainerHistoryData>(); FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader (appAttemptId.GetApplicationId()); try { while (hfReader.HasNext()) { FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next(); if (entry.key.id.StartsWith(ConverterUtils.ContainerPrefix)) { ContainerId containerId = ConverterUtils.ToContainerId(entry.key.id); if (containerId.GetApplicationAttemptId().Equals(appAttemptId)) { ContainerHistoryData historyData = historyDataMap[containerId]; if (historyData == null) { historyData = ContainerHistoryData.NewInstance(containerId, null, null, null, long.MinValue , long.MaxValue, null, int.MaxValue, null); historyDataMap[containerId] = historyData; } if (entry.key.suffix.Equals(StartDataSuffix)) { MergeContainerHistoryData(historyData, ParseContainerStartData(entry.value)); } else { if (entry.key.suffix.Equals(FinishDataSuffix)) { MergeContainerHistoryData(historyData, ParseContainerFinishData(entry.value)); } } } } } Log.Info("Completed reading history information of all conatiners" + " of application attempt " + appAttemptId); } catch (IOException) { Log.Info("Error when reading history information of some containers" + " of application attempt " + appAttemptId); } finally { hfReader.Close(); } return(historyDataMap); }
protected internal static ContainerId ParseContainerId(string containerId) { if (containerId == null || containerId.IsEmpty()) { throw new NotFoundException("containerId, " + containerId + ", is empty or null"); } ContainerId cid = ConverterUtils.ToContainerId(containerId); if (cid == null) { throw new NotFoundException("containerId is null"); } return(cid); }
/// <exception cref="System.IO.IOException"/> private static void LoadContainerToken(NMStateStoreService.RecoveredContainerTokensState state, string key, string containerIdStr, byte[] value) { ContainerId containerId; long expTime; try { containerId = ConverterUtils.ToContainerId(containerIdStr); expTime = long.Parse(JniDBFactory.AsString(value)); } catch (ArgumentException e) { throw new IOException("Bad container token state for " + key, e); } state.activeTokens[containerId] = expTime; }
public virtual void TestMRAppMasterForDifferentUser() { string applicationAttemptIdStr = "appattempt_1317529182569_0004_000001"; string containerIdStr = "container_1317529182569_0004_000001_1"; string userName = "******"; ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId , "host", -1, -1, Runtime.CurrentTimeMillis()); JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); Path userPath = new Path(stagingDir, userName); Path userStagingPath = new Path(userPath, ".staging"); NUnit.Framework.Assert.AreEqual(userStagingPath.ToString(), appMaster.stagingDirPath .ToString()); }
/// <exception cref="System.IO.IOException"/> private static void ConfigureTask(JobConf job, Task task, Credentials credentials , Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt) { job.SetCredentials(credentials); ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment .ContainerId.ToString())).GetApplicationAttemptId(); Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId); // Set it in conf, so as to be able to be used the the OutputCommitter. job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId()); // set tcp nodelay job.SetBoolean("ipc.client.tcpnodelay", true); job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile )); // set the jobToken and shuffle secrets into task task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword())); byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials); if (shuffleSecret == null) { Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret." ); shuffleSecret = jt.GetPassword(); } task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret)); // setup the child's MRConfig.LOCAL_DIR. ConfigureLocalDirs(task, job); // setup the child's attempt directories // Do the task-type specific localization task.LocalizeConfiguration(job); // Set up the DistributedCache related configs MRApps.SetupDistributedCacheLocal(job); // Overwrite the localized task jobconf which is linked to in the current // work-dir. Path localTaskFile = new Path(MRJobConfig.JobConfFile); WriteLocalJobFile(localTaskFile, job); task.SetJobFile(localTaskFile.ToString()); task.SetConf(job); }
public virtual Response GetLogs(string containerIdStr, string filename) { ContainerId containerId; try { containerId = ConverterUtils.ToContainerId(containerIdStr); } catch (ArgumentException) { return(Response.Status(Response.Status.BadRequest).Build()); } FilePath logFile = null; try { logFile = ContainerLogsUtils.GetContainerLogFile(containerId, filename, request.GetRemoteUser (), nmContext); } catch (NotFoundException ex) { return(Response.Status(Response.Status.NotFound).Entity(ex.Message).Build()); } catch (YarnException ex) { return(Response.ServerError().Entity(ex.Message).Build()); } try { FileInputStream fis = ContainerLogsUtils.OpenLogFileForRead(containerIdStr, logFile , nmContext); StreamingOutput stream = new _StreamingOutput_240(fis); return(Response.Ok(stream).Build()); } catch (IOException ex) { return(Response.ServerError().Entity(ex.Message).Build()); } }
public virtual ContainerInfo GetNodeContainer(string id) { ContainerId containerId = null; Init(); try { containerId = ConverterUtils.ToContainerId(id); } catch (Exception) { throw new BadRequestException("invalid container id, " + id); } Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = nmContext.GetContainers()[containerId]; if (container == null) { throw new NotFoundException("container with id, " + id + ", not found"); } return(new ContainerInfo(this.nmContext, container, uriInfo.GetBaseUri().ToString (), webapp.Name())); }
public virtual void TestMRAppMasterSuccessLock() { string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId ())); Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId); Path end = MRApps.GetEndJobCommitSuccessFile(conf, userName, jobId); FileSystem fs = FileSystem.Get(conf); fs.Create(start).Close(); fs.Create(end).Close(); ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host" , -1, -1, Runtime.CurrentTimeMillis(), false, false); bool caught = false; try { MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); } catch (IOException e) { //The IO Exception is expected Log.Info("Caught expected Exception", e); caught = true; } NUnit.Framework.Assert.IsTrue(caught); NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown); NUnit.Framework.Assert.AreEqual(JobStateInternal.Succeeded, appMaster.forcedState ); appMaster.Stop(); // verify the final status is SUCCEEDED VerifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED"); }
private ContainerId VerifyAndGetContainerId(HtmlBlock.Block html) { string containerIdStr = $(YarnWebParams.ContainerId); if (containerIdStr == null || containerIdStr.IsEmpty()) { html.H1().("Cannot get container logs without a ContainerId").(); return(null); } ContainerId containerId = null; try { containerId = ConverterUtils.ToContainerId(containerIdStr); } catch (ArgumentException) { html.H1().("Cannot get container logs for invalid containerId: " + containerIdStr ).(); return(null); } return(containerId); }
public virtual void Logs() { string containerIdStr = $(ContainerId); ContainerId containerId = null; try { containerId = ConverterUtils.ToContainerId(containerIdStr); } catch (ArgumentException) { Render(typeof(ContainerLogsPage)); return; } ApplicationId appId = containerId.GetApplicationAttemptId().GetApplicationId(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = nmContext.GetApplications()[appId]; if (app == null && nmConf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration .DefaultLogAggregationEnabled)) { string logServerUrl = nmConf.Get(YarnConfiguration.YarnLogServerUrl); string redirectUrl = null; if (logServerUrl == null || logServerUrl.IsEmpty()) { redirectUrl = "false"; } else { redirectUrl = Url(logServerUrl, nmContext.GetNodeId().ToString(), containerIdStr, containerIdStr, $(AppOwner)); } Set(ContainerLogsPage.RedirectUrl, redirectUrl); } Render(typeof(ContainerLogsPage)); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void TestNodeSingleContainersHelper(string media) { WebResource r = Resource(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = new MockApp(1); nmContext.GetApplications()[app.GetAppId()] = app; Dictionary <string, string> hash = AddAppContainers(app); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app2 = new MockApp(2); nmContext.GetApplications()[app2.GetAppId()] = app2; AddAppContainers(app2); foreach (string id in hash.Keys) { ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containers") .Path(id).Accept(media).Get <ClientResponse>(); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType() ); JSONObject json = response.GetEntity <JSONObject>(); VerifyNodeContainerInfo(json.GetJSONObject("container"), nmContext.GetContainers( )[ConverterUtils.ToContainerId(id)]); } }
public virtual void TestMRAppMasterMissingStaging() { string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; string containerIdStr = "container_1317529182569_0004_000002_1"; string userName = "******"; JobConf conf = new JobConf(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId (applicationAttemptIdStr); //Delete the staging directory FilePath dir = new FilePath(stagingDir); if (dir.Exists()) { FileUtils.DeleteDirectory(dir); } ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host" , -1, -1, Runtime.CurrentTimeMillis(), false, false); bool caught = false; try { MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName); } catch (IOException e) { //The IO Exception is expected Log.Info("Caught expected Exception", e); caught = true; } NUnit.Framework.Assert.IsTrue(caught); NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown); //Copying the history file is disabled, but it is not really visible from //here NUnit.Framework.Assert.AreEqual(JobStateInternal.Error, appMaster.forcedState); appMaster.Stop(); }
protected override void Render(HtmlBlock.Block html) { string redirectUrl = $(RedirectUrl); if (redirectUrl != null && redirectUrl.Equals("false")) { html.H1("Failed while trying to construct the redirect url to the log" + " server. Log Server url may not be configured" ); } //Intentional fallthrough. ContainerId containerId; try { containerId = ConverterUtils.ToContainerId($(YarnWebParams.ContainerId)); } catch (ArgumentException) { html.H1("Invalid container ID: " + $(YarnWebParams.ContainerId)); return; } try { if ($(ContainerLogType).IsEmpty()) { IList <FilePath> logFiles = ContainerLogsUtils.GetContainerLogDirs(containerId, Request ().GetRemoteUser(), nmContext); PrintLogFileDirectory(html, logFiles); } else { FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, $(ContainerLogType ), Request().GetRemoteUser(), nmContext); PrintLogFile(html, logFile); } }
/// <summary>Get the process id from specified file path.</summary> /// <remarks> /// Get the process id from specified file path. /// Parses each line to find a valid number /// and returns the first one found. /// </remarks> /// <returns>Process Id if obtained from path specified else null</returns> /// <exception cref="System.IO.IOException"/> public static string GetProcessId(Path path) { if (path == null) { throw new IOException("Trying to access process id from a null path"); } Log.Debug("Accessing pid from pid file " + path); string processId = null; BufferedReader bufReader = null; try { FilePath file = new FilePath(path.ToString()); if (file.Exists()) { FileInputStream fis = new FileInputStream(file); bufReader = new BufferedReader(new InputStreamReader(fis, "UTF-8")); while (true) { string line = bufReader.ReadLine(); if (line == null) { break; } string temp = line.Trim(); if (!temp.IsEmpty()) { if (Shell.Windows) { // On Windows, pid is expected to be a container ID, so find first // line that parses successfully as a container ID. try { ConverterUtils.ToContainerId(temp); processId = temp; break; } catch (Exception) { } } else { // do nothing // Otherwise, find first line containing a numeric pid. try { long pid = Sharpen.Extensions.ValueOf(temp); if (pid > 0) { processId = temp; break; } } catch (Exception) { } } } } } } finally { // do nothing if (bufReader != null) { bufReader.Close(); } } Log.Debug("Got pid " + (processId != null ? processId : "null") + " from path " + path); return(processId); }
public virtual void TestFencedState() { TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester (this); RMStateStore store = zkTester.GetRMStateStore(); // Move state to FENCED from ACTIVE store.UpdateFencedState(); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); long submitTime = Runtime.CurrentTimeMillis(); long startTime = submitTime + 1000; // Add a new app RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>(); ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime); Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime); Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context ); Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test"); store.StoreNewApplication(mockApp); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // Add a new attempt ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM (); ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId("appattempt_1234567894321_0001_000001" ); SecretKey clientTokenMasterKey = clientToAMTokenMgr.CreateMasterKey(attemptId); RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics >(); Container container = new ContainerPBImpl(); container.SetId(ConverterUtils.ToContainerId("container_1234567891234_0001_01_000001" )); RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>(); Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId); Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container); Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey ); Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics ); Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()). ThenReturn(new AggregateAppResourceUsage(0, 0)); store.StoreNewApplicationAttempt(mockAttempt); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); long finishTime = submitTime + 1000; // Update attempt ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance (attemptId, container, store.GetCredentialsFromAppAttempt(mockAttempt), startTime , RMAppAttemptState.Finished, "testUrl", "test", FinalApplicationStatus.Succeeded , 100, finishTime, 0, 0); store.UpdateApplicationAttemptState(newAttemptState); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // Update app ApplicationStateData appState = ApplicationStateData.NewInstance(submitTime, startTime , context, "test"); store.UpdateApplicationState(appState); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // Remove app store.RemoveApplication(mockApp); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // store RM delegation token; RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1" ), new Text("renewer1"), new Text("realuser1")); long renewDate1 = Runtime.CurrentTimeMillis(); dtId1.SetSequenceNumber(1111); store.StoreRMDelegationToken(dtId1, renewDate1); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); store.UpdateRMDelegationToken(dtId1, renewDate1); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // remove delegation key; store.RemoveRMDelegationToken(dtId1); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // store delegation master key; DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString ("keyBytes")); store.StoreRMDTMasterKey(key); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // remove delegation master key; store.RemoveRMDTMasterKey(key); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); // store or update AMRMToken; store.StoreOrUpdateAMRMTokenSecretManager(null, false); NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state", true, store.IsFencedState()); store.Close(); }
// TODO later: add explicit "isUber()" checks of some sort /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="System.TypeLoadException"/> public virtual void TestContainerRollingLog() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } SleepJob sleepJob = new SleepJob(); JobConf sleepConf = new JobConf(mrCluster.GetConfig()); sleepConf.Set(MRJobConfig.MapLogLevel, Level.All.ToString()); long userLogKb = 4; sleepConf.SetLong(MRJobConfig.TaskUserlogLimit, userLogKb); sleepConf.SetInt(MRJobConfig.TaskLogBackups, 3); sleepConf.Set(MRJobConfig.MrAmLogLevel, Level.All.ToString()); long amLogKb = 7; sleepConf.SetLong(MRJobConfig.MrAmLogKb, amLogKb); sleepConf.SetInt(MRJobConfig.MrAmLogBackups, 7); sleepJob.SetConf(sleepConf); Job job = sleepJob.CreateJob(1, 0, 1L, 100, 0L, 0); job.SetJarByClass(typeof(SleepJob)); job.AddFileToClassPath(AppJar); // The AppMaster jar itself. job.WaitForCompletion(true); JobId jobId = TypeConverter.ToYarn(job.GetJobID()); ApplicationId appID = jobId.GetAppId(); int pollElapsed = 0; while (true) { Sharpen.Thread.Sleep(1000); pollElapsed += 1000; if (TerminalRmAppStates.Contains(mrCluster.GetResourceManager().GetRMContext().GetRMApps ()[appID].GetState())) { break; } if (pollElapsed >= 60000) { Log.Warn("application did not reach terminal state within 60 seconds"); break; } } NUnit.Framework.Assert.AreEqual(RMAppState.Finished, mrCluster.GetResourceManager ().GetRMContext().GetRMApps()[appID].GetState()); // Job finished, verify logs // string appIdStr = appID.ToString(); string appIdSuffix = Sharpen.Runtime.Substring(appIdStr, "application_".Length, appIdStr .Length); string containerGlob = "container_" + appIdSuffix + "_*_*"; string syslogGlob = appIdStr + Path.Separator + containerGlob + Path.Separator + TaskLog.LogName.Syslog; int numAppMasters = 0; int numMapTasks = 0; for (int i = 0; i < NumNodeMgrs; i++) { Configuration nmConf = mrCluster.GetNodeManager(i).GetConfig(); foreach (string logDir in nmConf.GetTrimmedStrings(YarnConfiguration.NmLogDirs)) { Path absSyslogGlob = new Path(logDir + Path.Separator + syslogGlob); Log.Info("Checking for glob: " + absSyslogGlob); FileStatus[] syslogs = localFs.GlobStatus(absSyslogGlob); foreach (FileStatus slog in syslogs) { bool foundAppMaster = job.IsUber(); Path containerPathComponent = slog.GetPath().GetParent(); if (!foundAppMaster) { ContainerId cid = ConverterUtils.ToContainerId(containerPathComponent.GetName()); foundAppMaster = ((cid.GetContainerId() & ContainerId.ContainerIdBitmask) == 1); } FileStatus[] sysSiblings = localFs.GlobStatus(new Path(containerPathComponent, TaskLog.LogName .Syslog + "*")); // sort to ensure for i > 0 sysSiblings[i] == "syslog.i" Arrays.Sort(sysSiblings); if (foundAppMaster) { numAppMasters++; } else { numMapTasks++; } if (foundAppMaster) { NUnit.Framework.Assert.AreSame("Unexpected number of AM sylog* files", sleepConf. GetInt(MRJobConfig.MrAmLogBackups, 0) + 1, sysSiblings.Length); NUnit.Framework.Assert.IsTrue("AM syslog.1 length kb should be >= " + amLogKb, sysSiblings [1].GetLen() >= amLogKb * 1024); } else { NUnit.Framework.Assert.AreSame("Unexpected number of MR task sylog* files", sleepConf .GetInt(MRJobConfig.TaskLogBackups, 0) + 1, sysSiblings.Length); NUnit.Framework.Assert.IsTrue("MR syslog.1 length kb should be >= " + userLogKb, sysSiblings[1].GetLen() >= userLogKb * 1024); } } } } // Make sure we checked non-empty set // NUnit.Framework.Assert.AreEqual("No AppMaster log found!", 1, numAppMasters); if (sleepConf.GetBoolean(MRJobConfig.JobUbertaskEnable, false)) { NUnit.Framework.Assert.AreEqual("MapTask log with uber found!", 0, numMapTasks); } else { NUnit.Framework.Assert.AreEqual("No MapTask log found!", 1, numMapTasks); } }
private static ApplicationAttemptReport ConvertToApplicationAttemptReport(TimelineEntity entity) { string host = null; int rpcPort = -1; ContainerId amContainerId = null; string trackingUrl = null; string originalTrackingUrl = null; string diagnosticsInfo = null; YarnApplicationAttemptState state = null; IList <TimelineEvent> events = entity.GetEvents(); if (events != null) { foreach (TimelineEvent @event in events) { if (@event.GetEventType().Equals(AppAttemptMetricsConstants.RegisteredEventType)) { IDictionary <string, object> eventInfo = @event.GetEventInfo(); if (eventInfo == null) { continue; } if (eventInfo.Contains(AppAttemptMetricsConstants.HostEventInfo)) { host = eventInfo[AppAttemptMetricsConstants.HostEventInfo].ToString(); } if (eventInfo.Contains(AppAttemptMetricsConstants.RpcPortEventInfo)) { rpcPort = (int)eventInfo[AppAttemptMetricsConstants.RpcPortEventInfo]; } if (eventInfo.Contains(AppAttemptMetricsConstants.MasterContainerEventInfo)) { amContainerId = ConverterUtils.ToContainerId(eventInfo[AppAttemptMetricsConstants .MasterContainerEventInfo].ToString()); } } else { if (@event.GetEventType().Equals(AppAttemptMetricsConstants.FinishedEventType)) { IDictionary <string, object> eventInfo = @event.GetEventInfo(); if (eventInfo == null) { continue; } if (eventInfo.Contains(AppAttemptMetricsConstants.TrackingUrlEventInfo)) { trackingUrl = eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo].ToString (); } if (eventInfo.Contains(AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo)) { originalTrackingUrl = eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo ].ToString(); } if (eventInfo.Contains(AppAttemptMetricsConstants.DiagnosticsInfoEventInfo)) { diagnosticsInfo = eventInfo[AppAttemptMetricsConstants.DiagnosticsInfoEventInfo]. ToString(); } if (eventInfo.Contains(AppAttemptMetricsConstants.StateEventInfo)) { state = YarnApplicationAttemptState.ValueOf(eventInfo[AppAttemptMetricsConstants. StateEventInfo].ToString()); } } } } } return(ApplicationAttemptReport.NewInstance(ConverterUtils.ToApplicationAttemptId (entity.GetEntityId()), host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo , state, amContainerId)); }
/// <returns>the ContainerId for the MRAppMaster.</returns> public virtual ContainerId GetContainerId() { return(ConverterUtils.ToContainerId(datum.containerId.ToString())); }
private static ContainerReport ConvertToContainerReport(TimelineEntity entity, string serverHttpAddress, string user) { int allocatedMem = 0; int allocatedVcore = 0; string allocatedHost = null; int allocatedPort = -1; int allocatedPriority = 0; long createdTime = 0; long finishedTime = 0; string diagnosticsInfo = null; int exitStatus = ContainerExitStatus.Invalid; ContainerState state = null; string nodeHttpAddress = null; IDictionary <string, object> entityInfo = entity.GetOtherInfo(); if (entityInfo != null) { if (entityInfo.Contains(ContainerMetricsConstants.AllocatedMemoryEntityInfo)) { allocatedMem = (int)entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedVcoreEntityInfo)) { allocatedVcore = (int)entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostEntityInfo)) { allocatedHost = entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo].ToString (); } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPortEntityInfo)) { allocatedPort = (int)entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPriorityEntityInfo)) { allocatedPriority = (int)entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo )) { nodeHttpAddress = (string)entityInfo[ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo ]; } } IList <TimelineEvent> events = entity.GetEvents(); if (events != null) { foreach (TimelineEvent @event in events) { if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType)) { createdTime = @event.GetTimestamp(); } else { if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType)) { finishedTime = @event.GetTimestamp(); IDictionary <string, object> eventInfo = @event.GetEventInfo(); if (eventInfo == null) { continue; } if (eventInfo.Contains(ContainerMetricsConstants.DiagnosticsInfoEventInfo)) { diagnosticsInfo = eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo].ToString (); } if (eventInfo.Contains(ContainerMetricsConstants.ExitStatusEventInfo)) { exitStatus = (int)eventInfo[ContainerMetricsConstants.ExitStatusEventInfo]; } if (eventInfo.Contains(ContainerMetricsConstants.StateEventInfo)) { state = ContainerState.ValueOf(eventInfo[ContainerMetricsConstants.StateEventInfo ].ToString()); } } } } } NodeId allocatedNode = NodeId.NewInstance(allocatedHost, allocatedPort); ContainerId containerId = ConverterUtils.ToContainerId(entity.GetEntityId()); string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, allocatedNode. ToString(), containerId.ToString(), containerId.ToString(), user); return(ContainerReport.NewInstance(ConverterUtils.ToContainerId(entity.GetEntityId ()), Resource.NewInstance(allocatedMem, allocatedVcore), NodeId.NewInstance(allocatedHost , allocatedPort), Priority.NewInstance(allocatedPriority), createdTime, finishedTime , diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress)); }
/// <exception cref="System.IO.IOException"/> public override IList <NMStateStoreService.RecoveredContainerState> LoadContainersState () { AList <NMStateStoreService.RecoveredContainerState> containers = new AList <NMStateStoreService.RecoveredContainerState >(); AList <ContainerId> containersToRemove = new AList <ContainerId>(); LeveldbIterator iter = null; try { iter = new LeveldbIterator(db); iter.Seek(JniDBFactory.Bytes(ContainersKeyPrefix)); while (iter.HasNext()) { KeyValuePair <byte[], byte[]> entry = iter.PeekNext(); string key = JniDBFactory.AsString(entry.Key); if (!key.StartsWith(ContainersKeyPrefix)) { break; } int idEndPos = key.IndexOf('/', ContainersKeyPrefix.Length); if (idEndPos < 0) { throw new IOException("Unable to determine container in key: " + key); } ContainerId containerId = ConverterUtils.ToContainerId(Sharpen.Runtime.Substring( key, ContainersKeyPrefix.Length, idEndPos)); string keyPrefix = Sharpen.Runtime.Substring(key, 0, idEndPos + 1); NMStateStoreService.RecoveredContainerState rcs = LoadContainerState(containerId, iter, keyPrefix); // Don't load container without StartContainerRequest if (rcs.startRequest != null) { containers.AddItem(rcs); } else { containersToRemove.AddItem(containerId); } } } catch (DBException e) { throw new IOException(e); } finally { if (iter != null) { iter.Close(); } } // remove container without StartContainerRequest foreach (ContainerId containerId_1 in containersToRemove) { Log.Warn("Remove container " + containerId_1 + " with incomplete records"); try { RemoveContainer(containerId_1); } catch (IOException e) { // TODO: kill and cleanup the leaked container Log.Error("Unable to remove container " + containerId_1 + " in store", e); } } return(containers); }
protected override void Render(HtmlBlock.Block html) { string containerid = $(YarnWebParams.ContainerId); if (containerid.IsEmpty()) { Puts("Bad request: requires container ID"); return; } ContainerId containerId = null; try { containerId = ConverterUtils.ToContainerId(containerid); } catch (ArgumentException) { Puts("Invalid container ID: " + containerid); return; } UserGroupInformation callerUGI = GetCallerUGI(); ContainerReport containerReport = null; try { GetContainerReportRequest request = GetContainerReportRequest.NewInstance(containerId ); if (callerUGI == null) { containerReport = appBaseProt.GetContainerReport(request).GetContainerReport(); } else { containerReport = callerUGI.DoAs(new _PrivilegedExceptionAction_78(this, request) ); } } catch (Exception e) { string message = "Failed to read the container " + containerid + "."; Log.Error(message, e); html.P().(message).(); return; } if (containerReport == null) { Puts("Container not found: " + containerid); return; } ContainerInfo container = new ContainerInfo(containerReport); SetTitle(StringHelper.Join("Container ", containerid)); Info("Container Overview").("Container State:", container.GetContainerState() == null ? Unavailable : container.GetContainerState()).("Exit Status:", container.GetContainerExitStatus ()).("Node:", container.GetNodeHttpAddress() == null ? "#" : container.GetNodeHttpAddress (), container.GetNodeHttpAddress() == null ? "N/A" : container.GetNodeHttpAddress ()).("Priority:", container.GetPriority()).("Started:", Times.Format(container.GetStartedTime ())).("Elapsed:", StringUtils.FormatTime(Times.Elapsed(container.GetStartedTime( ), container.GetFinishedTime()))).("Resource:", container.GetAllocatedMB() + " Memory, " + container.GetAllocatedVCores() + " VCores").("Logs:", container.GetLogUrl() == null ? "#" : container.GetLogUrl(), container.GetLogUrl() == null ? "N/A" : "Logs" ).("Diagnostics:", container.GetDiagnosticsInfo() == null ? string.Empty : container .GetDiagnosticsInfo()); html.(typeof(InfoBlock)); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void TestNodeHelper(string path, string media) { WebResource r = Resource(); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = new MockApp(1); nmContext.GetApplications()[app.GetAppId()] = app; AddAppContainers(app); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app2 = new MockApp(2); nmContext.GetApplications()[app2.GetAppId()] = app2; AddAppContainers(app2); ClientResponse response = r.Path("ws").Path("v1").Path("node").Path(path).Accept( media).Get <ClientResponse>(); NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType() ); JSONObject json = response.GetEntity <JSONObject>(); JSONObject info = json.GetJSONObject("containers"); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, info.Length()); JSONArray conInfo = info.GetJSONArray("container"); NUnit.Framework.Assert.AreEqual("incorrect number of elements", 4, conInfo.Length ()); for (int i = 0; i < conInfo.Length(); i++) { VerifyNodeContainerInfo(conInfo.GetJSONObject(i), nmContext.GetContainers()[ConverterUtils .ToContainerId(conInfo.GetJSONObject(i).GetString("id"))]); } }
/// <summary>Prints the container report for an container id.</summary> /// <param name="containerId"/> /// <returns>exitCode</returns> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private int PrintContainerReport(string containerId) { ContainerReport containerReport = null; try { containerReport = client.GetContainerReport((ConverterUtils.ToContainerId(containerId ))); } catch (ApplicationNotFoundException) { sysout.WriteLine("Application for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server." ); return(-1); } catch (ApplicationAttemptNotFoundException) { sysout.WriteLine("Application Attempt for Container with id '" + containerId + "' doesn't exist in RM or Timeline Server." ); return(-1); } catch (ContainerNotFoundException) { sysout.WriteLine("Container with id '" + containerId + "' doesn't exist in RM or Timeline Server." ); return(-1); } // Use PrintWriter.println, which uses correct platform line ending. ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter containerReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding ("UTF-8"))); if (containerReport != null) { containerReportStr.WriteLine("Container Report : "); containerReportStr.Write("\tContainer-Id : "); containerReportStr.WriteLine(containerReport.GetContainerId()); containerReportStr.Write("\tStart-Time : "); containerReportStr.WriteLine(containerReport.GetCreationTime()); containerReportStr.Write("\tFinish-Time : "); containerReportStr.WriteLine(containerReport.GetFinishTime()); containerReportStr.Write("\tState : "); containerReportStr.WriteLine(containerReport.GetContainerState()); containerReportStr.Write("\tLOG-URL : "); containerReportStr.WriteLine(containerReport.GetLogUrl()); containerReportStr.Write("\tHost : "); containerReportStr.WriteLine(containerReport.GetAssignedNode()); containerReportStr.Write("\tNodeHttpAddress : "); containerReportStr.WriteLine(containerReport.GetNodeHttpAddress() == null ? "N/A" : containerReport.GetNodeHttpAddress()); containerReportStr.Write("\tDiagnostics : "); containerReportStr.Write(containerReport.GetDiagnosticsInfo()); } else { containerReportStr.Write("Container with id '" + containerId + "' doesn't exist in Timeline Server." ); containerReportStr.Close(); sysout.WriteLine(baos.ToString("UTF-8")); return(-1); } containerReportStr.Close(); sysout.WriteLine(baos.ToString("UTF-8")); return(0); }
public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType, long startTime , string trackerName, int httpPort, int shufflePort, string locality, string avataar ) : this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort, ConverterUtils .ToContainerId("container_-1_-1_-1_-1"), locality, avataar) { }