/// <exception cref="System.IO.IOException"/> private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app) { Dispatcher dispatcher = new AsyncDispatcher(); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 2); nmContext.GetContainers()[container1.GetContainerId()] = container1; nmContext.GetContainers()[container2.GetContainerId()] = container2; app.GetContainers()[container1.GetContainerId()] = container1; app.GetContainers()[container2.GetContainerId()] = container2; Dictionary <string, string> hash = new Dictionary <string, string>(); hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString (); hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString (); return(hash); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeAppInfoGeneric(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, string id, string state, string user) { WebServicesTestUtils.CheckStringMatch("id", app.GetAppId().ToString(), id); WebServicesTestUtils.CheckStringMatch("state", app.GetApplicationState().ToString (), state); WebServicesTestUtils.CheckStringMatch("user", app.GetUser().ToString(), user); }
/// <exception cref="System.Exception"/> private void WaitForAppState(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, ApplicationState state) { int msecPerSleep = 10; int msecLeft = 5000; while (app.GetApplicationState() != state && msecLeft > 0) { Sharpen.Thread.Sleep(msecPerSleep); msecLeft -= msecPerSleep; } NUnit.Framework.Assert.AreEqual(state, app.GetApplicationState()); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeAppInfo(JSONObject info, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, Dictionary <string, string> hash) { NUnit.Framework.Assert.AreEqual("incorrect number of elements", 4, info.Length()); VerifyNodeAppInfoGeneric(app, info.GetString("id"), info.GetString("state"), info .GetString("user")); JSONArray containerids = info.GetJSONArray("containerids"); for (int i = 0; i < containerids.Length(); i++) { string id = containerids.GetString(i); NUnit.Framework.Assert.AreEqual("extra containerid: " + id, id, Sharpen.Collections.Remove (hash, id)); } NUnit.Framework.Assert.IsTrue("missing containerids", hash.IsEmpty()); }
/// <exception cref="System.Exception"/> internal static void WaitForApplicationState(ContainerManagerImpl containerManager , ApplicationId appID, ApplicationState finalState) { // Wait for app-finish Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = containerManager.GetContext().GetApplications()[appID]; int timeout = 0; while (!(app.GetApplicationState().Equals(finalState)) && timeout++ < 15) { Log.Info("Waiting for app to reach " + finalState + ".. Current state is " + app. GetApplicationState()); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsTrue("App is not in " + finalState + " yet!! Timedout!!" , app.GetApplicationState().Equals(finalState)); }
public virtual AppInfo GetNodeApp(string appId) { Init(); ApplicationId id = ConverterUtils.ToApplicationId(recordFactory, appId); if (id == null) { throw new NotFoundException("app with id " + appId + " not found"); } Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = this.nmContext.GetApplications()[id]; if (app == null) { throw new NotFoundException("app with id " + appId + " not found"); } return(new AppInfo(app)); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeAppInfoXML(NodeList nodes, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app, Dictionary <string, string> hash) { for (int i = 0; i < nodes.GetLength(); i++) { Element element = (Element)nodes.Item(i); VerifyNodeAppInfoGeneric(app, WebServicesTestUtils.GetXmlString(element, "id"), WebServicesTestUtils .GetXmlString(element, "state"), WebServicesTestUtils.GetXmlString(element, "user" )); NodeList ids = element.GetElementsByTagName("containerids"); for (int j = 0; j < ids.GetLength(); j++) { Element line = (Element)ids.Item(j); Node first = line.GetFirstChild(); string val = first.GetNodeValue(); NUnit.Framework.Assert.AreEqual("extra containerid: " + val, val, Sharpen.Collections.Remove (hash, val)); } NUnit.Framework.Assert.IsTrue("missing containerids", hash.IsEmpty()); } }
public virtual void TestApplicationRecovery() { YarnConfiguration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true); conf.Set(YarnConfiguration.NmAddress, "localhost:1234"); conf.SetBoolean(YarnConfiguration.YarnAclEnable, true); conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user"); NMStateStoreService stateStore = new NMMemoryStateStoreService(); stateStore.Init(conf); stateStore.Start(); Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf ), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore ); ContainerManagerImpl cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); // simulate registration with RM MasterKey masterKey = new MasterKeyPBImpl(); masterKey.SetKeyId(123); masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 })); context.GetContainerTokenSecretManager().SetMasterKey(masterKey); context.GetNMTokenSecretManager().SetMasterKey(masterKey); // add an application by starting a container string appUser = "******"; string modUser = "******"; string viewUser = "******"; string enemyUser = "******"; ApplicationId appId = ApplicationId.NewInstance(0, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1); ContainerId cid = ContainerId.NewContainerId(attemptId, 1); IDictionary <string, LocalResource> localResources = Collections.EmptyMap(); IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap(); IList <string> containerCmds = Sharpen.Collections.EmptyList(); IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap(); Credentials containerCreds = new Credentials(); DataOutputBuffer dob = new DataOutputBuffer(); containerCreds.WriteTokenStorageToStream(dob); ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(); acls[ApplicationAccessType.ModifyApp] = modUser; acls[ApplicationAccessType.ViewApp] = viewUser; ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv , containerCmds, serviceData, containerTokens, acls); // create the logAggregationContext LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern" , "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation" ); StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext ); NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty()); NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = context.GetApplications()[appId]; NUnit.Framework.Assert.IsNotNull(app); WaitForAppState(app, ApplicationState.Initing); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId)); // reset container manager and verify app recovered with proper acls cm.Stop(); context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore); cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count); app = context.GetApplications()[appId]; NUnit.Framework.Assert.IsNotNull(app); // check whether LogAggregationContext is recovered correctly LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext (); NUnit.Framework.Assert.IsNotNull(recovered); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered .GetIncludePattern()); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered .GetExcludePattern()); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern (), recovered.GetRolledLogsIncludePattern()); NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern (), recovered.GetRolledLogsExcludePattern()); WaitForAppState(app, ApplicationState.Initing); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId)); // simulate application completion IList <ApplicationId> finishedApps = new AList <ApplicationId>(); finishedApps.AddItem(appId); cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason. ByResourcemanager)); WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup); // restart and verify app is marked for finishing cm.Stop(); context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore); cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count); app = context.GetApplications()[appId]; NUnit.Framework.Assert.IsNotNull(app); WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId)); NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId)); NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId)); // simulate log aggregation completion app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup )); NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished ); app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished )); // restart and verify app is no longer present after recovery cm.Stop(); context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore); cm = CreateContainerManager(context); cm.Init(conf); cm.Start(); NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty()); cm.Stop(); }
public override void Handle(LocalizationEvent @event) { switch (@event.GetType()) { case LocalizationEventType.InitApplicationResources: { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = ((ApplicationLocalizationEvent)@event).GetApplication(); // Simulate event from ApplicationLocalization. this.dispatcher.GetEventHandler().Handle(new ApplicationInitedEvent(app.GetAppId( ))); break; } case LocalizationEventType.InitContainerResources: { ContainerLocalizationRequestEvent rsrcReqs = (ContainerLocalizationRequestEvent)@event; // simulate localization of all requested resources foreach (ICollection <LocalResourceRequest> rc in rsrcReqs.GetRequestedResources() .Values) { foreach (LocalResourceRequest req in rc) { Org.Apache.Hadoop.Yarn.Server.Nodemanager.DummyContainerManager.Log.Info("DEBUG: " + req + ":" + rsrcReqs.GetContainer().GetContainerId()); this.dispatcher.GetEventHandler().Handle(new ContainerResourceLocalizedEvent(rsrcReqs .GetContainer().GetContainerId(), req, new Path("file:///local" + req.GetPath(). ToUri().GetPath()))); } } break; } case LocalizationEventType.CleanupContainerResources: { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = ((ContainerLocalizationEvent)@event).GetContainer(); // TODO: delete the container dir this.dispatcher.GetEventHandler().Handle(new ContainerEvent(container.GetContainerId (), ContainerEventType.ContainerResourcesCleanedup)); break; } case LocalizationEventType.DestroyApplicationResources: { Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application application = ((ApplicationLocalizationEvent)@event).GetApplication(); // decrement reference counts of all resources associated with this // app this.dispatcher.GetEventHandler().Handle(new ApplicationEvent(application.GetAppId (), ApplicationEventType.ApplicationResourcesCleanedup)); break; } default: { NUnit.Framework.Assert.Fail("Unexpected event: " + @event.GetType()); break; } } }