Example #1
0
        /// <summary>This tests whether a containerId is serialized/deserialized with epoch.</summary>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        private void TestContainerTokenWithEpoch(Configuration conf)
        {
            Log.Info("Running test for serializing/deserializing containerIds");
            NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.GetResourceManager
                                                                    ().GetRMContext().GetNMTokenSecretManager();
            ApplicationId            appId                    = ApplicationId.NewInstance(1, 1);
            ApplicationAttemptId     appAttemptId             = ApplicationAttemptId.NewInstance(appId, 0);
            ContainerId              cId                      = ContainerId.NewContainerId(appAttemptId, (5L << 40) | 3L);
            NodeManager              nm                       = yarnCluster.GetNodeManager(0);
            NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.GetNMContext().GetNMTokenSecretManager
                                                                    ();
            string user = "******";

            WaitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);
            NodeId nodeId = nm.GetNMContext().GetNodeId();

            // Both id should be equal.
            NUnit.Framework.Assert.AreEqual(nmTokenSecretManagerInNM.GetCurrentKey().GetKeyId
                                                (), nmTokenSecretManagerInRM.GetCurrentKey().GetKeyId());
            // Creating a normal Container Token
            RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.GetResourceManager
                                                                            ().GetRMContext().GetContainerTokenSecretManager();
            Resource r = Resource.NewInstance(1230, 2);

            Org.Apache.Hadoop.Yarn.Api.Records.Token containerToken = containerTokenSecretManager
                                                                      .CreateContainerToken(cId, nodeId, user, r, Priority.NewInstance(0), 0);
            ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
                );

            byte[]          tokenIdentifierContent = ((byte[])containerToken.GetIdentifier().Array());
            DataInputBuffer dib = new DataInputBuffer();

            dib.Reset(tokenIdentifierContent, tokenIdentifierContent.Length);
            containerTokenIdentifier.ReadFields(dib);
            NUnit.Framework.Assert.AreEqual(cId, containerTokenIdentifier.GetContainerID());
            NUnit.Framework.Assert.AreEqual(cId.ToString(), containerTokenIdentifier.GetContainerID
                                                ().ToString());
            Org.Apache.Hadoop.Yarn.Api.Records.Token nmToken = nmTokenSecretManagerInRM.CreateNMToken
                                                                   (appAttemptId, nodeId, user);
            YarnRPC rpc = YarnRPC.Create(conf);

            TestStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false);
            IList <ContainerId> containerIds = new List <ContainerId>();

            containerIds.AddItem(cId);
            ContainerManagementProtocol proxy = GetContainerManagementProtocolProxy(rpc, nmToken
                                                                                    , nodeId, user);
            GetContainerStatusesResponse res = proxy.GetContainerStatuses(GetContainerStatusesRequest
                                                                          .NewInstance(containerIds));

            NUnit.Framework.Assert.IsNotNull(res.GetContainerStatuses()[0]);
            NUnit.Framework.Assert.AreEqual(cId, res.GetContainerStatuses()[0].GetContainerId
                                                ());
            NUnit.Framework.Assert.AreEqual(cId.ToString(), res.GetContainerStatuses()[0].GetContainerId
                                                ().ToString());
        }
Example #2
0
 private void ClearLimits(ContainerId containerId)
 {
     if (IsCpuWeightEnabled())
     {
         DeleteCgroup(PathForCgroup(ControllerCpu, containerId.ToString()));
     }
 }
            /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
            public StartContainersResponse StartContainers(StartContainersRequest requests)
            {
                StartContainerRequest request = requests.GetStartContainerRequests()[0];

                Log.Info("Container started by MyContainerManager: " + request);
                launched = true;
                IDictionary <string, string> env = request.GetContainerLaunchContext().GetEnvironment
                                                       ();
                Token containerToken             = request.GetContainerToken();
                ContainerTokenIdentifier tokenId = null;

                try
                {
                    tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
                }
                catch (IOException e)
                {
                    throw RPCUtil.GetRemoteException(e);
                }
                ContainerId containerId = tokenId.GetContainerID();

                containerIdAtContainerManager = containerId.ToString();
                attemptIdAtContainerManager   = containerId.GetApplicationAttemptId().ToString();
                nmHostAtContainerManager      = tokenId.GetNmHostAddress();
                submitTimeAtContainerManager  = long.Parse(env[ApplicationConstants.AppSubmitTimeEnv
                                                           ]);
                maxAppAttempts = System.Convert.ToInt32(env[ApplicationConstants.MaxAppAttemptsEnv
                                                        ]);
                return(StartContainersResponse.NewInstance(new Dictionary <string, ByteBuffer>(),
                                                           new AList <ContainerId>(), new Dictionary <ContainerId, SerializedException>()));
            }
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestContainerLaunchNullImage()
        {
            string appSubmitter = "nobody";
            string appId        = "APP_ID";
            string containerId  = "CONTAINER_ID";
            string testImage    = string.Empty;

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                            >(Org.Mockito.Mockito.ReturnsDeepStubs);
            ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
                                                                     );
            ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
                                                 ();
            Dictionary <string, string> env = new Dictionary <string, string>();

            Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
            Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
            Org.Mockito.Mockito.When(cId.GetApplicationAttemptId().GetApplicationId().ToString
                                         ()).ThenReturn(appId);
            Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
            env[YarnConfiguration.NmDockerContainerExecutorImageName] = testImage;
            dockerContainerExecutor.GetConf().Set(YarnConfiguration.NmDockerContainerExecutorImageName
                                                  , testImage);
            Path scriptPath = new Path("file:///bin/echo");
            Path tokensPath = new Path("file:///dev/null");
            Path pidFile    = new Path(workDir, "pid.txt");

            dockerContainerExecutor.ActivateContainer(cId, pidFile);
            dockerContainerExecutor.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
                                                    , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());
        }
Example #5
0
        public AMAttemptInfo(AMInfo amInfo, string jobId, string user)
        {
            this.nodeHttpAddress = string.Empty;
            this.nodeId          = string.Empty;
            string nmHost     = amInfo.GetNodeManagerHost();
            int    nmHttpPort = amInfo.GetNodeManagerHttpPort();
            int    nmPort     = amInfo.GetNodeManagerPort();

            if (nmHost != null)
            {
                this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
                NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
                this.nodeId = nodeId.ToString();
            }
            this.id          = amInfo.GetAppAttemptId().GetAttemptId();
            this.startTime   = amInfo.GetStartTime();
            this.containerId = string.Empty;
            this.logsLink    = string.Empty;
            ContainerId containerId = amInfo.GetContainerId();

            if (containerId != null)
            {
                this.containerId = containerId.ToString();
                this.logsLink    = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress
                                                     , StringHelper.Ujoin("node", "containerlogs", this.containerId, user));
            }
        }
        public virtual void TestContainerLaunch()
        {
            string appSubmitter = "nobody";
            string cmd          = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
                );
            string appId       = "APP_ID";
            string containerId = "CONTAINER_ID";

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                            >();
            ContainerId            cId     = Org.Mockito.Mockito.Mock <ContainerId>();
            ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
                                                 ();
            Dictionary <string, string> env = new Dictionary <string, string>();

            Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
            Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
            Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
            Path scriptPath = new Path("file:///bin/echo");
            Path tokensPath = new Path("file:///dev/null");
            Path workDir    = new Path("/tmp");
            Path pidFile    = new Path(workDir, "pid.txt");

            mockExec.ActivateContainer(cId, pidFile);
            int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
                                               , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());

            NUnit.Framework.Assert.AreEqual(0, ret);
            NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
                                                          , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
                                                          , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
                                                          .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
        }
        public virtual void TestContainerIdWithEpoch()
        {
            ContainerId id = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(0, 0,
                                                                                       0, 25645811);
            string cid = ConverterUtils.ToString(id);

            NUnit.Framework.Assert.AreEqual("container_0_0000_00_25645811", cid);
            ContainerId gen = ConverterUtils.ToContainerId(cid);

            NUnit.Framework.Assert.AreEqual(gen.ToString(), id.ToString());
            long        ts  = Runtime.CurrentTimeMillis();
            ContainerId id2 = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(36473
                                                                                        , 4365472, ts, 4298334883325L);
            string cid2 = ConverterUtils.ToString(id2);

            NUnit.Framework.Assert.AreEqual("container_e03_" + ts + "_36473_4365472_999799999997"
                                            , cid2);
            ContainerId gen2 = ConverterUtils.ToContainerId(cid2);

            NUnit.Framework.Assert.AreEqual(gen2.ToString(), id2.ToString());
            ContainerId id3 = Org.Apache.Hadoop.Yarn.Api.TestContainerId.NewContainerId(36473
                                                                                        , 4365472, ts, 844424930131965L);
            string cid3 = ConverterUtils.ToString(id3);

            NUnit.Framework.Assert.AreEqual("container_e767_" + ts + "_36473_4365472_1099511627773"
                                            , cid3);
            ContainerId gen3 = ConverterUtils.ToContainerId(cid3);

            NUnit.Framework.Assert.AreEqual(gen3.ToString(), id3.ToString());
        }
Example #8
0
        /// <exception cref="System.IO.IOException"/>
        public override void RemoveContainer(ContainerId containerId)
        {
            string keyPrefix = ContainersKeyPrefix + containerId.ToString();

            try
            {
                WriteBatch batch = db.CreateWriteBatch();
                try
                {
                    batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerRequestKeySuffix));
                    batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerDiagsKeySuffix));
                    batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerLaunchedKeySuffix));
                    batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerKilledKeySuffix));
                    batch.Delete(JniDBFactory.Bytes(keyPrefix + ContainerExitCodeKeySuffix));
                    db.Write(batch);
                }
                finally
                {
                    batch.Close();
                }
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
        }
Example #9
0
        public AMAttemptInfo(AMInfo amInfo, string jobId, string user, string host, string
                             pathPrefix)
        {
            this.nodeHttpAddress = string.Empty;
            this.nodeId          = string.Empty;
            string nmHost     = amInfo.GetNodeManagerHost();
            int    nmHttpPort = amInfo.GetNodeManagerHttpPort();
            int    nmPort     = amInfo.GetNodeManagerPort();

            if (nmHost != null)
            {
                this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
                NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
                this.nodeId = nodeId.ToString();
            }
            this.id            = amInfo.GetAppAttemptId().GetAttemptId();
            this.startTime     = amInfo.GetStartTime();
            this.containerId   = string.Empty;
            this.logsLink      = string.Empty;
            this.shortLogsLink = string.Empty;
            ContainerId containerId = amInfo.GetContainerId();

            if (containerId != null)
            {
                this.containerId = containerId.ToString();
                this.logsLink    = StringHelper.Join(host, pathPrefix, StringHelper.Ujoin("logs", this
                                                                                          .nodeId, this.containerId, jobId, user));
                this.shortLogsLink = StringHelper.Ujoin("logs", this.nodeId, this.containerId, jobId
                                                        , user);
            }
        }
        public virtual void TestForCorruptedAggregatedLogs()
        {
            Configuration conf             = new Configuration();
            FilePath      workDir          = new FilePath(testWorkDir, "testReadAcontainerLogs1");
            Path          remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile");
            Path          srcFileRoot      = new Path(workDir.GetAbsolutePath(), "srcFiles");
            ContainerId   testContainerId  = TestContainerId.NewContainerId(1, 1, 1, 1);
            Path          t = new Path(srcFileRoot, testContainerId.GetApplicationAttemptId().GetApplicationId
                                           ().ToString());
            Path srcFilePath = new Path(t, testContainerId.ToString());
            long numChars    = 950000;

            WriteSrcFileAndALog(srcFilePath, "stdout", numChars, remoteAppLogFile, srcFileRoot
                                , testContainerId);
            AggregatedLogFormat.LogReader logReader = new AggregatedLogFormat.LogReader(conf,
                                                                                        remoteAppLogFile);
            AggregatedLogFormat.LogKey rLogKey = new AggregatedLogFormat.LogKey();
            DataInputStream            dis     = logReader.Next(rLogKey);
            TextWriter writer = new StringWriter();

            try
            {
                AggregatedLogFormat.LogReader.ReadAcontainerLogs(dis, writer);
            }
            catch (Exception e)
            {
                if (e.ToString().Contains("NumberFormatException"))
                {
                    NUnit.Framework.Assert.Fail("Aggregated logs are corrupted.");
                }
            }
        }
Example #11
0
        /// <summary>A helper api for creating an audit log for a failure event.</summary>
        internal static string CreateFailureLog(string user, string operation, string perm
                                                , string target, string description, ApplicationId appId, ApplicationAttemptId attemptId
                                                , ContainerId containerId)
        {
            StringBuilder b = new StringBuilder();

            Start(RMAuditLogger.Keys.User, user, b);
            AddRemoteIP(b);
            Add(RMAuditLogger.Keys.Operation, operation, b);
            Add(RMAuditLogger.Keys.Target, target, b);
            Add(RMAuditLogger.Keys.Result, RMAuditLogger.AuditConstants.Failure, b);
            Add(RMAuditLogger.Keys.Description, description, b);
            Add(RMAuditLogger.Keys.Permissions, perm, b);
            if (appId != null)
            {
                Add(RMAuditLogger.Keys.Appid, appId.ToString(), b);
            }
            if (attemptId != null)
            {
                Add(RMAuditLogger.Keys.Appattemptid, attemptId.ToString(), b);
            }
            if (containerId != null)
            {
                Add(RMAuditLogger.Keys.Containerid, containerId.ToString(), b);
            }
            return(b.ToString());
        }
        private ContainerId GetNextContainerId()
        {
            ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>();
            string      id  = "CONTAINER_" + GetNextId();

            Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(id);
            return(cId);
        }
Example #13
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
                                          , FilePath scriptFileDir, FilePath processStartFile)
        {
            FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
                                                            );
            ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                             >();
            NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
                                                                                       ).ToString(), 12345);
            URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
                                                                                           Path(scriptFile.GetAbsolutePath())));
            LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();

            localResource.SetResource(localResourceUri);
            localResource.SetSize(-1);
            localResource.SetVisibility(LocalResourceVisibility.Application);
            localResource.SetType(LocalResourceType.File);
            localResource.SetTimestamp(scriptFile.LastModified());
            string destinationFile = "dest_file";
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            localResources[destinationFile] = localResource;
            containerLaunchContext.SetLocalResources(localResources);
            IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));

            containerLaunchContext.SetCommands(commands);
            IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
                                                                                      , 12345);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
                                                                                         ());

            Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
                                                                                 .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
                                                                                                                                                                (), nodeId, user), containerManagerBindAddress);
            currentUser.AddToken(nmToken);
            ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
                                                                                ());
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
                                                                                                                                ().GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            IList <ContainerId> containerIds = new AList <ContainerId>();

            containerIds.AddItem(cId);
            GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
                                                                                          );
            ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
                                              GetContainerStatuses()[0];

            NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
                                                ));
        }
Example #14
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidContainerReleaseRequest()
        {
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                // Register node1
                MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                // Submit an application
                RMApp app1 = rm.SubmitApp(1024);
                // kick the scheduling
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                am1.RegisterAppAttempt();
                am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
                AllocateResponse alloc1Response = am1.Schedule();
                // send the request
                // kick the scheduler
                nm1.NodeHeartbeat(true);
                while (alloc1Response.GetAllocatedContainers().Count < 1)
                {
                    Log.Info("Waiting for containers to be created for app 1...");
                    Sharpen.Thread.Sleep(1000);
                    alloc1Response = am1.Schedule();
                }
                NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
                RMApp app2 = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
                MockAM       am2      = rm.SendAMLaunched(attempt2.GetAppAttemptId());
                am2.RegisterAppAttempt();
                // Now trying to release container allocated for app1 -> appAttempt1.
                ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
                am2.AddContainerToBeReleased(cId);
                try
                {
                    am2.Schedule();
                    NUnit.Framework.Assert.Fail("Exception was expected!!");
                }
                catch (InvalidContainerReleaseException e)
                {
                    StringBuilder sb = new StringBuilder("Cannot release container : ");
                    sb.Append(cId.ToString());
                    sb.Append(" not belonging to this application attempt : ");
                    sb.Append(attempt2.GetAppAttemptId().ToString());
                    NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
                }
            }
            finally
            {
                if (rm != null)
                {
                    rm.Stop();
                }
            }
        }
Example #15
0
 /// <summary>Print all the information about this AM.</summary>
 public virtual void PrintAll()
 {
     System.Console.Out.WriteLine("APPLICATION_ATTEMPT_ID:" + appAttemptId.ToString());
     System.Console.Out.WriteLine("START_TIME: " + startTime);
     System.Console.Out.WriteLine("CONTAINER_ID: " + containerId.ToString());
     System.Console.Out.WriteLine("NODE_MANAGER_HOST: " + nodeManagerHost);
     System.Console.Out.WriteLine("NODE_MANAGER_PORT: " + nodeManagerPort);
     System.Console.Out.WriteLine("NODE_MANAGER_HTTP_PORT: " + nodeManagerHttpPort);
 }
        private ContainerId GetNextContainerId()
        {
            ContainerId cId = Org.Mockito.Mockito.Mock <ContainerId>(Org.Mockito.Mockito.ReturnsDeepStubs
                                                                     );
            string id = "CONTAINER_" + Runtime.CurrentTimeMillis();

            Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(id);
            return(cId);
        }
Example #17
0
 /// <exception cref="System.IO.IOException"/>
 private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
                                  )
 {
     // read application history data
     NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count);
     for (int i = 1; i <= num; ++i)
     {
         ApplicationId          appId   = ApplicationId.NewInstance(0, i);
         ApplicationHistoryData appData = store.GetApplication(appId);
         NUnit.Framework.Assert.IsNotNull(appData);
         NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName());
         NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo());
         // read application attempt history data
         NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count);
         for (int j = 1; j <= num; ++j)
         {
             ApplicationAttemptId          appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
             ApplicationAttemptHistoryData attemptData  = store.GetApplicationAttempt(appAttemptId
                                                                                      );
             NUnit.Framework.Assert.IsNotNull(attemptData);
             NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost());
             if (missingApplicationAttempt && j == num)
             {
                 NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo());
                 continue;
             }
             else
             {
                 NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo
                                                     ());
             }
             // read container history data
             NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count);
             for (int k = 1; k <= num; ++k)
             {
                 ContainerId          containerId   = ContainerId.NewContainerId(appAttemptId, k);
                 ContainerHistoryData containerData = store.GetContainer(containerId);
                 NUnit.Framework.Assert.IsNotNull(containerData);
                 NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData
                                                 .GetPriority());
                 if (missingContainer && k == num)
                 {
                     NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo());
                 }
                 else
                 {
                     NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo
                                                         ());
                 }
             }
             ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
             NUnit.Framework.Assert.IsNotNull(masterContainer);
             NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
                                             .GetContainerId());
         }
     }
 }
        public virtual void TestContainerLaunchError()
        {
            // reinitialize executer
            FilePath f = new FilePath("./src/test/resources/mock-container-executer-with-error"
                                      );

            if (!FileUtil.CanExecute(f))
            {
                FileUtil.SetExecutable(f, true);
            }
            string        executorPath = f.GetAbsolutePath();
            Configuration conf         = new Configuration();

            conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
            conf.Set(YarnConfiguration.NmLocalDirs, "file:///bin/echo");
            conf.Set(YarnConfiguration.NmLogDirs, "file:///dev/null");
            mockExec = Org.Mockito.Mockito.Spy(new LinuxContainerExecutor());
            Org.Mockito.Mockito.DoAnswer(new _Answer_226()).When(mockExec).LogOutput(Matchers.Any
                                                                                     <string>());
            dirsHandler = new LocalDirsHandlerService();
            dirsHandler.Init(conf);
            mockExec.SetConf(conf);
            string appSubmitter = "nobody";
            string cmd          = LinuxContainerExecutor.Commands.LaunchContainer.GetValue().ToString(
                );
            string appId       = "APP_ID";
            string containerId = "CONTAINER_ID";

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                            >();
            ContainerId            cId     = Org.Mockito.Mockito.Mock <ContainerId>();
            ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
                                                 ();
            Dictionary <string, string> env = new Dictionary <string, string>();

            Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
            Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
            Org.Mockito.Mockito.DoAnswer(new _Answer_254()).When(container).Handle(Matchers.Any
                                                                                   <ContainerDiagnosticsUpdateEvent>());
            Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
            Path scriptPath = new Path("file:///bin/echo");
            Path tokensPath = new Path("file:///dev/null");
            Path workDir    = new Path("/tmp");
            Path pidFile    = new Path(workDir, "pid.txt");

            mockExec.ActivateContainer(cId, pidFile);
            int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
                                               , appId, workDir, dirsHandler.GetLocalDirs(), dirsHandler.GetLogDirs());

            NUnit.Framework.Assert.AreNotSame(0, ret);
            NUnit.Framework.Assert.AreEqual(Arrays.AsList(YarnConfiguration.DefaultNmNonsecureModeLocalUser
                                                          , appSubmitter, cmd, appId, containerId, workDir.ToString(), "/bin/echo", "/dev/null"
                                                          , pidFile.ToString(), StringUtils.Join(",", dirsHandler.GetLocalDirs()), StringUtils
                                                          .Join(",", dirsHandler.GetLogDirs()), "cgroups=none"), ReadMockParams());
        }
 /// <exception cref="System.IO.IOException"/>
 public virtual ContainerHistoryData GetContainer(ContainerId containerId)
 {
     FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
                                                                        (containerId.GetApplicationAttemptId().GetApplicationId());
     try
     {
         bool readStartData  = false;
         bool readFinishData = false;
         ContainerHistoryData historyData = ContainerHistoryData.NewInstance(containerId,
                                                                             null, null, null, long.MinValue, long.MaxValue, null, int.MaxValue, null);
         while ((!readStartData || !readFinishData) && hfReader.HasNext())
         {
             FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
             if (entry.key.id.Equals(containerId.ToString()))
             {
                 if (entry.key.suffix.Equals(StartDataSuffix))
                 {
                     ContainerStartData startData = ParseContainerStartData(entry.value);
                     MergeContainerHistoryData(historyData, startData);
                     readStartData = true;
                 }
                 else
                 {
                     if (entry.key.suffix.Equals(FinishDataSuffix))
                     {
                         ContainerFinishData finishData = ParseContainerFinishData(entry.value);
                         MergeContainerHistoryData(historyData, finishData);
                         readFinishData = true;
                     }
                 }
             }
         }
         if (!readStartData && !readFinishData)
         {
             return(null);
         }
         if (!readStartData)
         {
             Log.Warn("Start information is missing for container " + containerId);
         }
         if (!readFinishData)
         {
             Log.Warn("Finish information is missing for container " + containerId);
         }
         Log.Info("Completed reading history information of container " + containerId);
         return(historyData);
     }
     catch (IOException e)
     {
         Log.Error("Error when reading history file of container " + containerId, e);
         throw;
     }
     finally
     {
         hfReader.Close();
     }
 }
Example #20
0
        private static TimelineEntity CreateContainerEntity(ContainerId containerId)
        {
            TimelineEntity entity = new TimelineEntity();

            entity.SetEntityType(ContainerMetricsConstants.EntityType);
            entity.SetEntityId(containerId.ToString());
            entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId
                                    .GetApplicationAttemptId().ToString());
            return(entity);
        }
Example #21
0
 /// <summary>Create an event to record the start of an MR AppMaster</summary>
 /// <param name="appAttemptId">the application attempt id.</param>
 /// <param name="startTime">the start time of the AM.</param>
 /// <param name="containerId">the containerId of the AM.</param>
 /// <param name="nodeManagerHost">the node on which the AM is running.</param>
 /// <param name="nodeManagerPort">the port on which the AM is running.</param>
 /// <param name="nodeManagerHttpPort">the httpPort for the node running the AM.</param>
 /// <param name="forcedJobStateOnShutDown">the state to force the job into</param>
 public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime, ContainerId
                       containerId, string nodeManagerHost, int nodeManagerPort, int nodeManagerHttpPort
                       , string forcedJobStateOnShutDown, long submitTime)
 {
     datum.applicationAttemptId    = new Utf8(appAttemptId.ToString());
     datum.startTime               = startTime;
     datum.containerId             = new Utf8(containerId.ToString());
     datum.nodeManagerHost         = new Utf8(nodeManagerHost);
     datum.nodeManagerPort         = nodeManagerPort;
     datum.nodeManagerHttpPort     = nodeManagerHttpPort;
     this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
     this.submitTime               = submitTime;
 }
Example #22
0
        /// <exception cref="System.IO.IOException"/>
        public override void StoreContainerKilled(ContainerId containerId)
        {
            string key = ContainersKeyPrefix + containerId.ToString() + ContainerKilledKeySuffix;

            try
            {
                db.Put(JniDBFactory.Bytes(key), EmptyValue);
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
        }
Example #23
0
        /// <exception cref="System.IO.IOException"/>
        public override void StoreContainerDiagnostics(ContainerId containerId, StringBuilder
                                                       diagnostics)
        {
            string key = ContainersKeyPrefix + containerId.ToString() + ContainerDiagsKeySuffix;

            try
            {
                db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(diagnostics.ToString()));
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
        }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual void TestContainerLogFile()
        {
            FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
                                              ).GetAbsoluteFile();
            string        logdirwithFile = absLogDir.ToURI().ToString();
            Configuration conf           = new Configuration();

            conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
            conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
            LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();

            dirsHandler.Init(conf);
            NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
                                                                        , new ApplicationACLsManager(conf), new NMNullStateStoreService());
            // Add an application and the corresponding containers
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);

            nmContext.GetApplications()[appId] = app;
            MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                        conf, user, appId, 1);

            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[containerId] = container;
            FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
                                                        (appId.ToString(), containerId.ToString()));

            containerLogDir.Mkdirs();
            string   fileName         = "fileName";
            FilePath containerLogFile = new FilePath(containerLogDir, fileName);

            containerLogFile.CreateNewFile();
            FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
                                                                   , nmContext);

            NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
                                            .ToString());
            FileUtil.FullyDelete(absLogDir);
        }
Example #25
0
        /// <exception cref="System.IO.IOException"/>
        public override void StoreContainerCompleted(ContainerId containerId, int exitCode
                                                     )
        {
            string key = ContainersKeyPrefix + containerId.ToString() + ContainerExitCodeKeySuffix;

            try
            {
                db.Put(JniDBFactory.Bytes(key), JniDBFactory.Bytes(Sharpen.Extensions.ToString(exitCode
                                                                                               )));
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
        }
Example #26
0
        /// <exception cref="System.IO.IOException"/>
        public override void StoreContainer(ContainerId containerId, StartContainerRequest
                                            startRequest)
        {
            string key = ContainersKeyPrefix + containerId.ToString() + ContainerRequestKeySuffix;

            try
            {
                db.Put(JniDBFactory.Bytes(key), ((StartContainerRequestPBImpl)startRequest).GetProto
                           ().ToByteArray());
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
        }
Example #27
0
        /// <exception cref="System.Exception"/>
        private static void CreateContainerLogInLocalDir(Path appLogsDir, ContainerId containerId
                                                         , FileSystem fs)
        {
            Path containerLogsDir = new Path(appLogsDir, containerId.ToString());

            if (fs.Exists(containerLogsDir))
            {
                fs.Delete(containerLogsDir, true);
            }
            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(containerLogsDir));
            TextWriter writer = new FileWriter(new FilePath(containerLogsDir.ToString(), "sysout"
                                                            ));

            writer.Write("Hello " + containerId + "!");
            writer.Close();
        }
Example #28
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        internal static IList <FilePath> GetContainerLogDirs(ContainerId containerId, LocalDirsHandlerService
                                                             dirsHandler)
        {
            IList <string>   logDirs          = dirsHandler.GetLogDirsForRead();
            IList <FilePath> containerLogDirs = new AList <FilePath>(logDirs.Count);

            foreach (string logDir in logDirs)
            {
                logDir = new FilePath(logDir).ToURI().GetPath();
                string appIdStr = ConverterUtils.ToString(containerId.GetApplicationAttemptId().GetApplicationId
                                                              ());
                FilePath appLogDir = new FilePath(logDir, appIdStr);
                containerLogDirs.AddItem(new FilePath(appLogDir, containerId.ToString()));
            }
            return(containerLogDirs);
        }
Example #29
0
        public virtual string GetResourcesOption(ContainerId containerId)
        {
            string        containerName = containerId.ToString();
            StringBuilder sb            = new StringBuilder("cgroups=");

            if (IsCpuWeightEnabled())
            {
                sb.Append(PathForCgroup(ControllerCpu, containerName) + "/tasks");
                sb.Append(",");
            }
            if (sb[sb.Length - 1] == ',')
            {
                Sharpen.Runtime.DeleteCharAt(sb, sb.Length - 1);
            }
            return(sb.ToString());
        }
        public virtual void TestLogFileWithDriveLetter()
        {
            ContainerImpl       container = Org.Mockito.Mockito.Mock <ContainerImpl>();
            ApplicationIdPBImpl appId     = Org.Mockito.Mockito.Mock <ApplicationIdPBImpl>();

            Org.Mockito.Mockito.When(appId.ToString()).ThenReturn("appId");
            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptIdPBImpl appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptIdPBImpl
                                                                                >();

            Org.Mockito.Mockito.When(appAttemptId.GetApplicationId()).ThenReturn(appId);
            ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                           > applications = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                                   >();

            applications[appId] = app;
            ContainerId containerId = Org.Mockito.Mockito.Mock <ContainerIdPBImpl>();

            Org.Mockito.Mockito.When(containerId.ToString()).ThenReturn("containerId");
            Org.Mockito.Mockito.When(containerId.GetApplicationAttemptId()).ThenReturn(appAttemptId
                                                                                       );
            ConcurrentMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                           > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                                                 >();

            containers[containerId] = container;
            LocalDirsHandlerService localDirs = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
                                                                          >();

            Org.Mockito.Mockito.When(localDirs.GetLogPathToRead("appId" + Path.Separator + "containerId"
                                                                + Path.Separator + "fileName")).ThenReturn(new Path("F:/nmlogs/appId/containerId/fileName"
                                                                                                                    ));
            NodeManager.NMContext context = Org.Mockito.Mockito.Mock <NodeManager.NMContext>();
            Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(localDirs);
            Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(applications);
            Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
            FilePath logFile = ContainerLogsUtils.GetContainerLogFile(containerId, "fileName"
                                                                      , null, context);

            NUnit.Framework.Assert.IsTrue("logFile lost drive letter " + logFile, logFile.ToString
                                              ().IndexOf("F:" + FilePath.separator + "nmlogs") > -1);
        }