Esempio n. 1
0
 public _NodeStatusUpdaterImpl_121(BaseContainerManagerTest _enclosing, Context baseArg1
                                   , Dispatcher baseArg2, NodeHealthCheckerService baseArg3, NodeManagerMetrics baseArg4
                                   )
     : base(baseArg1, baseArg2, baseArg3, baseArg4)
 {
     this._enclosing = _enclosing;
 }
        public virtual void TestLocalFilesCleanup()
        {
            // Real del service
            delSrvc = new DeletionService(exec);
            delSrvc.Init(conf);
            containerManager = CreateContainerManager(delSrvc);
            containerManager.Init(conf);
            containerManager.Start();
            // ////// Create the resources for the container
            FilePath dir = new FilePath(tmpDir, "dir");

            dir.Mkdirs();
            FilePath    file       = new FilePath(dir, "file");
            PrintWriter fileWriter = new PrintWriter(file);

            fileWriter.Write("Hello World!");
            fileWriter.Close();
            // ////// Construct the Container-id
            ContainerId   cId   = CreateContainerId(0);
            ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
            // ////// Construct the container-spec.
            ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                             >();
            //    containerLaunchContext.resources =
            //        new HashMap<CharSequence, LocalResource>();
            URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext
                                                                       ().MakeQualified(new Path(file.GetAbsolutePath())));
            LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();

            rsrc_alpha.SetResource(resource_alpha);
            rsrc_alpha.SetSize(-1);
            rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
            rsrc_alpha.SetType(LocalResourceType.File);
            rsrc_alpha.SetTimestamp(file.LastModified());
            string destinationFile = "dest_file";
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            localResources[destinationFile] = rsrc_alpha;
            containerLaunchContext.SetLocalResources(localResources);
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
                                                                                                       .GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
                                                           .Complete);
            BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
                                                                 ().GetApplicationId(), ApplicationState.Running);
            // Now ascertain that the resources are localised correctly.
            string   appIDStr       = ConverterUtils.ToString(appId);
            string   containerIDStr = ConverterUtils.ToString(cId);
            FilePath userCacheDir   = new FilePath(localDir, ContainerLocalizer.Usercache);
            FilePath userDir        = new FilePath(userCacheDir, user);
            FilePath appCache       = new FilePath(userDir, ContainerLocalizer.Appcache);
            FilePath appDir         = new FilePath(appCache, appIDStr);
            FilePath containerDir   = new FilePath(appDir, containerIDStr);
            FilePath targetFile     = new FilePath(containerDir, destinationFile);
            FilePath sysDir         = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
                                                   );
            FilePath appSysDir       = new FilePath(sysDir, appIDStr);
            FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);

            // AppDir should still exist
            NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!"
                                          , appDir.Exists());
            NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!"
                                          , appSysDir.Exists());
            foreach (FilePath f in new FilePath[] { containerDir, containerSysDir })
            {
                NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists());
            }
            NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
                                           .Exists());
            // Simulate RM sending an AppFinish event.
            containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId
                                                                             [] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown));
            BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
                                                                 ().GetApplicationId(), ApplicationState.Finished);
            // Now ascertain that the resources are localised correctly.
            foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir })
            {
                // Wait for deletion. Deletion can happen long after AppFinish because of
                // the async DeletionService
                int timeout = 0;
                while (f_1.Exists() && timeout++ < 15)
                {
                    Sharpen.Thread.Sleep(1000);
                }
                NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists());
            }
            // Wait for deletion
            int timeout_1 = 0;

            while (targetFile.Exists() && timeout_1++ < 15)
            {
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
                                           .Exists());
        }
        public virtual void TestContainerLaunchAndStop()
        {
            containerManager.Start();
            FilePath    scriptFile       = Shell.AppendScriptExtension(tmpDir, "scriptFile");
            PrintWriter fileWriter       = new PrintWriter(scriptFile);
            FilePath    processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
                                               ();
            // ////// Construct the Container-id
            ContainerId cId = CreateContainerId(0);

            if (Shell.Windows)
            {
                fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
                fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
                fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul");
            }
            else
            {
                fileWriter.Write("\numask 0");
                // So that start file is readable by the test
                fileWriter.Write("\necho Hello World! > " + processStartFile);
                fileWriter.Write("\necho $$ >> " + processStartFile);
                fileWriter.Write("\nexec sleep 100");
            }
            fileWriter.Close();
            ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                             >();
            URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
                                                                                         Path(scriptFile.GetAbsolutePath())));
            LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();

            rsrc_alpha.SetResource(resource_alpha);
            rsrc_alpha.SetSize(-1);
            rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
            rsrc_alpha.SetType(LocalResourceType.File);
            rsrc_alpha.SetTimestamp(scriptFile.LastModified());
            string destinationFile = "dest_file";
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            localResources[destinationFile] = rsrc_alpha;
            containerLaunchContext.SetLocalResources(localResources);
            IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));

            containerLaunchContext.SetCommands(commands);
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
                                                                                                       .GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            int timeoutSecs = 0;

            while (!processStartFile.Exists() && timeoutSecs++ < 20)
            {
                Sharpen.Thread.Sleep(1000);
                Log.Info("Waiting for process start-file to be created");
            }
            NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
                                          .Exists());
            // Now verify the contents of the file
            BufferedReader reader = new BufferedReader(new FileReader(processStartFile));

            NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
            // Get the pid of the process
            string pid = reader.ReadLine().Trim();

            // No more lines
            NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
            // Now test the stop functionality.
            // Assert that the process is alive
            NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
                                              (pid));
            // Once more
            NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
                                              (pid));
            IList <ContainerId> containerIds = new AList <ContainerId>();

            containerIds.AddItem(cId);
            StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
                                                                                  );

            containerManager.StopContainers(stopRequest);
            BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
                                                           .Complete);
            GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
                containerIds);
            ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
                                                                                    ).GetContainerStatuses()[0];
            int expectedExitCode = ContainerExitStatus.KilledByAppmaster;

            NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus()
                                            );
            // Assert that the process is not alive anymore
            NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
                                           .ContainerIsAlive(pid));
        }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        private void TestContainerLaunchAndExit(int exitCode)
        {
            FilePath    scriptFile       = Shell.AppendScriptExtension(tmpDir, "scriptFile");
            PrintWriter fileWriter       = new PrintWriter(scriptFile);
            FilePath    processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
                                               ();
            // ////// Construct the Container-id
            ContainerId cId = CreateContainerId(0);

            if (Shell.Windows)
            {
                fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
                fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
                if (exitCode != 0)
                {
                    fileWriter.WriteLine("@exit " + exitCode);
                }
            }
            else
            {
                fileWriter.Write("\numask 0");
                // So that start file is readable by the test
                fileWriter.Write("\necho Hello World! > " + processStartFile);
                fileWriter.Write("\necho $$ >> " + processStartFile);
                // Have script throw an exit code at the end
                if (exitCode != 0)
                {
                    fileWriter.Write("\nexit " + exitCode);
                }
            }
            fileWriter.Close();
            ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                             >();
            URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
                                                                                         Path(scriptFile.GetAbsolutePath())));
            LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();

            rsrc_alpha.SetResource(resource_alpha);
            rsrc_alpha.SetSize(-1);
            rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
            rsrc_alpha.SetType(LocalResourceType.File);
            rsrc_alpha.SetTimestamp(scriptFile.LastModified());
            string destinationFile = "dest_file";
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            localResources[destinationFile] = rsrc_alpha;
            containerLaunchContext.SetLocalResources(localResources);
            IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));

            containerLaunchContext.SetCommands(commands);
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
                                                                                                       .GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
                                                           .Complete);
            IList <ContainerId> containerIds = new AList <ContainerId>();

            containerIds.AddItem(cId);
            GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
                containerIds);
            ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
                                                                                    ).GetContainerStatuses()[0];

            // Verify exit status matches exit state of script
            NUnit.Framework.Assert.AreEqual(exitCode, containerStatus.GetExitStatus());
        }
        public virtual void TestContainerSetup()
        {
            containerManager.Start();
            // ////// Create the resources for the container
            FilePath dir = new FilePath(tmpDir, "dir");

            dir.Mkdirs();
            FilePath    file       = new FilePath(dir, "file");
            PrintWriter fileWriter = new PrintWriter(file);

            fileWriter.Write("Hello World!");
            fileWriter.Close();
            // ////// Construct the Container-id
            ContainerId cId = CreateContainerId(0);
            // ////// Construct the container-spec.
            ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                             >();
            URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
                                                                                         Path(file.GetAbsolutePath())));
            LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();

            rsrc_alpha.SetResource(resource_alpha);
            rsrc_alpha.SetSize(-1);
            rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
            rsrc_alpha.SetType(LocalResourceType.File);
            rsrc_alpha.SetTimestamp(file.LastModified());
            string destinationFile = "dest_file";
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            localResources[destinationFile] = rsrc_alpha;
            containerLaunchContext.SetLocalResources(localResources);
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                , CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
                                                                                                       .GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
                                                           .Complete);
            // Now ascertain that the resources are localised correctly.
            ApplicationId appId          = cId.GetApplicationAttemptId().GetApplicationId();
            string        appIDStr       = ConverterUtils.ToString(appId);
            string        containerIDStr = ConverterUtils.ToString(cId);
            FilePath      userCacheDir   = new FilePath(localDir, ContainerLocalizer.Usercache);
            FilePath      userDir        = new FilePath(userCacheDir, user);
            FilePath      appCache       = new FilePath(userDir, ContainerLocalizer.Appcache);
            FilePath      appDir         = new FilePath(appCache, appIDStr);
            FilePath      containerDir   = new FilePath(appDir, containerIDStr);
            FilePath      targetFile     = new FilePath(containerDir, destinationFile);
            FilePath      sysDir         = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
                                                        );
            FilePath appSysDir       = new FilePath(sysDir, appIDStr);
            FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);

            foreach (FilePath f in new FilePath[] { localDir, sysDir, userCacheDir, appDir, appSysDir
                                                    , containerDir, containerSysDir })
            {
                NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " doesn't exist!!", f.Exists(
                                                  ));
                NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " is not a directory!!", f.IsDirectory
                                                  ());
            }
            NUnit.Framework.Assert.IsTrue(targetFile.GetAbsolutePath() + " doesn't exist!!",
                                          targetFile.Exists());
            // Now verify the contents of the file
            BufferedReader reader = new BufferedReader(new FileReader(targetFile));

            NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
            NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
        }