示例#1
0
        private void VerifyClientConnection()
        {
            int numRetries = 3;

            while (numRetries-- > 0)
            {
                Configuration conf   = new YarnConfiguration(this.conf);
                YarnClient    client = YarnClient.CreateYarnClient();
                client.Init(conf);
                client.Start();
                try
                {
                    client.GetApplications();
                    return;
                }
                catch (Exception e)
                {
                    Log.Error(e);
                }
                finally
                {
                    client.Stop();
                }
            }
            NUnit.Framework.Assert.Fail("Client couldn't connect to the Active RM");
        }
示例#2
0
        public virtual void TestGetClusterNodeLabelsWithLocalAccess()
        {
            YarnClient client = Org.Mockito.Mockito.Mock <YarnClient>();

            Org.Mockito.Mockito.When(client.GetClusterNodeLabels()).ThenReturn(ImmutableSet.Of
                                                                                   ("remote1", "remote2"));
            ClusterCLI cli = new ClusterCLI();

            cli.SetClient(client);
            cli.SetSysOutPrintStream(sysOut);
            cli.SetSysErrPrintStream(sysErr);
            ClusterCLI.localNodeLabelsManager = Org.Mockito.Mockito.Mock <CommonNodeLabelsManager
                                                                          >();
            Org.Mockito.Mockito.When(ClusterCLI.localNodeLabelsManager.GetClusterNodeLabels()
                                     ).ThenReturn(ImmutableSet.Of("local1", "local2"));
            int rc = cli.Run(new string[] { ClusterCLI.Cmd, "-" + ClusterCLI.ListLabelsCmd, "-"
                                            + ClusterCLI.DirectlyAccessNodeLabelStore });

            NUnit.Framework.Assert.AreEqual(0, rc);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintWriter           pw   = new PrintWriter(baos);

            // it should return local* instead of remote*
            pw.Write("Node Labels: local1,local2");
            pw.Close();
            Org.Mockito.Mockito.Verify(sysOut).WriteLine(baos.ToString("UTF-8"));
        }
示例#3
0
        public virtual void Initiate()
        {
            StartHACluster(1, true, false, false);
            Configuration conf = new YarnConfiguration(this.conf);

            client = CreateAndStartYarnClient(conf);
        }
示例#4
0
 public YarnCLI()
     : base(new YarnConfiguration())
 {
     client = YarnClient.CreateYarnClient();
     client.Init(GetConf());
     client.Start();
 }
示例#5
0
        /// <exception cref="System.Exception"/>
        public virtual void TestHelpMessage()
        {
            Configuration conf           = new YarnConfiguration();
            YarnClient    mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
            LogsCLI       dumper         = new TestLogsCLI.LogsCLIForTest(mockYarnClient);

            dumper.SetConf(conf);
            int exitCode = dumper.Run(new string[] {  });

            NUnit.Framework.Assert.IsTrue(exitCode == -1);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintWriter           pw   = new PrintWriter(baos);

            pw.WriteLine("Retrieve logs for completed YARN applications.");
            pw.WriteLine("usage: yarn logs -applicationId <application ID> [OPTIONS]");
            pw.WriteLine();
            pw.WriteLine("general options are:");
            pw.WriteLine(" -appOwner <Application Owner>   AppOwner (assumed to be current user if"
                         );
            pw.WriteLine("                                 not specified)");
            pw.WriteLine(" -containerId <Container ID>     ContainerId (must be specified if node"
                         );
            pw.WriteLine("                                 address is specified)");
            pw.WriteLine(" -help                           Displays help for all commands.");
            pw.WriteLine(" -nodeAddress <Node Address>     NodeAddress in the format nodename:port"
                         );
            pw.WriteLine("                                 (must be specified if container id is"
                         );
            pw.WriteLine("                                 specified)");
            pw.Close();
            string appReportStr = baos.ToString("UTF-8");

            NUnit.Framework.Assert.AreEqual(appReportStr, sysOutStream.ToString());
        }
示例#6
0
        protected internal virtual void VerifyClientConnection()
        {
            int numRetries = 3;

            while (numRetries-- > 0)
            {
                Configuration conf   = new YarnConfiguration(this.conf);
                YarnClient    client = CreateAndStartYarnClient(conf);
                try
                {
                    Sharpen.Thread.Sleep(100);
                    client.GetApplications();
                    return;
                }
                catch (Exception e)
                {
                    Log.Error(e.Message);
                }
                finally
                {
                    client.Stop();
                }
            }
            NUnit.Framework.Assert.Fail("Client couldn't connect to the Active RM");
        }
示例#7
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        private int VerifyApplicationState(ApplicationId appId)
        {
            YarnClient yarnClient = CreateYarnClient();

            try
            {
                ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
                switch (appReport.GetYarnApplicationState())
                {
                case YarnApplicationState.New:
                case YarnApplicationState.NewSaving:
                case YarnApplicationState.Submitted:
                {
                    return(-1);
                }

                case YarnApplicationState.Accepted:
                case YarnApplicationState.Running:
                case YarnApplicationState.Failed:
                case YarnApplicationState.Finished:
                case YarnApplicationState.Killed:
                default:
                {
                    break;
                }
                }
            }
            finally
            {
                yarnClient.Close();
            }
            return(0);
        }
示例#8
0
 /// <summary>
 /// Delegate responsible for communicating with the Resource Manager's
 /// <see cref="Org.Apache.Hadoop.Yarn.Api.ApplicationClientProtocol"/>
 /// .
 /// </summary>
 /// <param name="conf">the configuration object.</param>
 public ResourceMgrDelegate(YarnConfiguration conf)
     : base(typeof(Org.Apache.Hadoop.Mapred.ResourceMgrDelegate).FullName)
 {
     this.conf   = conf;
     this.client = YarnClient.CreateYarnClient();
     Init(conf);
     Start();
 }
示例#9
0
        protected internal virtual YarnClient CreateYarnClient()
        {
            YarnClient yarnClient = YarnClient.CreateYarnClient();

            yarnClient.Init(GetConf());
            yarnClient.Start();
            return(yarnClient);
        }
示例#10
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        private YarnClient CreateMockYarnClientUnknownApp()
        {
            YarnClient mockClient = Org.Mockito.Mockito.Mock <YarnClient>();

            Org.Mockito.Mockito.DoThrow(new YarnException("Unknown AppId")).When(mockClient).
            GetApplicationReport(Matchers.Any <ApplicationId>());
            return(mockClient);
        }
示例#11
0
 internal Client(string appMasterMainClass, Configuration conf)
 {
     this.conf = conf;
     this.appMasterMainClass = appMasterMainClass;
     yarnClient = YarnClient.CreateYarnClient();
     yarnClient.Init(conf);
     opts = new Options();
     opts.AddOption("appname", true, "Application Name. Default value - DistributedShell"
                    );
     opts.AddOption("priority", true, "Application Priority. Default 0");
     opts.AddOption("queue", true, "RM Queue in which this application is to be submitted"
                    );
     opts.AddOption("timeout", true, "Application timeout in milliseconds");
     opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"
                    );
     opts.AddOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"
                    );
     opts.AddOption("jar", true, "Jar file containing the application master");
     opts.AddOption("shell_command", true, "Shell command to be executed by " + "the Application Master. Can only specify either --shell_command "
                    + "or --shell_script");
     opts.AddOption("shell_script", true, "Location of the shell script to be " + "executed. Can only specify either --shell_command or --shell_script"
                    );
     opts.AddOption("shell_args", true, "Command line args for the shell script." + "Multiple args can be separated by empty space."
                    );
     opts.GetOption("shell_args").SetArgs(Option.UnlimitedValues);
     opts.AddOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"
                    );
     opts.AddOption("shell_cmd_priority", true, "Priority for the shell command containers"
                    );
     opts.AddOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"
                    );
     opts.AddOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command"
                    );
     opts.AddOption("num_containers", true, "No. of containers on which the shell command needs to be executed"
                    );
     opts.AddOption("log_properties", true, "log4j.properties file");
     opts.AddOption("keep_containers_across_application_attempts", false, "Flag to indicate whether to keep containers across application attempts."
                    + " If the flag is true, running containers will not be killed when" + " application attempt fails and these containers will be retrieved by"
                    + " the new application attempt ");
     opts.AddOption("attempt_failures_validity_interval", true, "when attempt_failures_validity_interval in milliseconds is set to > 0,"
                    + "the failure number will not take failures which happen out of " + "the validityInterval into failure count. "
                    + "If failure count reaches to maxAppAttempts, " + "the application will be failed."
                    );
     opts.AddOption("debug", false, "Dump out debug information");
     opts.AddOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put"
                    );
     opts.AddOption("view_acls", true, "Users and groups that allowed to " + "view the timeline entities in the given domain"
                    );
     opts.AddOption("modify_acls", true, "Users and groups that allowed to " + "modify the timeline entities in the given domain"
                    );
     opts.AddOption("create", false, "Flag to indicate whether to create the " + "domain specified with -domain."
                    );
     opts.AddOption("help", false, "Print usage");
     opts.AddOption("node_label_expression", true, "Node label expression to determine the nodes"
                    + " where all the containers of this application" + " will be allocated, \"\" means containers"
                    + " can be allocated anywhere, if you don't specify the option," + " default node_label_expression of queue will be used."
                    );
 }
示例#12
0
        protected internal virtual YarnClient CreateAndStartYarnClient(Configuration conf
                                                                       )
        {
            Configuration configuration = new YarnConfiguration(conf);
            YarnClient    client        = YarnClient.CreateYarnClient();

            client.Init(configuration);
            client.Start();
            return(client);
        }
示例#13
0
        /// <summary>
        /// Creates/initializes/starts a RemoteAppChecker with a spied
        /// DummyYarnClientImpl.
        /// </summary>
        /// <returns>the spied DummyYarnClientImpl in the created AppChecker</returns>
        private YarnClient CreateCheckerWithMockedClient()
        {
            YarnClient client = Org.Mockito.Mockito.Spy(new TestRemoteAppChecker.DummyYarnClientImpl
                                                            (this));

            checker = new RemoteAppChecker(client);
            checker.Init(new Configuration());
            checker.Start();
            return(client);
        }
示例#14
0
        public virtual void TestRunningApp()
        {
            YarnClient    client = CreateCheckerWithMockedClient();
            ApplicationId id     = ApplicationId.NewInstance(1, 1);
            // create a report and set the state to an active one
            ApplicationReport report = new ApplicationReportPBImpl();

            report.SetYarnApplicationState(YarnApplicationState.Accepted);
            Org.Mockito.Mockito.DoReturn(report).When(client).GetApplicationReport(id);
            NUnit.Framework.Assert.IsTrue(checker.IsApplicationActive(id));
        }
示例#15
0
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        /// <exception cref="System.IO.IOException"/>
        private YarnClient CreateMockYarnClient(YarnApplicationState appState)
        {
            YarnClient        mockClient    = Org.Mockito.Mockito.Mock <YarnClient>();
            ApplicationReport mockAppReport = Org.Mockito.Mockito.Mock <ApplicationReport>();

            Org.Mockito.Mockito.DoReturn(appState).When(mockAppReport).GetYarnApplicationState
                ();
            Org.Mockito.Mockito.DoReturn(mockAppReport).When(mockClient).GetApplicationReport
                (Matchers.Any <ApplicationId>());
            return(mockClient);
        }
示例#16
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidApplicationId()
        {
            Configuration conf           = new YarnConfiguration();
            YarnClient    mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
            LogsCLI       cli            = new TestLogsCLI.LogsCLIForTest(mockYarnClient);

            cli.SetConf(conf);
            int exitCode = cli.Run(new string[] { "-applicationId", "not_an_app_id" });

            NUnit.Framework.Assert.IsTrue(exitCode == -1);
            NUnit.Framework.Assert.IsTrue(sysErrStream.ToString().StartsWith("Invalid ApplicationId specified"
                                                                             ));
        }
示例#17
0
        public virtual void TestNonExistentApp()
        {
            YarnClient    client = CreateCheckerWithMockedClient();
            ApplicationId id     = ApplicationId.NewInstance(1, 1);

            // test for null
            Org.Mockito.Mockito.DoReturn(null).When(client).GetApplicationReport(id);
            NUnit.Framework.Assert.IsFalse(checker.IsApplicationActive(id));
            // test for ApplicationNotFoundException
            Org.Mockito.Mockito.DoThrow(new ApplicationNotFoundException("Throw!")).When(client
                                                                                         ).GetApplicationReport(id);
            NUnit.Framework.Assert.IsFalse(checker.IsApplicationActive(id));
        }
示例#18
0
        /// <exception cref="System.Exception"/>
        public virtual void TestUnknownApplicationId()
        {
            Configuration conf           = new YarnConfiguration();
            YarnClient    mockYarnClient = CreateMockYarnClientUnknownApp();
            LogsCLI       cli            = new TestLogsCLI.LogsCLIForTest(mockYarnClient);

            cli.SetConf(conf);
            int exitCode = cli.Run(new string[] { "-applicationId", ApplicationId.NewInstance
                                                      (1, 1).ToString() });

            // Error since no logs present for the app.
            NUnit.Framework.Assert.IsTrue(exitCode != 0);
            NUnit.Framework.Assert.IsTrue(sysErrStream.ToString().StartsWith("Unable to get ApplicationState"
                                                                             ));
        }
示例#19
0
        /// <exception cref="Org.Apache.Commons.Cli.ParseException"/>
        public virtual bool Init(string[] args)
        {
            Options opts = new Options();

            opts.AddOption("appname", true, "Application Name. Default value - UnmanagedAM");
            opts.AddOption("priority", true, "Application Priority. Default 0");
            opts.AddOption("queue", true, "RM Queue in which this application is to be submitted"
                           );
            opts.AddOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"
                           );
            opts.AddOption("cmd", true, "command to start unmanaged AM (required)");
            opts.AddOption("classpath", true, "additional classpath");
            opts.AddOption("help", false, "Print usage");
            CommandLine cliParser = new GnuParser().Parse(opts, args);

            if (args.Length == 0)
            {
                PrintUsage(opts);
                throw new ArgumentException("No args specified for client to initialize");
            }
            if (cliParser.HasOption("help"))
            {
                PrintUsage(opts);
                return(false);
            }
            appName    = cliParser.GetOptionValue("appname", "UnmanagedAM");
            amPriority = System.Convert.ToInt32(cliParser.GetOptionValue("priority", "0"));
            amQueue    = cliParser.GetOptionValue("queue", "default");
            classpath  = cliParser.GetOptionValue("classpath", null);
            amCmd      = cliParser.GetOptionValue("cmd");
            if (amCmd == null)
            {
                PrintUsage(opts);
                throw new ArgumentException("No cmd specified for application master");
            }
            YarnConfiguration yarnConf = new YarnConfiguration(conf);

            rmClient = YarnClient.CreateYarnClient();
            rmClient.Init(yarnConf);
            return(true);
        }
示例#20
0
        /// <exception cref="System.Exception"/>
        public virtual void TestFailResultCodes()
        {
            Configuration conf = new YarnConfiguration();

            conf.SetClass("fs.file.impl", typeof(LocalFileSystem), typeof(FileSystem));
            LogCLIHelpers cliHelper = new LogCLIHelpers();

            cliHelper.SetConf(conf);
            YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
            LogsCLI    dumper         = new TestLogsCLI.LogsCLIForTest(mockYarnClient);

            dumper.SetConf(conf);
            // verify dumping a non-existent application's logs returns a failure code
            int exitCode = dumper.Run(new string[] { "-applicationId", "application_0_0" });

            NUnit.Framework.Assert.IsTrue("Should return an error code", exitCode != 0);
            // verify dumping a non-existent container log is a failure code
            exitCode = cliHelper.DumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234"
                                                     , "nobody");
            NUnit.Framework.Assert.IsTrue("Should return an error code", exitCode != 0);
        }
示例#21
0
        public virtual void TestGetEmptyClusterNodeLabels()
        {
            YarnClient client = Org.Mockito.Mockito.Mock <YarnClient>();

            Org.Mockito.Mockito.When(client.GetClusterNodeLabels()).ThenReturn(new HashSet <string
                                                                                            >());
            ClusterCLI cli = new ClusterCLI();

            cli.SetClient(client);
            cli.SetSysOutPrintStream(sysOut);
            cli.SetSysErrPrintStream(sysErr);
            int rc = cli.Run(new string[] { ClusterCLI.Cmd, "-" + ClusterCLI.ListLabelsCmd });

            NUnit.Framework.Assert.AreEqual(0, rc);
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            PrintWriter           pw   = new PrintWriter(baos);

            pw.Write("Node Labels: ");
            pw.Close();
            Org.Mockito.Mockito.Verify(sysOut).WriteLine(baos.ToString("UTF-8"));
        }
示例#22
0
 public virtual void SetClient(YarnClient client)
 {
     this.client = client;
 }
示例#23
0
        public virtual void Setup()
        {
            // start minicluster
            conf        = new YarnConfiguration();
            yarnCluster = new MiniYARNCluster(typeof(TestAMRMClient).FullName, nodeCount, 1,
                                              1);
            yarnCluster.Init(conf);
            yarnCluster.Start();
            NUnit.Framework.Assert.IsNotNull(yarnCluster);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnCluster.GetServiceState
                                                ());
            // start rm client
            yarnClient = (YarnClientImpl)YarnClient.CreateYarnClient();
            yarnClient.Init(conf);
            yarnClient.Start();
            NUnit.Framework.Assert.IsNotNull(yarnClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, yarnClient.GetServiceState
                                                ());
            // get node info
            nodeReports = yarnClient.GetNodeReports(NodeState.Running);
            // submit new app
            ApplicationSubmissionContext appContext = yarnClient.CreateApplication().GetApplicationSubmissionContext
                                                          ();
            ApplicationId appId = appContext.GetApplicationId();

            // set the application name
            appContext.SetApplicationName("Test");
            // Set the priority for the application master
            Priority pri = Priority.NewInstance(0);

            appContext.SetPriority(pri);
            // Set the queue to which this application is to be submitted in the RM
            appContext.SetQueue("default");
            // Set up the container launch context for the application master
            ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                 <ContainerLaunchContext>();

            appContext.SetAMContainerSpec(amContainer);
            // unmanaged AM
            appContext.SetUnmanagedAM(true);
            // Create the request to send to the applications manager
            SubmitApplicationRequest appRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                  <SubmitApplicationRequest>();

            appRequest.SetApplicationSubmissionContext(appContext);
            // Submit the application to the applications manager
            yarnClient.SubmitApplication(appContext);
            // wait for app to start
            int          iterationsLeft = 30;
            RMAppAttempt appAttempt     = null;

            while (iterationsLeft > 0)
            {
                ApplicationReport appReport = yarnClient.GetApplicationReport(appId);
                if (appReport.GetYarnApplicationState() == YarnApplicationState.Accepted)
                {
                    attemptId  = appReport.GetCurrentApplicationAttemptId();
                    appAttempt = yarnCluster.GetResourceManager().GetRMContext().GetRMApps()[attemptId
                                                                                             .GetApplicationId()].GetCurrentAppAttempt();
                    while (true)
                    {
                        if (appAttempt.GetAppAttemptState() == RMAppAttemptState.Launched)
                        {
                            break;
                        }
                    }
                    break;
                }
                Sleep(1000);
                --iterationsLeft;
            }
            if (iterationsLeft == 0)
            {
                NUnit.Framework.Assert.Fail("Application hasn't bee started");
            }
            // Just dig into the ResourceManager and get the AMRMToken just for the sake
            // of testing.
            UserGroupInformation.SetLoginUser(UserGroupInformation.CreateRemoteUser(UserGroupInformation
                                                                                    .GetCurrentUser().GetUserName()));
            UserGroupInformation.GetCurrentUser().AddToken(appAttempt.GetAMRMToken());
            //creating an instance NMTokenCase
            nmTokenCache = new NMTokenCache();
            // start am rm client
            rmClient = (AMRMClientImpl <AMRMClient.ContainerRequest>)AMRMClient.CreateAMRMClient
                       <AMRMClient.ContainerRequest>();
            //setting an instance NMTokenCase
            rmClient.SetNMTokenCache(nmTokenCache);
            rmClient.Init(conf);
            rmClient.Start();
            NUnit.Framework.Assert.IsNotNull(rmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, rmClient.GetServiceState()
                                            );
            // start am nm client
            nmClient = (NMClientImpl)NMClient.CreateNMClient();
            //propagating the AMRMClient NMTokenCache instance
            nmClient.SetNMTokenCache(rmClient.GetNMTokenCache());
            nmClient.Init(conf);
            nmClient.Start();
            NUnit.Framework.Assert.IsNotNull(nmClient);
            NUnit.Framework.Assert.AreEqual(Service.STATE.Started, nmClient.GetServiceState()
                                            );
        }
示例#24
0
 public LogsCLIForTest(YarnClient yarnClient)
     : base()
 {
     this.yarnClient = yarnClient;
 }
示例#25
0
 public RemoteAppChecker()
     : this(YarnClient.CreateYarnClient())
 {
 }
示例#26
0
        /// <exception cref="System.Exception"/>
        public virtual void TestFetchApplictionLogs()
        {
            string        remoteLogRootDir = "target/logs/";
            Configuration configuration    = new Configuration();

            configuration.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
            configuration.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogRootDir);
            configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
            configuration.Set(YarnConfiguration.YarnAdminAcl, "admin");
            FileSystem           fs           = FileSystem.Get(configuration);
            UserGroupInformation ugi          = UserGroupInformation.GetCurrentUser();
            ApplicationId        appId        = ApplicationIdPBImpl.NewInstance(0, 1);
            ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
                                                                                       1);
            ContainerId containerId0 = ContainerIdPBImpl.NewContainerId(appAttemptId, 0);
            ContainerId containerId1 = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
            ContainerId containerId2 = ContainerIdPBImpl.NewContainerId(appAttemptId, 2);
            NodeId      nodeId       = NodeId.NewInstance("localhost", 1234);
            // create local logs
            string rootLogDir     = "target/LocalLogs";
            Path   rootLogDirPath = new Path(rootLogDir);

            if (fs.Exists(rootLogDirPath))
            {
                fs.Delete(rootLogDirPath, true);
            }
            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(rootLogDirPath));
            Path appLogsDir = new Path(rootLogDirPath, appId.ToString());

            if (fs.Exists(appLogsDir))
            {
                fs.Delete(appLogsDir, true);
            }
            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(appLogsDir));
            IList <string> rootLogDirs = Arrays.AsList(rootLogDir);

            // create container logs in localLogDir
            CreateContainerLogInLocalDir(appLogsDir, containerId1, fs);
            CreateContainerLogInLocalDir(appLogsDir, containerId2, fs);
            Path path = new Path(remoteLogRootDir + ugi.GetShortUserName() + "/logs/application_0_0001"
                                 );

            if (fs.Exists(path))
            {
                fs.Delete(path, true);
            }
            NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
            // upload container logs into remote directory
            // the first two logs is empty. When we try to read first two logs,
            // we will meet EOF exception, but it will not impact other logs.
            // Other logs should be read successfully.
            UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0
                                                 , path, fs);
            UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
                                                 , path, fs);
            UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
                                            , path, fs);
            UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2
                                            , path, fs);
            YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
            LogsCLI    cli            = new TestLogsCLI.LogsCLIForTest(mockYarnClient);

            cli.SetConf(configuration);
            int exitCode = cli.Run(new string[] { "-applicationId", appId.ToString() });

            NUnit.Framework.Assert.IsTrue(exitCode == 0);
            NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
                                                                           ));
            NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000002!"
                                                                           ));
            sysOutStream.Reset();
            // uploaded two logs for container1. The first log is empty.
            // The second one is not empty.
            // We can still successfully read logs for container1.
            exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
                                              , nodeId.ToString(), "-containerId", containerId1.ToString() });
            NUnit.Framework.Assert.IsTrue(exitCode == 0);
            NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
                                                                           ));
            NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Log Upload Time")
                                          );
            NUnit.Framework.Assert.IsTrue(!sysOutStream.ToString().Contains("Logs for container "
                                                                            + containerId1.ToString() + " are not present in this log-file."));
            sysOutStream.Reset();
            // Uploaded the empty log for container0.
            // We should see the message showing the log for container0
            // are not present.
            exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
                                              , nodeId.ToString(), "-containerId", containerId0.ToString() });
            NUnit.Framework.Assert.IsTrue(exitCode == -1);
            NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Logs for container "
                                                                           + containerId0.ToString() + " are not present in this log-file."));
            fs.Delete(new Path(remoteLogRootDir), true);
            fs.Delete(new Path(rootLogDir), true);
        }
示例#27
0
 internal RemoteAppChecker(YarnClient client)
     : base("RemoteAppChecker")
 {
     this.client = client;
 }
示例#28
0
        /// <exception cref="System.Exception"/>
        public virtual void TestDSShell(bool haveDomain)
        {
            string[] args = new string[] { "--jar", AppmasterJar, "--num_containers", "2", "--shell_command"
                                           , Shell.Windows ? "dir" : "ls", "--master_memory", "512", "--master_vcores", "2"
                                           , "--container_memory", "128", "--container_vcores", "1" };
            if (haveDomain)
            {
                string[] domainArgs = new string[] { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group"
                                                     , "--modify_acls", "writer_user writer_group", "--create" };
                IList <string> argsList = new AList <string>(Arrays.AsList(args));
                Sharpen.Collections.AddAll(argsList, Arrays.AsList(domainArgs));
                args = Sharpen.Collections.ToArray(argsList, new string[argsList.Count]);
            }
            Log.Info("Initializing DS Client");
            Client client      = new Client(new Configuration(yarnCluster.GetConfig()));
            bool   initSuccess = client.Init(args);

            NUnit.Framework.Assert.IsTrue(initSuccess);
            Log.Info("Running DS Client");
            AtomicBoolean result = new AtomicBoolean(false);

            Sharpen.Thread t = new _Thread_194(result, client);
            t.Start();
            YarnClient yarnClient = YarnClient.CreateYarnClient();

            yarnClient.Init(new Configuration(yarnCluster.GetConfig()));
            yarnClient.Start();
            string hostName     = NetUtils.GetHostname();
            bool   verified     = false;
            string errorMessage = string.Empty;

            while (!verified)
            {
                IList <ApplicationReport> apps = yarnClient.GetApplications();
                if (apps.Count == 0)
                {
                    Sharpen.Thread.Sleep(10);
                    continue;
                }
                ApplicationReport appReport = apps[0];
                if (appReport.GetHost().Equals("N/A"))
                {
                    Sharpen.Thread.Sleep(10);
                    continue;
                }
                errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport
                               .GetHost() + "'. Expected rpc port to be '-1', was '" + appReport.GetRpcPort() +
                               "'.";
                if (CheckHostname(appReport.GetHost()) && appReport.GetRpcPort() == -1)
                {
                    verified = true;
                }
                if (appReport.GetYarnApplicationState() == YarnApplicationState.Finished)
                {
                    break;
                }
            }
            NUnit.Framework.Assert.IsTrue(errorMessage, verified);
            t.Join();
            Log.Info("Client run completed. Result=" + result);
            NUnit.Framework.Assert.IsTrue(result.Get());
            TimelineDomain domain = null;

            if (haveDomain)
            {
                domain = yarnCluster.GetApplicationHistoryServer().GetTimelineStore().GetDomain("TEST_DOMAIN"
                                                                                                );
                NUnit.Framework.Assert.IsNotNull(domain);
                NUnit.Framework.Assert.AreEqual("reader_user reader_group", domain.GetReaders());
                NUnit.Framework.Assert.AreEqual("writer_user writer_group", domain.GetWriters());
            }
            TimelineEntities entitiesAttempts = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
                                                    ().GetEntities(ApplicationMaster.DSEntity.DsAppAttempt.ToString(), null, null, null
                                                                   , null, null, null, null, null, null);

            NUnit.Framework.Assert.IsNotNull(entitiesAttempts);
            NUnit.Framework.Assert.AreEqual(1, entitiesAttempts.GetEntities().Count);
            NUnit.Framework.Assert.AreEqual(2, entitiesAttempts.GetEntities()[0].GetEvents().
                                            Count);
            NUnit.Framework.Assert.AreEqual(entitiesAttempts.GetEntities()[0].GetEntityType()
                                            .ToString(), ApplicationMaster.DSEntity.DsAppAttempt.ToString());
            if (haveDomain)
            {
                NUnit.Framework.Assert.AreEqual(domain.GetId(), entitiesAttempts.GetEntities()[0]
                                                .GetDomainId());
            }
            else
            {
                NUnit.Framework.Assert.AreEqual("DEFAULT", entitiesAttempts.GetEntities()[0].GetDomainId
                                                    ());
            }
            TimelineEntities entities = yarnCluster.GetApplicationHistoryServer().GetTimelineStore
                                            ().GetEntities(ApplicationMaster.DSEntity.DsContainer.ToString(), null, null, null
                                                           , null, null, null, null, null, null);

            NUnit.Framework.Assert.IsNotNull(entities);
            NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
            NUnit.Framework.Assert.AreEqual(entities.GetEntities()[0].GetEntityType().ToString
                                                (), ApplicationMaster.DSEntity.DsContainer.ToString());
            if (haveDomain)
            {
                NUnit.Framework.Assert.AreEqual(domain.GetId(), entities.GetEntities()[0].GetDomainId
                                                    ());
            }
            else
            {
                NUnit.Framework.Assert.AreEqual("DEFAULT", entities.GetEntities()[0].GetDomainId(
                                                    ));
            }
        }