Exemplo n.º 1
0
        public virtual void TestShuffleProviders()
        {
            ApplicationId       appId        = ApplicationId.NewInstance(1, 1);
            JobId               jobId        = MRBuilderUtils.NewJobId(appId, 1);
            TaskId              taskId       = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
            Path                jobFile      = Org.Mockito.Mockito.Mock <Path>();
            EventHandler        eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
            TaskAttemptListener taListener   = Org.Mockito.Mockito.Mock <TaskAttemptListener>();

            Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
                                                                                        , 0));
            JobConf jobConf = new JobConf();

            jobConf.SetClass("fs.file.impl", typeof(TestShuffleProvider.StubbedFS), typeof(FileSystem
                                                                                           ));
            jobConf.SetBoolean("fs.file.impl.disable.cache", true);
            jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
            jobConf.Set(YarnConfiguration.NmAuxServices, TestShuffleProvider.TestShuffleHandler1
                        .MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
                        );
            string serviceName = TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid;
            string serviceStr  = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);

            jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler1).FullName);
            serviceName = TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid;
            serviceStr  = string.Format(YarnConfiguration.NmAuxServiceFmt, serviceName);
            jobConf.Set(serviceStr, typeof(TestShuffleProvider.TestShuffleHandler2).FullName);
            jobConf.Set(MRJobConfig.MapreduceJobShuffleProviderServices, TestShuffleProvider.TestShuffleHandler1
                        .MapreduceTestShuffleServiceid + "," + TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid
                        );
            Credentials credentials = new Credentials();

            Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
                                                                                   <JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
                                                                                                            (("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
            TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
                                                            1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
                                                            jobToken, credentials, new SystemClock(), null);

            jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
            ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(null
                                                                                            , jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
                                                                                            <WrappedJvmID>(), taListener, credentials);
            IDictionary <string, ByteBuffer> serviceDataMap = launchCtx.GetServiceData();

            NUnit.Framework.Assert.IsNotNull("TestShuffleHandler1 is missing", serviceDataMap
                                             [TestShuffleProvider.TestShuffleHandler1.MapreduceTestShuffleServiceid]);
            NUnit.Framework.Assert.IsNotNull("TestShuffleHandler2 is missing", serviceDataMap
                                             [TestShuffleProvider.TestShuffleHandler2.MapreduceTestShuffleServiceid]);
            NUnit.Framework.Assert.IsTrue("mismatch number of services in map", serviceDataMap
                                          .Count == 3);
        }
Exemplo n.º 2
0
        public virtual void TestContainerPassThrough()
        {
            MRApp app = new _MRApp_494(this, 0, 1, true, this.GetType().FullName, true);

            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration());
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            ICollection <Task>        tasks        = job.GetTasks().Values;
            ICollection <TaskAttempt> taskAttempts = tasks.GetEnumerator().Next().GetAttempts(
                ).Values;
            TaskAttemptImpl taskAttempt = (TaskAttemptImpl)taskAttempts.GetEnumerator().Next(
                );

            // Container from RM should pass through to the launcher. Container object
            // should be the same.
            NUnit.Framework.Assert.IsTrue(taskAttempt.container == containerObtainedByContainerLauncher
                                          );
        }
Exemplo n.º 3
0
        /// <exception cref="System.Exception"/>
        public virtual void WaitForInternalState(TaskAttemptImpl attempt, TaskAttemptStateInternal
                                                 finalState)
        {
            int timeoutSecs = 0;
            TaskAttemptReport        report = attempt.GetReport();
            TaskAttemptStateInternal iState = attempt.GetInternalState();

            while (!finalState.Equals(iState) && timeoutSecs++ < 20)
            {
                System.Console.Out.WriteLine("TaskAttempt Internal State is : " + iState + " Waiting for Internal state : "
                                             + finalState + "   progress : " + report.GetProgress());
                Sharpen.Thread.Sleep(500);
                report = attempt.GetReport();
                iState = attempt.GetInternalState();
            }
            System.Console.Out.WriteLine("TaskAttempt Internal State is : " + iState);
            NUnit.Framework.Assert.AreEqual("TaskAttempt Internal state is not correct (timedout)"
                                            , finalState, iState);
        }
Exemplo n.º 4
0
        public virtual void TestAttemptContainerRequest()
        {
            Text SecretKeyAlias = new Text("secretkeyalias");

            byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey"));
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(1);

            acls[ApplicationAccessType.ViewApp] = "otheruser";
            ApplicationId       appId        = ApplicationId.NewInstance(1, 1);
            JobId               jobId        = MRBuilderUtils.NewJobId(appId, 1);
            TaskId              taskId       = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map);
            Path                jobFile      = Org.Mockito.Mockito.Mock <Path>();
            EventHandler        eventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
            TaskAttemptListener taListener   = Org.Mockito.Mockito.Mock <TaskAttemptListener>();

            Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost"
                                                                                        , 0));
            JobConf jobConf = new JobConf();

            jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS
                                                    ), typeof(FileSystem));
            jobConf.SetBoolean("fs.file.impl.disable.cache", true);
            jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty);
            // setup UGI for security so tokens and keys are preserved
            jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
                        );
            UserGroupInformation.SetConfiguration(jobConf);
            Credentials credentials = new Credentials();

            credentials.AddSecretKey(SecretKeyAlias, SecretKey);
            Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token
                                                                                   <JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString
                                                                                                            (("tokenpw")), new Text("tokenkind"), new Text("tokenservice"));
            TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile,
                                                            1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener,
                                                            jobToken, credentials, new SystemClock(), null);

            jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString());
            ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls
                                                                                            , jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock
                                                                                            <WrappedJvmID>(), taListener, credentials);

            NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs
                                                ());
            Credentials         launchCredentials = new Credentials();
            DataInputByteBuffer dibb = new DataInputByteBuffer();

            dibb.Reset(launchCtx.GetTokens());
            launchCredentials.ReadTokenStorageStream(dibb);
            // verify all tokens specified for the task attempt are in the launch context
            foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials
                     .GetAllTokens())
            {
                Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials
                                                                                       .GetToken(token.GetService());
                NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken
                                                 );
                NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token
                                                , launchToken);
            }
            // verify the secret key is in the launch context
            NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey
                                                 (SecretKeyAlias));
            NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials
                                                                               .GetSecretKey(SecretKeyAlias)));
        }