Esempio n. 1
0
        public virtual void TestBasic()
        {
            AppContext            mockContext   = Org.Mockito.Mockito.Mock <AppContext>();
            OutputCommitter       mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>();
            Clock                 mockClock     = Org.Mockito.Mockito.Mock <Clock>();
            CommitterEventHandler handler       = new CommitterEventHandler(mockContext, mockCommitter
                                                                            , new TestCommitterEventHandler.TestingRMHeartbeatHandler());
            YarnConfiguration conf = new YarnConfiguration();

            conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
            JobContext           mockJobContext = Org.Mockito.Mockito.Mock <JobContext>();
            ApplicationAttemptId attemptid      = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
                                                                                        );
            JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId
                                                                          ()));

            TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler
                                                                              ();
            Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
                                                                                    ());
            Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid
                                                                                       );
            Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler
                                                                               );
            Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock);
            handler.Init(conf);
            handler.Start();
            try
            {
                handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext));
                string user                          = UserGroupInformation.GetCurrentUser().GetShortUserName();
                Path   startCommitFile               = MRApps.GetStartJobCommitFile(conf, user, jobId);
                Path   endCommitSuccessFile          = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId);
                Path   endCommitFailureFile          = MRApps.GetEndJobCommitFailureFile(conf, user, jobId);
                Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent();
                NUnit.Framework.Assert.IsNotNull(e);
                NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent);
                FileSystem fs = FileSystem.Get(conf);
                NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile
                                                                                    ));
                NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile
                                                                                         ));
                NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile
                                                                                          ));
                Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>());
            }
            finally
            {
                handler.Stop();
            }
        }
 /// <exception cref="System.Exception"/>
 protected override void ServiceInit(Configuration conf)
 {
     base.ServiceInit(conf);
     commitThreadCancelTimeoutMs = conf.GetInt(MRJobConfig.MrAmCommitterCancelTimeoutMs
                                               , MRJobConfig.DefaultMrAmCommitterCancelTimeoutMs);
     commitWindowMs = conf.GetLong(MRJobConfig.MrAmCommitWindowMs, MRJobConfig.DefaultMrAmCommitWindowMs
                                   );
     try
     {
         fs = FileSystem.Get(conf);
         JobID  id    = TypeConverter.FromYarn(context.GetApplicationID());
         JobId  jobId = TypeConverter.ToYarn(id);
         string user  = UserGroupInformation.GetCurrentUser().GetShortUserName();
         startCommitFile      = MRApps.GetStartJobCommitFile(conf, user, jobId);
         endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId);
         endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId);
     }
     catch (IOException e)
     {
         throw new YarnRuntimeException(e);
     }
 }
Esempio n. 3
0
		public virtual void TestMRAppMasterFailLock()
		{
			string applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
			string containerIdStr = "container_1317529182569_0004_000002_1";
			string userName = "******";
			JobConf conf = new JobConf();
			conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
			ApplicationAttemptId applicationAttemptId = ConverterUtils.ToApplicationAttemptId
				(applicationAttemptIdStr);
			JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId
				()));
			Path start = MRApps.GetStartJobCommitFile(conf, userName, jobId);
			Path end = MRApps.GetEndJobCommitFailureFile(conf, userName, jobId);
			FileSystem fs = FileSystem.Get(conf);
			fs.Create(start).Close();
			fs.Create(end).Close();
			ContainerId containerId = ConverterUtils.ToContainerId(containerIdStr);
			MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host"
				, -1, -1, Runtime.CurrentTimeMillis(), false, false);
			bool caught = false;
			try
			{
				MRAppMaster.InitAndStartAppMaster(appMaster, conf, userName);
			}
			catch (IOException e)
			{
				//The IO Exception is expected
				Log.Info("Caught expected Exception", e);
				caught = true;
			}
			NUnit.Framework.Assert.IsTrue(caught);
			NUnit.Framework.Assert.IsTrue(appMaster.errorHappenedShutDown);
			NUnit.Framework.Assert.AreEqual(JobStateInternal.Failed, appMaster.forcedState);
			appMaster.Stop();
			// verify the final status is FAILED
			VerifyFailedStatus((MRAppMasterTest)appMaster, "FAILED");
		}