public virtual void TestDeletionofStaging()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
NUnit.Framework.Assert.IsTrue(MRJobConfig.DefaultMrAmMaxAttempts > 1);
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
, JobStateInternal.Running, MRJobConfig.DefaultMrAmMaxAttempts);
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
//test whether notifyIsLastAMRetry called
NUnit.Framework.Assert.AreEqual(true, ((TestStagingCleanup.TestMRApp)appMaster).GetTestIsLastAMRetry
());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
// FIXME:
// Disabled this test because currently, when shutdown hook triggered at
// lastRetry in RM view, cleanup will not do. This should be supported after
// YARN-2261 completed
// @Test (timeout = 30000)
/// <exception cref="System.IO.IOException"/>
public virtual void TestDeletionofStagingOnKillLastTry()
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 0);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
ContainerAllocator mockAlloc = Org.Mockito.Mockito.Mock <ContainerAllocator>();
MRAppMaster appMaster = new TestStagingCleanup.TestMRApp(this, attemptId, mockAlloc
);
//no retry
appMaster.Init(conf);
NUnit.Framework.Assert.IsTrue("appMaster.isLastAMRetry() is false", appMaster.IsLastAMRetry
());
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook
(appMaster);
hook.Run();
NUnit.Framework.Assert.IsTrue("MRAppMaster isn't stopped", appMaster.IsInState(Service.STATE
.Stopped));
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
public static JobId NewJobId(ApplicationId appId, int id)
{
JobId jobId = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobId>();
jobId.SetAppId(appId);
jobId.SetId(id);
return(jobId);
}
public virtual void TestJobIDtoString()
{
JobId jid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId>
();
jid.SetAppId(ApplicationId.NewInstance(0, 0));
NUnit.Framework.Assert.AreEqual("job_0_0000", MRApps.ToString(jid));
}
public static JobId NewJobID(ApplicationId appID, int i)
{
JobId id = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobId>();
id.SetAppId(appID);
id.SetId(i);
return(id);
}
public static JobId ToYarn(JobID id)
{
JobId jobId = recordFactory.NewRecordInstance <JobId>();
jobId.SetId(id.GetId());
//currently there is 1-1 mapping between appid and jobid
ApplicationId appId = ApplicationId.NewInstance(ToClusterTimeStamp(id.GetJtIdentifier
()), id.GetId());
jobId.SetAppId(appId);
return(jobId);
}
public virtual void TestFromYarnJobReport()
{
int jobStartTime = 612354;
int jobFinishTime = 612355;
JobState state = JobState.Running;
JobId jobId = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobId>();
JobReport jobReport = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobReport>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
jobId.SetAppId(applicationId);
jobId.SetId(0);
jobReport.SetJobId(jobId);
jobReport.SetJobState(state);
jobReport.SetStartTime(jobStartTime);
jobReport.SetFinishTime(jobFinishTime);
jobReport.SetUser("TestTypeConverter-user");
JobStatus jobStatus = TypeConverter.FromYarn(jobReport, "dummy-jobfile");
NUnit.Framework.Assert.AreEqual(jobStartTime, jobStatus.GetStartTime());
NUnit.Framework.Assert.AreEqual(jobFinishTime, jobStatus.GetFinishTime());
NUnit.Framework.Assert.AreEqual(state.ToString(), jobStatus.GetState().ToString()
);
}
public virtual void Setup()
{
dispatcher = new InlineDispatcher();
++startCount;
conf = new JobConf();
taskAttemptListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>();
jobToken = (Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>)Org.Mockito.Mockito.Mock
<Org.Apache.Hadoop.Security.Token.Token>();
remoteJobConfFile = Org.Mockito.Mockito.Mock <Path>();
credentials = null;
clock = new SystemClock();
metrics = Org.Mockito.Mockito.Mock <MRAppMetrics>();
dataLocations = new string[1];
appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
jobId = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <JobId>();
jobId.SetId(1);
jobId.SetAppId(appId);
appContext = Org.Mockito.Mockito.Mock <AppContext>();
taskSplitMetaInfo = Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>();
Org.Mockito.Mockito.When(taskSplitMetaInfo.GetLocations()).ThenReturn(dataLocations
);
taskAttempts = new AList <TestTaskImpl.MockTaskAttemptImpl>();
}
/// <exception cref="System.IO.IOException"/>
private void TestDeletionofStagingOnUnregistrationFailure(int maxAttempts, bool shouldHaveDeleted
)
{
conf.Set(MRJobConfig.MapreduceJobDir, stagingJobDir);
fs = Org.Mockito.Mockito.Mock <FileSystem>();
Org.Mockito.Mockito.When(fs.Delete(Matchers.Any <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
//Staging Dir exists
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path stagingDir = MRApps.GetStagingAreaDir(conf, user);
Org.Mockito.Mockito.When(fs.Exists(stagingDir)).ThenReturn(true);
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
JobId jobid = recordFactory.NewRecordInstance <JobId>();
jobid.SetAppId(appId);
TestStagingCleanup.TestMRApp appMaster = new TestStagingCleanup.TestMRApp(this, attemptId
, null, JobStateInternal.Running, maxAttempts);
appMaster.crushUnregistration = true;
appMaster.Init(conf);
appMaster.Start();
appMaster.ShutDownJob();
((MRAppMaster.RunningAppContext)appMaster.GetContext()).ResetIsLastAMRetry();
if (shouldHaveDeleted)
{
NUnit.Framework.Assert.AreEqual(true, appMaster.IsLastAMRetry());
Org.Mockito.Mockito.Verify(fs).Delete(stagingJobPath, true);
}
else
{
NUnit.Framework.Assert.AreEqual(false, appMaster.IsLastAMRetry());
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Delete(stagingJobPath
, true);
}
}