/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
if (conf.Get(MRJobConfig.MrAmStagingDir) == null)
{
conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(GetTestWorkDir(), "apps_staging_dir/"
).GetAbsolutePath());
}
// By default, VMEM monitoring disabled, PMEM monitoring enabled.
if (!conf.GetBoolean(MRConfig.MapreduceMiniclusterControlResourceMonitoring, MRConfig
.DefaultMapreduceMiniclusterControlResourceMonitoring))
{
conf.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false);
conf.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false);
}
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "000");
try
{
Path stagingPath = FileContext.GetFileContext(conf).MakeQualified(new Path(conf.Get
(MRJobConfig.MrAmStagingDir)));
/*
* Re-configure the staging path on Windows if the file system is localFs.
* We need to use a absolute path that contains the drive letter. The unit
* test could run on a different drive than the AM. We can run into the
* issue that job files are localized to the drive where the test runs on,
* while the AM starts on a different drive and fails to find the job
* metafiles. Using absolute path can avoid this ambiguity.
*/
if (Path.Windows)
{
if (typeof(LocalFileSystem).IsInstanceOfType(stagingPath.GetFileSystem(conf)))
{
conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(conf.Get(MRJobConfig.MrAmStagingDir
)).GetAbsolutePath());
}
}
FileContext fc = FileContext.GetFileContext(stagingPath.ToUri(), conf);
if (fc.Util().Exists(stagingPath))
{
Log.Info(stagingPath + " exists! deleting...");
fc.Delete(stagingPath, true);
}
Log.Info("mkdir: " + stagingPath);
//mkdir the staging directory so that right permissions are set while running as proxy user
fc.Mkdir(stagingPath, null, true);
//mkdir done directory as well
string doneDir = JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf);
Path doneDirPath = fc.MakeQualified(new Path(doneDir));
fc.Mkdir(doneDirPath, null, true);
}
catch (IOException e)
{
throw new YarnRuntimeException("Could not create staging directory. ", e);
}
conf.Set(MRConfig.MasterAddress, "test");
// The default is local because of
// which shuffle doesn't happen
//configure the shuffle service in NM
conf.SetStrings(YarnConfiguration.NmAuxServices, new string[] { ShuffleHandler.MapreduceShuffleServiceid });
conf.SetClass(string.Format(YarnConfiguration.NmAuxServiceFmt, ShuffleHandler.MapreduceShuffleServiceid
), typeof(ShuffleHandler), typeof(Org.Apache.Hadoop.Service.Service));
// Non-standard shuffle port
conf.SetInt(ShuffleHandler.ShufflePortConfigKey, 0);
conf.SetClass(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor
), typeof(ContainerExecutor));
// TestMRJobs is for testing non-uberized operation only; see TestUberAM
// for corresponding uberized tests.
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
base.ServiceInit(conf);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override string GetJobHistoryDir()
{
return(JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf));
}