public virtual void TestContainerLocalizerClosesFilesystems()
{
// mocked generics
// verify filesystems are closed when localizer doesn't fail
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
// verify filesystems are closed when localizer fails
localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoThrow(new YarnRuntimeException("Forced Failure")).When(localizer
).LocalizeFiles(Matchers.Any <LocalizationProtocol>(), Matchers.Any <CompletionService
>(), Matchers.Any <UserGroupInformation>());
Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems
(Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation
>());
}
public virtual void TestFcResolveAfs()
{
Configuration conf = new Configuration();
FileContext fcLocal = FileContext.GetLocalFSFileContext();
FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
string localTestRoot = helper.GetAbsoluteTestRootDir(fcLocal);
Path alphaLocalPath = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString()
, new FilePath(localTestRoot, "alpha").GetAbsolutePath());
DFSTestUtil.CreateFile(FileSystem.GetLocal(conf), alphaLocalPath, 16, (short)1, 2
);
Path linkTarget = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString(), localTestRoot
);
Path hdfsLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString(), "/tmp/link"
);
fcHdfs.CreateSymlink(linkTarget, hdfsLink, true);
Path alphaHdfsPathViaLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString
() + "/tmp/link/alpha");
ICollection <AbstractFileSystem> afsList = fcHdfs.ResolveAbstractFileSystems(alphaHdfsPathViaLink
);
NUnit.Framework.Assert.AreEqual(2, afsList.Count);
foreach (AbstractFileSystem afs in afsList)
{
if ((!afs.Equals(fcHdfs.GetDefaultFileSystem())) && (!afs.Equals(fcLocal.GetDefaultFileSystem
())))
{
NUnit.Framework.Assert.Fail("Failed to resolve AFS correctly");
}
}
}
public virtual void TestGetMountPoints()
{
ViewFs viewfs = (ViewFs)fcView.GetDefaultFileSystem();
ViewFs.MountPoint[] mountPoints = viewfs.GetMountPoints();
Assert.Equal(8, mountPoints.Length);
}
/// <summary>
/// Get default file system URI for the cluster (used to ensure consistency
/// of history done/staging locations) over different context
/// </summary>
/// <returns>Default file context</returns>
private static FileContext GetDefaultFileContext()
{
// If FS_DEFAULT_NAME_KEY was set solely by core-default.xml then we ignore
// ignore it. This prevents defaulting history paths to file system specified
// by core-default.xml which would not make sense in any case. For a test
// case to exploit this functionality it should create core-site.xml
FileContext fc = null;
Configuration defaultConf = new Configuration();
string[] sources;
sources = defaultConf.GetPropertySources(CommonConfigurationKeysPublic.FsDefaultNameKey
);
if (sources != null && (!Arrays.AsList(sources).Contains("core-default.xml") || sources
.Length > 1))
{
try
{
fc = FileContext.GetFileContext(defaultConf);
Log.Info("Default file system [" + fc.GetDefaultFileSystem().GetUri() + "]");
}
catch (UnsupportedFileSystemException e)
{
Log.Error("Unable to create default file context [" + defaultConf.Get(CommonConfigurationKeysPublic
.FsDefaultNameKey) + "]", e);
}
}
else
{
Log.Info("Default file system is set solely " + "by core-default.xml therefore - ignoring"
);
}
return(fc);
}
// nothing to do or verify here
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint
nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler
)
{
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
CreateUserLocalDirs(localDirs, user);
CreateUserCacheDirs(localDirs, user);
CreateAppDirs(localDirs, user, appId);
CreateAppLogDirs(appId, logDirs, user);
// randomly choose the local directory
Path appStorageDir = GetWorkingDir(localDirs, user, appId);
string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
CopyFile(nmPrivateContainerTokensPath, tokenDst, user);
Log.Info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
FileContext localizerFc = FileContext.GetFileContext(lfs.GetDefaultFileSystem(),
GetConf());
localizerFc.SetUMask(lfs.GetUMask());
localizerFc.SetWorkingDirectory(appStorageDir);
Log.Info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.GetWorkingDirectory
());
ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId
, GetPaths(localDirs), RecordFactoryProvider.GetRecordFactory(GetConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.RunLocalization(nmAddr);
}
public virtual void TestFcDelegationToken()
{
FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
AbstractFileSystem afs = fcHdfs.GetDefaultFileSystem();
IList <Org.Apache.Hadoop.Security.Token.Token <object> > tokenList = afs.GetDelegationTokens
(UserGroupInformation.GetCurrentUser().GetUserName());
((Org.Apache.Hadoop.FS.Hdfs)afs).RenewDelegationToken((Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>)tokenList[0]);
((Org.Apache.Hadoop.FS.Hdfs)afs).CancelDelegationToken((Org.Apache.Hadoop.Security.Token.Token
<AbstractDelegationTokenIdentifier>)tokenList[0]);
}
public virtual void TestBasicPaths()
{
URI uri = fc.GetDefaultFileSystem().GetUri();
Assert.Equal(chrootedTo.ToUri(), uri);
Assert.Equal(fc.MakeQualified(new Path(Runtime.GetProperty("user.home"
))), fc.GetWorkingDirectory());
Assert.Equal(fc.MakeQualified(new Path(Runtime.GetProperty("user.home"
))), fc.GetHomeDirectory());
/*
* ChRootedFs as its uri like file:///chrootRoot.
* This is questionable since path.makequalified(uri, path) ignores
* the pathPart of a uri. So our notion of chrooted URI is questionable.
* But if we were to fix Path#makeQualified() then the next test should
* have been:
*
* Assert.assertEquals(
* new Path(chrootedTo + "/foo/bar").makeQualified(
* FsConstants.LOCAL_FS_URI, null),
* fc.makeQualified(new Path( "/foo/bar")));
*/
Assert.Equal(new Path("/foo/bar").MakeQualified(FsConstants.LocalFsUri
, null), fc.MakeQualified(new Path("/foo/bar")));
}
public virtual void TestLocalizerTokenIsGettingRemoved()
{
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol
>(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>());
localizer.RunLocalization(nmAddr);
Org.Mockito.Mockito.Verify(spylfs, Org.Mockito.Mockito.Times(1)).Delete(tokenPath
, false);
}
/// <summary>
/// Ensure that path belongs to cluster's default file system unless
/// 1.
/// </summary>
/// <remarks>
/// Ensure that path belongs to cluster's default file system unless
/// 1. it is already fully qualified.
/// 2. current job configuration uses default file system
/// 3. running from a test case without core-site.xml
/// </remarks>
/// <param name="sourcePath">source path</param>
/// <param name="conf">the job configuration</param>
/// <returns>full qualified path (if necessary) in default file system</returns>
private static string EnsurePathInDefaultFileSystem(string sourcePath, Configuration
conf)
{
Path path = new Path(sourcePath);
FileContext fc = GetDefaultFileContext();
if (fc == null || fc.GetDefaultFileSystem().GetUri().ToString().Equals(conf.Get(CommonConfigurationKeysPublic
.FsDefaultNameKey, string.Empty)) || path.ToUri().GetAuthority() != null || path
.ToUri().GetScheme() != null)
{
return(sourcePath);
}
return(fc.MakeQualified(path).ToString());
}
public virtual void SetUp()
{
// create the test root on local_fs
fcTarget = FileContext.GetLocalFSFileContext();
chrootedTo = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget);
// In case previous test was killed before cleanup
fcTarget.Delete(chrootedTo, true);
fcTarget.Mkdir(chrootedTo, FileContext.DefaultPerm, true);
Configuration conf = new Configuration();
// ChRoot to the root of the testDirectory
fc = FileContext.GetFileContext(new ChRootedFs(fcTarget.GetDefaultFileSystem(), chrootedTo
), conf);
}
/// <exception cref="System.IO.IOException"/>
private LocalResource CreateApplicationResource(FileContext fs, Path p, LocalResourceType
type)
{
LocalResource rsrc = recordFactory.NewRecordInstance <LocalResource>();
FileStatus rsrcStat = fs.GetFileStatus(p);
rsrc.SetResource(ConverterUtils.GetYarnUrlFromPath(fs.GetDefaultFileSystem().ResolvePath
(rsrcStat.GetPath())));
rsrc.SetSize(rsrcStat.GetLen());
rsrc.SetTimestamp(rsrcStat.GetModificationTime());
rsrc.SetType(type);
rsrc.SetVisibility(LocalResourceVisibility.Application);
return(rsrc);
}
public virtual void TestFileContextDoesntDnsResolveLogicalURI()
{
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
NameService spyNS = SpyOnNameService();
string logicalHost = fs.GetUri().GetHost();
Configuration haClientConf = fs.GetConf();
FileContext fc = FileContext.GetFileContext(haClientConf);
Path root = new Path("/");
fc.ListStatus(root);
fc.ListStatus(fc.MakeQualified(root));
fc.GetDefaultFileSystem().GetCanonicalServiceName();
// Ensure that the logical hostname was never resolved.
Org.Mockito.Mockito.Verify(spyNS, Org.Mockito.Mockito.Never()).LookupAllHostAddr(
Org.Mockito.Mockito.Eq(logicalHost));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestAccessLinkFromAbstractFileSystem()
{
Path file = new Path(TestBaseDir1(), "file");
Path link = new Path(TestBaseDir1(), "linkToFile");
CreateAndWriteFile(file);
wrapper.CreateSymlink(file, link, false);
try
{
AbstractFileSystem afs = fc.GetDefaultFileSystem();
afs.Open(link);
NUnit.Framework.Assert.Fail("Opened a link using AFS");
}
catch (UnresolvedLinkException)
{
}
}
public Path GetDefaultWorkingDirectory(FileContext fc)
{
return(GetTestRootPath(fc, "/user/" + Runtime.GetProperty("user.name")).MakeQualified
(fc.GetDefaultFileSystem().GetUri(), fc.GetWorkingDirectory()));
}
public virtual void TestContainerLocalizerMain()
{
FileContext fs = FileContext.GetLocalFSFileContext();
spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem());
ContainerLocalizer localizer = SetupContainerLocalizerForTest();
// verify created cache
IList <Path> privCacheList = new AList <Path>();
IList <Path> appCacheList = new AList <Path>();
foreach (Path p in localDirs)
{
Path @base = new Path(new Path(p, ContainerLocalizer.Usercache), appUser);
Path privcache = new Path(@base, ContainerLocalizer.Filecache);
privCacheList.AddItem(privcache);
Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId));
Path appcache = new Path(appDir, ContainerLocalizer.Filecache);
appCacheList.AddItem(appcache);
}
// mock heartbeat responses from NM
ResourceLocalizationSpec rsrcA = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
ResourceLocalizationSpec rsrcB = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
ResourceLocalizationSpec rsrcC = GetMockRsrc(random, LocalResourceVisibility.Application
, appCacheList[0]);
ResourceLocalizationSpec rsrcD = GetMockRsrc(random, LocalResourceVisibility.Private
, privCacheList[0]);
Org.Mockito.Mockito.When(nmProxy.Heartbeat(Matchers.IsA <LocalizerStatus>())).ThenReturn
(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.SingletonList
(rsrcA))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections
.SingletonList(rsrcB))).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction
.Live, Sharpen.Collections.SingletonList(rsrcC))).ThenReturn(new MockLocalizerHeartbeatResponse
(LocalizerAction.Live, Sharpen.Collections.SingletonList(rsrcD))).ThenReturn(new
MockLocalizerHeartbeatResponse(LocalizerAction.Live, Sharpen.Collections.EmptyList
<ResourceLocalizationSpec>())).ThenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction
.Die, null));
LocalResource tRsrcA = rsrcA.GetResource();
LocalResource tRsrcB = rsrcB.GetResource();
LocalResource tRsrcC = rsrcC.GetResource();
LocalResource tRsrcD = rsrcD.GetResource();
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcA.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcA), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcB.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcB), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcC.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcC), Matchers.IsA <UserGroupInformation>());
Org.Mockito.Mockito.DoReturn(new TestContainerLocalizer.FakeDownload(rsrcD.GetResource
().GetResource().GetFile(), true)).When(localizer).Download(Matchers.IsA <Path>()
, Matchers.Eq(tRsrcD), Matchers.IsA <UserGroupInformation>());
// run localization
NUnit.Framework.Assert.AreEqual(0, localizer.RunLocalization(nmAddr));
foreach (Path p_1 in localDirs)
{
Path @base = new Path(new Path(p_1, ContainerLocalizer.Usercache), appUser);
Path privcache = new Path(@base, ContainerLocalizer.Filecache);
// $x/usercache/$user/filecache
Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(privcache), Matchers.Eq(CacheDirPerm
), Matchers.Eq(false));
Path appDir = new Path(@base, new Path(ContainerLocalizer.Appcache, appId));
// $x/usercache/$user/appcache/$appId/filecache
Path appcache = new Path(appDir, ContainerLocalizer.Filecache);
Org.Mockito.Mockito.Verify(spylfs).Mkdir(Matchers.Eq(appcache), Matchers.Eq(CacheDirPerm
), Matchers.Eq(false));
}
// verify tokens read at expected location
Org.Mockito.Mockito.Verify(spylfs).Open(tokenPath);
// verify downloaded resources reported to NM
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcA.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcB.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcC.GetResource())));
Org.Mockito.Mockito.Verify(nmProxy).Heartbeat(Matchers.ArgThat(new TestContainerLocalizer.HBMatches
(rsrcD.GetResource())));
// verify all HB use localizerID provided
Org.Mockito.Mockito.Verify(nmProxy, Org.Mockito.Mockito.Never()).Heartbeat(Matchers.ArgThat
(new _ArgumentMatcher_193()));
}
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration
jobConf, string jobSubmitDir, Credentials ts)
{
ApplicationId applicationId = resMgrDelegate.GetApplicationId();
// Setup resource requirements
Resource capability = recordFactory.NewRecordInstance <Resource>();
capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb
));
capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores
));
Log.Debug("AppMaster capability = " + capability);
// Setup LocalResources
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile);
URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext
.GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir
))));
Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir
);
localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext
, jobConfPath, LocalResourceType.File);
if (jobConf.Get(MRJobConfig.Jar) != null)
{
Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar));
LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath
.ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern);
string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault
).Pattern();
rc.SetPattern(pattern);
localResources[MRJobConfig.JobJar] = rc;
}
else
{
// Job jar may be null. For e.g, for pipes, the job jar is the hadoop
// mapreduce jar itself which is already on the classpath.
Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources."
);
}
// TODO gross hack
foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo })
{
localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext
, new Path(jobSubmitDir, s), LocalResourceType.File);
}
// Setup security tokens
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
// Setup the command to run the AM
IList <string> vargs = new AList <string>(8);
vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment
.JavaHome) + "/bin/java");
Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), YarnConfiguration.DefaultContainerTempDir);
vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir);
MRApps.AddLog4jSystemProperties(null, vargs, conf);
// Check for Java Lib Path usage in MAP and REDUCE configs
WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig
.MapJavaOpts, MRJobConfig.MapEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map"
, MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce",
MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty),
"reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig
.DefaultMrAmAdminCommandOpts);
WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts
, MRJobConfig.MrAmAdminUserEnv);
vargs.AddItem(mrAppMasterAdminOptions);
// Add AM user command opts
string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig
.DefaultMrAmCommandOpts);
WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts
, MRJobConfig.MrAmEnv);
vargs.AddItem(mrAppMasterUserOptions);
if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile))
{
string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams
);
if (profileParams != null)
{
vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar
+ Path.Separator + TaskLog.LogName.Profile));
}
}
vargs.AddItem(MRJobConfig.ApplicationMasterClass);
vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stdout);
vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stderr);
Vector <string> vargsFinal = new Vector <string>(8);
// Final command
StringBuilder mergedCommand = new StringBuilder();
foreach (CharSequence str in vargs)
{
mergedCommand.Append(str).Append(" ");
}
vargsFinal.AddItem(mergedCommand.ToString());
Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand
);
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, conf);
// Shell
environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig
.MapredAdminUserShell, MRJobConfig.DefaultShell);
// Add the container working directory at the front of LD_LIBRARY_PATH
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath
.ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), conf);
// Setup the environment variables for Admin first
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv),
conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf);
// Parse distributed cache
MRApps.SetupDistributedCache(jobConf, localResources);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig
.DefaultJobAclViewJob);
acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob,
MRJobConfig.DefaultJobAclModifyJob);
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
, environment, vargsFinal, null, securityTokens, acls);
ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig
.JobTags);
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
appContext.SetApplicationId(applicationId);
// ApplicationId
appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName
));
// Queue name
// add reservationID if present
ReservationId reservationID = null;
try
{
reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId
));
}
catch (FormatException)
{
// throw exception as reservationid as is invalid
string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId)
+ " specified for the app: " + applicationId;
Log.Warn(errMsg);
throw new IOException(errMsg);
}
if (reservationID != null)
{
appContext.SetReservationID(reservationID);
Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:"
+ appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID(
));
}
appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName
));
// Job name
appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken
, true));
appContext.SetAMContainerSpec(amContainer);
// AM Container
appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig
.DefaultMrAmMaxAttempts));
appContext.SetResource(capability);
appContext.SetApplicationType(MRJobConfig.MrApplicationType);
if (tagsFromConf != null && !tagsFromConf.IsEmpty())
{
appContext.SetApplicationTags(new HashSet <string>(tagsFromConf));
}
return(appContext);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
private void DownloadWithFileType(TestFSDownload.TEST_FILE_TYPE fileType)
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int size = rand.Next(512) + 512;
LocalResourceVisibility vis = LocalResourceVisibility.Private;
Path p = new Path(basedir, string.Empty + 1);
string strFileName = string.Empty;
LocalResource rsrc = null;
switch (fileType)
{
case TestFSDownload.TEST_FILE_TYPE.Tar:
{
rsrc = CreateTarFile(files, p, size, rand, vis);
break;
}
case TestFSDownload.TEST_FILE_TYPE.Jar:
{
rsrc = CreateJarFile(files, p, size, rand, vis);
rsrc.SetType(LocalResourceType.Pattern);
break;
}
case TestFSDownload.TEST_FILE_TYPE.Zip:
{
rsrc = CreateZipFile(files, p, size, rand, vis);
strFileName = p.GetName() + ".ZIP";
break;
}
case TestFSDownload.TEST_FILE_TYPE.Tgz:
{
rsrc = CreateTgzFile(files, p, size, rand, vis);
break;
}
}
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
try
{
pending[rsrc].Get();
// see if there was an Exception during download
FileStatus[] filesstatus = files.GetDefaultFileSystem().ListStatus(basedir);
foreach (FileStatus filestatus in filesstatus)
{
if (filestatus.IsDirectory())
{
FileStatus[] childFiles = files.GetDefaultFileSystem().ListStatus(filestatus.GetPath
());
foreach (FileStatus childfile in childFiles)
{
if (strFileName.EndsWith(".ZIP") && childfile.GetPath().GetName().Equals(strFileName
) && !childfile.IsDirectory())
{
NUnit.Framework.Assert.Fail("Failure...After unzip, there should have been a" + " directory formed with zip file name but found a file. "
+ childfile.GetPath());
}
if (childfile.GetPath().GetName().StartsWith("tmp"))
{
NUnit.Framework.Assert.Fail("Tmp File should not have been there " + childfile.GetPath
());
}
}
}
}
}
catch (Exception e)
{
throw new IOException("Failed exec", e);
}
}
