/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
public static ResourceLocalizationSpec NewResourceLocalizationSpec(LocalResource
rsrc, Path path)
{
URL local = ConverterUtils.GetYarnUrlFromPath(path);
ResourceLocalizationSpec resourceLocalizationSpec = Org.Apache.Hadoop.Yarn.Util.Records
.NewRecord <ResourceLocalizationSpec>();
resourceLocalizationSpec.SetDestinationDirectory(local);
resourceLocalizationSpec.SetResource(rsrc);
return(resourceLocalizationSpec);
}
internal static LocalResource CreateResource()
{
LocalResource ret = recordFactory.NewRecordInstance <LocalResource>();
NUnit.Framework.Assert.IsTrue(ret is LocalResourcePBImpl);
ret.SetResource(ConverterUtils.GetYarnUrlFromPath(new Path("hdfs://y.ak:8020/foo/bar"
)));
ret.SetSize(4344L);
ret.SetTimestamp(3141592653589793L);
ret.SetVisibility(LocalResourceVisibility.Public);
return(ret);
}
/// <exception cref="System.IO.IOException"/>
private LocalResource CreateApplicationResource(FileContext fs, Path p, LocalResourceType
type)
{
LocalResource rsrc = recordFactory.NewRecordInstance <LocalResource>();
FileStatus rsrcStat = fs.GetFileStatus(p);
rsrc.SetResource(ConverterUtils.GetYarnUrlFromPath(fs.GetDefaultFileSystem().ResolvePath
(rsrcStat.GetPath())));
rsrc.SetSize(rsrcStat.GetLen());
rsrc.SetTimestamp(rsrcStat.GetModificationTime());
rsrc.SetType(type);
rsrc.SetVisibility(LocalResourceVisibility.Application);
return(rsrc);
}
/// <summary>Create the payload for the HeartBeat.</summary>
/// <remarks>
/// Create the payload for the HeartBeat. Mainly the list of
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalResourceStatus
/// "/>
/// es
/// </remarks>
/// <returns>
/// a
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Nodemanager.Api.Protocolrecords.LocalizerStatus
/// "/>
/// that can be sent via heartbeat.
/// </returns>
/// <exception cref="System.Exception"/>
private LocalizerStatus CreateStatus()
{
IList <LocalResourceStatus> currentResources = new AList <LocalResourceStatus>();
// TODO: Synchronization??
for (IEnumerator <LocalResource> i = pendingResources.Keys.GetEnumerator(); i.HasNext
();)
{
LocalResource rsrc = i.Next();
LocalResourceStatus stat = recordFactory.NewRecordInstance <LocalResourceStatus>();
stat.SetResource(rsrc);
Future <Path> fPath = pendingResources[rsrc];
if (fPath.IsDone())
{
try
{
Path localPath = fPath.Get();
stat.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(localPath));
stat.SetLocalSize(FileUtil.GetDU(new FilePath(localPath.GetParent().ToUri())));
stat.SetStatus(ResourceStatusType.FetchSuccess);
}
catch (ExecutionException e)
{
stat.SetStatus(ResourceStatusType.FetchFailure);
stat.SetException(SerializedException.NewInstance(e.InnerException));
}
catch (CancellationException e)
{
stat.SetStatus(ResourceStatusType.FetchFailure);
stat.SetException(SerializedException.NewInstance(e));
}
// TODO shouldn't remove until ACK
i.Remove();
}
else
{
stat.SetStatus(ResourceStatusType.FetchPending);
}
currentResources.AddItem(stat);
}
LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
status.SetLocalizerId(localizerId);
status.AddAllResources(currentResources);
return(status);
}
internal static LocalResourceStatus CreateLocalResourceStatus()
{
LocalResourceStatus ret = recordFactory.NewRecordInstance <LocalResourceStatus>();
NUnit.Framework.Assert.IsTrue(ret is LocalResourceStatusPBImpl);
ret.SetResource(CreateResource());
ret.SetLocalPath(ConverterUtils.GetYarnUrlFromPath(new Path("file:///local/foo/bar"
)));
ret.SetStatus(ResourceStatusType.FetchSuccess);
ret.SetLocalSize(4443L);
Exception e = new Exception("Dingos.");
e.SetStackTrace(new StackTraceElement[] { new StackTraceElement("foo", "bar", "baz"
, 10), new StackTraceElement("sbb", "one", "onm", 10) });
ret.SetException(SerializedException.NewInstance(e));
return(ret);
}
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalizerHeartbeatResponse CreateLocalizerHeartbeatResponse()
{
LocalizerHeartbeatResponse ret = recordFactory.NewRecordInstance <LocalizerHeartbeatResponse
>();
NUnit.Framework.Assert.IsTrue(ret is LocalizerHeartbeatResponsePBImpl);
ret.SetLocalizerAction(LocalizerAction.Live);
LocalResource rsrc = CreateResource();
AList <ResourceLocalizationSpec> rsrcs = new AList <ResourceLocalizationSpec>();
ResourceLocalizationSpec resource = recordFactory.NewRecordInstance <ResourceLocalizationSpec
>();
resource.SetResource(rsrc);
resource.SetDestinationDirectory(ConverterUtils.GetYarnUrlFromPath(new Path("/tmp"
+ Runtime.CurrentTimeMillis())));
rsrcs.AddItem(resource);
ret.SetResourceSpecs(rsrcs);
System.Console.Out.WriteLine(resource);
return(ret);
}
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility
vis, Path p)
{
ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec
>();
LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>();
string name = long.ToHexString(r.NextLong());
URL uri = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name);
Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri);
Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L);
Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L);
Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File);
Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn
(ConverterUtils.GetYarnUrlFromPath(p));
return(resourceLocalizationSpec);
}
public override URL GetResource()
{
return(ConverterUtils.GetYarnUrlFromPath(loc));
}
public virtual void TestLocalFilesCleanup()
{
// Real del service
delSrvc = new DeletionService(exec);
delSrvc.Init(conf);
containerManager = CreateContainerManager(delSrvc);
containerManager.Init(conf);
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// containerLaunchContext.resources =
// new HashMap<CharSequence, LocalResource>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext
().MakeQualified(new Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Running);
// Now ascertain that the resources are localised correctly.
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
// AppDir should still exist
NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!"
, appDir.Exists());
NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!"
, appSysDir.Exists());
foreach (FilePath f in new FilePath[] { containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists());
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
// Simulate RM sending an AppFinish event.
containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId
[] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown));
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Finished);
// Now ascertain that the resources are localised correctly.
foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir })
{
// Wait for deletion. Deletion can happen long after AppFinish because of
// the async DeletionService
int timeout = 0;
while (f_1.Exists() && timeout++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists());
}
// Wait for deletion
int timeout_1 = 0;
while (targetFile.Exists() && timeout_1++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerLaunchAndExit(int exitCode)
{
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
if (exitCode != 0)
{
fileWriter.WriteLine("@exit " + exitCode);
}
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
// Have script throw an exit code at the end
if (exitCode != 0)
{
fileWriter.Write("\nexit " + exitCode);
}
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
// Verify exit status matches exit state of script
NUnit.Framework.Assert.AreEqual(exitCode, containerStatus.GetExitStatus());
}
public virtual void TestContainerLaunchAndStop()
{
containerManager.Start();
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
fileWriter.WriteLine("@ping -n 100 127.0.0.1 >nul");
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nexec sleep 100");
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
// Now test the stop functionality.
// Assert that the process is alive
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
// Once more
NUnit.Framework.Assert.IsTrue("Process is not alive!", DefaultContainerExecutor.ContainerIsAlive
(pid));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
int expectedExitCode = ContainerExitStatus.KilledByAppmaster;
NUnit.Framework.Assert.AreEqual(expectedExitCode, containerStatus.GetExitStatus()
);
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
.ContainerIsAlive(pid));
}
public virtual void TestContainerSetup()
{
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
// Now ascertain that the resources are localised correctly.
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
foreach (FilePath f in new FilePath[] { localDir, sysDir, userCacheDir, appDir, appSysDir
, containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " doesn't exist!!", f.Exists(
));
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath() + " is not a directory!!", f.IsDirectory
());
}
NUnit.Framework.Assert.IsTrue(targetFile.GetAbsolutePath() + " doesn't exist!!",
targetFile.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(targetFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
}
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationSubmissionContext CreateApplicationSubmissionContext(Configuration
jobConf, string jobSubmitDir, Credentials ts)
{
ApplicationId applicationId = resMgrDelegate.GetApplicationId();
// Setup resource requirements
Resource capability = recordFactory.NewRecordInstance <Resource>();
capability.SetMemory(conf.GetInt(MRJobConfig.MrAmVmemMb, MRJobConfig.DefaultMrAmVmemMb
));
capability.SetVirtualCores(conf.GetInt(MRJobConfig.MrAmCpuVcores, MRJobConfig.DefaultMrAmCpuVcores
));
Log.Debug("AppMaster capability = " + capability);
// Setup LocalResources
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JobConfFile);
URL yarnUrlForJobSubmitDir = ConverterUtils.GetYarnUrlFromPath(defaultFileContext
.GetDefaultFileSystem().ResolvePath(defaultFileContext.MakeQualified(new Path(jobSubmitDir
))));
Log.Debug("Creating setup context, jobSubmitDir url is " + yarnUrlForJobSubmitDir
);
localResources[MRJobConfig.JobConfFile] = CreateApplicationResource(defaultFileContext
, jobConfPath, LocalResourceType.File);
if (jobConf.Get(MRJobConfig.Jar) != null)
{
Path jobJarPath = new Path(jobConf.Get(MRJobConfig.Jar));
LocalResource rc = CreateApplicationResource(FileContext.GetFileContext(jobJarPath
.ToUri(), jobConf), jobJarPath, LocalResourceType.Pattern);
string pattern = conf.GetPattern(JobContext.JarUnpackPattern, JobConf.UnpackJarPatternDefault
).Pattern();
rc.SetPattern(pattern);
localResources[MRJobConfig.JobJar] = rc;
}
else
{
// Job jar may be null. For e.g, for pipes, the job jar is the hadoop
// mapreduce jar itself which is already on the classpath.
Log.Info("Job jar is not present. " + "Not adding any jar to the list of resources."
);
}
// TODO gross hack
foreach (string s in new string[] { MRJobConfig.JobSplit, MRJobConfig.JobSplitMetainfo })
{
localResources[MRJobConfig.JobSubmitDir + "/" + s] = CreateApplicationResource(defaultFileContext
, new Path(jobSubmitDir, s), LocalResourceType.File);
}
// Setup security tokens
DataOutputBuffer dob = new DataOutputBuffer();
ts.WriteTokenStorageToStream(dob);
ByteBuffer securityTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
// Setup the command to run the AM
IList <string> vargs = new AList <string>(8);
vargs.AddItem(MRApps.CrossPlatformifyMREnv(jobConf, ApplicationConstants.Environment
.JavaHome) + "/bin/java");
Path amTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), YarnConfiguration.DefaultContainerTempDir);
vargs.AddItem("-Djava.io.tmpdir=" + amTmpDir);
MRApps.AddLog4jSystemProperties(null, vargs, conf);
// Check for Java Lib Path usage in MAP and REDUCE configs
WarnForJavaLibPath(conf.Get(MRJobConfig.MapJavaOpts, string.Empty), "map", MRJobConfig
.MapJavaOpts, MRJobConfig.MapEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredMapAdminJavaOpts, string.Empty), "map"
, MRJobConfig.MapredMapAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.ReduceJavaOpts, string.Empty), "reduce",
MRJobConfig.ReduceJavaOpts, MRJobConfig.ReduceEnv);
WarnForJavaLibPath(conf.Get(MRJobConfig.MapredReduceAdminJavaOpts, string.Empty),
"reduce", MRJobConfig.MapredReduceAdminJavaOpts, MRJobConfig.MapredAdminUserEnv);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
string mrAppMasterAdminOptions = conf.Get(MRJobConfig.MrAmAdminCommandOpts, MRJobConfig
.DefaultMrAmAdminCommandOpts);
WarnForJavaLibPath(mrAppMasterAdminOptions, "app master", MRJobConfig.MrAmAdminCommandOpts
, MRJobConfig.MrAmAdminUserEnv);
vargs.AddItem(mrAppMasterAdminOptions);
// Add AM user command opts
string mrAppMasterUserOptions = conf.Get(MRJobConfig.MrAmCommandOpts, MRJobConfig
.DefaultMrAmCommandOpts);
WarnForJavaLibPath(mrAppMasterUserOptions, "app master", MRJobConfig.MrAmCommandOpts
, MRJobConfig.MrAmEnv);
vargs.AddItem(mrAppMasterUserOptions);
if (jobConf.GetBoolean(MRJobConfig.MrAmProfile, MRJobConfig.DefaultMrAmProfile))
{
string profileParams = jobConf.Get(MRJobConfig.MrAmProfileParams, MRJobConfig.DefaultTaskProfileParams
);
if (profileParams != null)
{
vargs.AddItem(string.Format(profileParams, ApplicationConstants.LogDirExpansionVar
+ Path.Separator + TaskLog.LogName.Profile));
}
}
vargs.AddItem(MRJobConfig.ApplicationMasterClass);
vargs.AddItem("1>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stdout);
vargs.AddItem("2>" + ApplicationConstants.LogDirExpansionVar + Path.Separator + ApplicationConstants
.Stderr);
Vector <string> vargsFinal = new Vector <string>(8);
// Final command
StringBuilder mergedCommand = new StringBuilder();
foreach (CharSequence str in vargs)
{
mergedCommand.Append(str).Append(" ");
}
vargsFinal.AddItem(mergedCommand.ToString());
Log.Debug("Command to launch container for ApplicationMaster is : " + mergedCommand
);
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, conf);
// Shell
environment[ApplicationConstants.Environment.Shell.ToString()] = conf.Get(MRJobConfig
.MapredAdminUserShell, MRJobConfig.DefaultShell);
// Add the container working directory at the front of LD_LIBRARY_PATH
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.LdLibraryPath
.ToString(), MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), conf);
// Setup the environment variables for Admin first
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmAdminUserEnv),
conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MrAmEnv), conf);
// Parse distributed cache
MRApps.SetupDistributedCache(jobConf, localResources);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = jobConf.Get(MRJobConfig.JobAclViewJob, MRJobConfig
.DefaultJobAclViewJob);
acls[ApplicationAccessType.ModifyApp] = jobConf.Get(MRJobConfig.JobAclModifyJob,
MRJobConfig.DefaultJobAclModifyJob);
// Setup ContainerLaunchContext for AM container
ContainerLaunchContext amContainer = ContainerLaunchContext.NewInstance(localResources
, environment, vargsFinal, null, securityTokens, acls);
ICollection <string> tagsFromConf = jobConf.GetTrimmedStringCollection(MRJobConfig
.JobTags);
// Set up the ApplicationSubmissionContext
ApplicationSubmissionContext appContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
appContext.SetApplicationId(applicationId);
// ApplicationId
appContext.SetQueue(jobConf.Get(JobContext.QueueName, YarnConfiguration.DefaultQueueName
));
// Queue name
// add reservationID if present
ReservationId reservationID = null;
try
{
reservationID = ReservationId.ParseReservationId(jobConf.Get(JobContext.ReservationId
));
}
catch (FormatException)
{
// throw exception as reservationid as is invalid
string errMsg = "Invalid reservationId: " + jobConf.Get(JobContext.ReservationId)
+ " specified for the app: " + applicationId;
Log.Warn(errMsg);
throw new IOException(errMsg);
}
if (reservationID != null)
{
appContext.SetReservationID(reservationID);
Log.Info("SUBMITTING ApplicationSubmissionContext app:" + applicationId + " to queue:"
+ appContext.GetQueue() + " with reservationId:" + appContext.GetReservationID(
));
}
appContext.SetApplicationName(jobConf.Get(JobContext.JobName, YarnConfiguration.DefaultApplicationName
));
// Job name
appContext.SetCancelTokensWhenComplete(conf.GetBoolean(MRJobConfig.JobCancelDelegationToken
, true));
appContext.SetAMContainerSpec(amContainer);
// AM Container
appContext.SetMaxAppAttempts(conf.GetInt(MRJobConfig.MrAmMaxAttempts, MRJobConfig
.DefaultMrAmMaxAttempts));
appContext.SetResource(capability);
appContext.SetApplicationType(MRJobConfig.MrApplicationType);
if (tagsFromConf != null && !tagsFromConf.IsEmpty())
{
appContext.SetApplicationTags(new HashSet <string>(tagsFromConf));
}
return(appContext);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.Exception"/>
public virtual void TestClearLocalDirWhenNodeReboot()
{
nm = new TestNodeManagerReboot.MyNodeManager(this);
nm.Start();
ContainerManagementProtocol containerManager = nm.GetContainerManager();
// create files under fileCache
CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100);
localResourceDir.Mkdirs();
ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext
>();
// Construct the Container-id
ContainerId cId = CreateContainerId();
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(localResourceDir.GetAbsolutePath())));
LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType
.File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
containerLaunchContext.SetCommands(commands);
NodeId nodeId = nm.GetNMContext().GetNodeId();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm.
GetNMContext().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId
().ToString());
NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId
(), nodeId, user, 123);
currentUser.AddTokenIdentifier(nmIdentifier);
currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]];
int MaxTries = 20;
int numTries = 0;
while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <=
MaxTries)
{
try
{
Sharpen.Thread.Sleep(500);
}
catch (Exception)
{
}
// Do nothing
numTries++;
}
NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState(
));
NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(
), ContainerLocalizer.Usercache) > 0);
NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when "
+ "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService
.NmPrivateDir) > 0);
// restart the NodeManager
RestartNM(MaxTries);
CheckNumOfLocalDirs();
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ResourceLocalizationService.NmPrivateDir + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ContainerLocalizer.Filecache + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null
, new string[] { destinationFile })));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer
.Usercache + "_DEL_", new string[] { })));
// restart the NodeManager again
// this time usercache directory should be empty
RestartNM(MaxTries);
CheckNumOfLocalDirs();
}
public virtual void TestContainerKillOnMemoryOverflow()
{
if (!ProcfsBasedProcessTree.IsAvailable())
{
return;
}
containerManager.Start();
FilePath scriptFile = new FilePath(tmpDir, "scriptFile.sh");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
fileWriter.Write("\numask 0");
// So that start file is readable by the
// test.
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
fileWriter.Write("\nsleep 15");
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cId = ContainerId.NewContainerId(appAttemptId, 0);
int port = 12345;
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
commands.AddItem("/bin/bash");
commands.AddItem(scriptFile.GetAbsolutePath());
containerLaunchContext.SetCommands(commands);
Resource r = BuilderUtils.NewResource(8 * 1024 * 1024, 1);
ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId,
context.GetNodeId().ToString(), user, r, Runtime.CurrentTimeMillis() + 120000, 123
, DummyRmIdentifier, Priority.NewInstance(0), 0);
Token containerToken = BuilderUtils.NewContainerToken(context.GetNodeId(), containerManager
.GetContext().GetContainerTokenSecretManager().CreatePassword(containerIdentifier
), containerIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.Exists() && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
Log.Info("Waiting for process start-file to be created");
}
NUnit.Framework.Assert.IsTrue("ProcessStartFile doesn't exist!", processStartFile
.Exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
NUnit.Framework.Assert.AreEqual("Hello World!", reader.ReadLine());
// Get the pid of the process
string pid = reader.ReadLine().Trim();
// No more lines
NUnit.Framework.Assert.AreEqual(null, reader.ReadLine());
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete, 60);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledExceededVmem, containerStatus
.GetExitStatus());
string expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: "
+ "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. "
+ "Killing container.\nDump of the process-tree for " + cId + " :\n";
Sharpen.Pattern pat = Sharpen.Pattern.Compile(expectedMsgPattern);
NUnit.Framework.Assert.AreEqual("Expected message pattern is: " + expectedMsgPattern
+ "\n\nObserved message is: " + containerStatus.GetDiagnostics(), true, pat.Matcher
(containerStatus.GetDiagnostics()).Find());
// Assert that the process is not alive anymore
NUnit.Framework.Assert.IsFalse("Process is still alive!", exec.SignalContainer(user
, pid, ContainerExecutor.Signal.Null));
}