public virtual void TestLocalFilesCleanup()
{
// Real del service
delSrvc = new DeletionService(exec);
delSrvc.Init(conf);
containerManager = CreateContainerManager(delSrvc);
containerManager.Init(conf);
containerManager.Start();
// ////// Create the resources for the container
FilePath dir = new FilePath(tmpDir, "dir");
dir.Mkdirs();
FilePath file = new FilePath(dir, "file");
PrintWriter fileWriter = new PrintWriter(file);
fileWriter.Write("Hello World!");
fileWriter.Close();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
ApplicationId appId = cId.GetApplicationAttemptId().GetApplicationId();
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// containerLaunchContext.resources =
// new HashMap<CharSequence, LocalResource>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(FileContext.GetLocalFSFileContext
().MakeQualified(new Path(file.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(file.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Running);
// Now ascertain that the resources are localised correctly.
string appIDStr = ConverterUtils.ToString(appId);
string containerIDStr = ConverterUtils.ToString(cId);
FilePath userCacheDir = new FilePath(localDir, ContainerLocalizer.Usercache);
FilePath userDir = new FilePath(userCacheDir, user);
FilePath appCache = new FilePath(userDir, ContainerLocalizer.Appcache);
FilePath appDir = new FilePath(appCache, appIDStr);
FilePath containerDir = new FilePath(appDir, containerIDStr);
FilePath targetFile = new FilePath(containerDir, destinationFile);
FilePath sysDir = new FilePath(localDir, ResourceLocalizationService.NmPrivateDir
);
FilePath appSysDir = new FilePath(sysDir, appIDStr);
FilePath containerSysDir = new FilePath(appSysDir, containerIDStr);
// AppDir should still exist
NUnit.Framework.Assert.IsTrue("AppDir " + appDir.GetAbsolutePath() + " doesn't exist!!"
, appDir.Exists());
NUnit.Framework.Assert.IsTrue("AppSysDir " + appSysDir.GetAbsolutePath() + " doesn't exist!!"
, appSysDir.Exists());
foreach (FilePath f in new FilePath[] { containerDir, containerSysDir })
{
NUnit.Framework.Assert.IsFalse(f.GetAbsolutePath() + " exists!!", f.Exists());
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
// Simulate RM sending an AppFinish event.
containerManager.Handle(new CMgrCompletedAppsEvent(Arrays.AsList(new ApplicationId
[] { appId }), CMgrCompletedAppsEvent.Reason.OnShutdown));
BaseContainerManagerTest.WaitForApplicationState(containerManager, cId.GetApplicationAttemptId
().GetApplicationId(), ApplicationState.Finished);
// Now ascertain that the resources are localised correctly.
foreach (FilePath f_1 in new FilePath[] { appDir, containerDir, appSysDir, containerSysDir })
{
// Wait for deletion. Deletion can happen long after AppFinish because of
// the async DeletionService
int timeout = 0;
while (f_1.Exists() && timeout++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(f_1.GetAbsolutePath() + " exists!!", f_1.Exists());
}
// Wait for deletion
int timeout_1 = 0;
while (targetFile.Exists() && timeout_1++ < 15)
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsFalse(targetFile.GetAbsolutePath() + " exists!!", targetFile
.Exists());
}
/// <exception cref="System.Exception"/>
private void AmRestartTests(bool keepRunningContainers)
{
MockRM rm = new MockRM(conf);
rm.Start();
RMApp app = rm.SubmitApp(200, "name", "user", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "MAPREDUCE", false, keepRunningContainers
);
MockNM nm = new MockNM("127.0.0.1:1234", 10240, rm.GetResourceTrackerService());
nm.RegisterNode();
MockAM am0 = MockRM.LaunchAndRegisterAM(app, rm, nm);
int NumContainers = 1;
// allocate NUM_CONTAINERS containers
am0.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm.NodeHeartbeat(true);
// wait for containers to be allocated.
IList <Container> containers = am0.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am0.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
// launch the 2nd container.
ContainerId containerId2 = ContainerId.NewContainerId(am0.GetApplicationAttemptId
(), 2);
nm.NodeHeartbeat(am0.GetApplicationAttemptId(), containerId2.GetContainerId(), ContainerState
.Running);
rm.WaitForState(nm, containerId2, RMContainerState.Running);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
ICollection <RMContainer> rmContainers = rm.scheduler.GetSchedulerAppInfo(am0.GetApplicationAttemptId
()).GetLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId = app.GetCurrentAppAttempt().GetMasterContainer().GetId
();
nm.NodeHeartbeat(am0.GetApplicationAttemptId(), amContainerId.GetContainerId(), ContainerState
.Complete);
am0.WaitForState(RMAppAttemptState.Failed);
long memorySeconds = 0;
long vcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers)
{
// Only calculate the usage for the one container that has completed.
foreach (RMContainer c in rmContainers)
{
if (c.GetContainerId().Equals(amContainerId))
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
else
{
// The remaining container should be RUNNING.
NUnit.Framework.Assert.IsTrue("After first attempt failed, remaining container "
+ "should still be running. ", c.GetContainerState().Equals(ContainerState.Running
));
}
}
}
else
{
// If keepRunningContainers is false, all live containers should now
// be completed. Calculate the resource usage metrics for all of them.
foreach (RMContainer c in rmContainers)
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
}
// wait for app to start a new attempt.
rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
// assert this is a new AM.
RMAppAttempt attempt2 = app.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsFalse(attempt2.GetAppAttemptId().Equals(am0.GetApplicationAttemptId
()));
// launch the new AM
nm.NodeHeartbeat(true);
MockAM am1 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am1.RegisterAppAttempt();
// allocate NUM_CONTAINERS containers
am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm.NodeHeartbeat(true);
// wait for containers to be allocated.
containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>())
.GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
// Capture running containers for later use by metrics calculations.
rmContainers = rm.scheduler.GetSchedulerAppInfo(attempt2.GetAppAttemptId()).GetLiveContainers
();
// complete container by sending the container complete event which has
// earlier attempt's attemptId
amContainerId = app.GetCurrentAppAttempt().GetMasterContainer().GetId();
nm.NodeHeartbeat(am0.GetApplicationAttemptId(), amContainerId.GetContainerId(), ContainerState
.Complete);
MockRM.FinishAMAndVerifyAppState(app, rm, nm, am1);
// Calculate container usage metrics for second attempt.
foreach (RMContainer c_1 in rmContainers)
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c_1);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", memorySeconds,
rmAppMetrics.GetMemorySeconds());
NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", vcoreSeconds, rmAppMetrics
.GetVcoreSeconds());
rm.Stop();
return;
}
public virtual void TestApplicationRecovery()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
);
ContainerManagerImpl cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
context.GetNMTokenSecretManager().SetMasterKey(masterKey);
// add an application by starting a container
string appUser = "******";
string modUser = "******";
string viewUser = "******";
string enemyUser = "******";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId cid = ContainerId.NewContainerId(attemptId, 1);
IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
IDictionary <string, string> containerEnv = Sharpen.Collections.EmptyMap();
IList <string> containerCmds = Sharpen.Collections.EmptyList();
IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
Credentials containerCreds = new Credentials();
DataOutputBuffer dob = new DataOutputBuffer();
containerCreds.WriteTokenStorageToStream(dob);
ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>();
acls[ApplicationAccessType.ModifyApp] = modUser;
acls[ApplicationAccessType.ViewApp] = viewUser;
ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
, containerCmds, serviceData, containerTokens, acls);
// create the logAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
);
StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
);
NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// reset container manager and verify app recovered with proper acls
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
// check whether LogAggregationContext is recovered correctly
LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
();
NUnit.Framework.Assert.IsNotNull(recovered);
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
.GetIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
.GetExcludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
(), recovered.GetRolledLogsIncludePattern());
NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
(), recovered.GetRolledLogsExcludePattern());
WaitForAppState(app, ApplicationState.Initing);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate application completion
IList <ApplicationId> finishedApps = new AList <ApplicationId>();
finishedApps.AddItem(appId);
cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
ByResourcemanager));
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
// restart and verify app is marked for finishing
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
app = context.GetApplications()[appId];
NUnit.Framework.Assert.IsNotNull(app);
WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
.CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
// simulate log aggregation completion
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
));
NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
);
app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
));
// restart and verify app is no longer present after recovery
cm.Stop();
context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
cm = CreateContainerManager(context);
cm.Init(conf);
cm.Start();
NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
cm.Stop();
}
public virtual void SetExistingContainerId(ContainerId cId)
{
this.existingCid = cId;
}
public abstract void GetContainerStatusAsync(ContainerId containerId, NodeId nodeId
);
public virtual T GetCurrentAttemptForContainer(ContainerId containerId)
{
return(GetApplicationAttempt(containerId.GetApplicationAttemptId()));
}
/// <exception cref="System.Exception"/>
public virtual void TestFetchApplictionLogs()
{
string remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
configuration.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
configuration.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogRootDir);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
configuration.Set(YarnConfiguration.YarnAdminAcl, "admin");
FileSystem fs = FileSystem.Get(configuration);
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ApplicationId appId = ApplicationIdPBImpl.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
1);
ContainerId containerId0 = ContainerIdPBImpl.NewContainerId(appAttemptId, 0);
ContainerId containerId1 = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);
ContainerId containerId2 = ContainerIdPBImpl.NewContainerId(appAttemptId, 2);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
// create local logs
string rootLogDir = "target/LocalLogs";
Path rootLogDirPath = new Path(rootLogDir);
if (fs.Exists(rootLogDirPath))
{
fs.Delete(rootLogDirPath, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(rootLogDirPath));
Path appLogsDir = new Path(rootLogDirPath, appId.ToString());
if (fs.Exists(appLogsDir))
{
fs.Delete(appLogsDir, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(appLogsDir));
IList <string> rootLogDirs = Arrays.AsList(rootLogDir);
// create container logs in localLogDir
CreateContainerLogInLocalDir(appLogsDir, containerId1, fs);
CreateContainerLogInLocalDir(appLogsDir, containerId2, fs);
Path path = new Path(remoteLogRootDir + ugi.GetShortUserName() + "/logs/application_0_0001"
);
if (fs.Exists(path))
{
fs.Delete(path, true);
}
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
// upload container logs into remote directory
// the first two logs is empty. When we try to read first two logs,
// we will meet EOF exception, but it will not impact other logs.
// Other logs should be read successfully.
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0
, path, fs);
UploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1
, path, fs);
UploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2
, path, fs);
YarnClient mockYarnClient = CreateMockYarnClient(YarnApplicationState.Finished);
LogsCLI cli = new TestLogsCLI.LogsCLIForTest(mockYarnClient);
cli.SetConf(configuration);
int exitCode = cli.Run(new string[] { "-applicationId", appId.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000002!"
));
sysOutStream.Reset();
// uploaded two logs for container1. The first log is empty.
// The second one is not empty.
// We can still successfully read logs for container1.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId1.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == 0);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Hello container_0_0001_01_000001!"
));
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Log Upload Time")
);
NUnit.Framework.Assert.IsTrue(!sysOutStream.ToString().Contains("Logs for container "
+ containerId1.ToString() + " are not present in this log-file."));
sysOutStream.Reset();
// Uploaded the empty log for container0.
// We should see the message showing the log for container0
// are not present.
exitCode = cli.Run(new string[] { "-applicationId", appId.ToString(), "-nodeAddress"
, nodeId.ToString(), "-containerId", containerId0.ToString() });
NUnit.Framework.Assert.IsTrue(exitCode == -1);
NUnit.Framework.Assert.IsTrue(sysOutStream.ToString().Contains("Logs for container "
+ containerId0.ToString() + " are not present in this log-file."));
fs.Delete(new Path(remoteLogRootDir), true);
fs.Delete(new Path(rootLogDir), true);
}
// NOT IMPLEMENTED
public override RMContainer GetRMContainer(ContainerId containerId)
{
FiCaSchedulerApp attempt = GetCurrentAttemptForContainer(containerId);
return((attempt == null) ? null : attempt.GetRMContainer(containerId));
}
public RMContainerEvent(ContainerId containerId, RMContainerEventType type)
: base(type)
{
this.containerId = containerId;
}
public ApplicationContainerFinishedEvent(ContainerId containerID)
: base(containerID.GetApplicationAttemptId().GetApplicationId(), ApplicationEventType
.ApplicationContainerFinished)
{
this.containerID = containerID;
}
public override void Handle(SchedulerEvent @event)
{
switch (@event.GetType())
{
case SchedulerEventType.NodeAdded:
{
NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)@event;
AddNode(nodeAddedEvent.GetAddedRMNode());
RecoverContainersOnNode(nodeAddedEvent.GetContainerReports(), nodeAddedEvent.GetAddedRMNode
());
break;
}
case SchedulerEventType.NodeRemoved:
{
NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)@event;
RemoveNode(nodeRemovedEvent.GetRemovedRMNode());
break;
}
case SchedulerEventType.NodeResourceUpdate:
{
NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent
)@event;
UpdateNodeResource(nodeResourceUpdatedEvent.GetRMNode(), nodeResourceUpdatedEvent
.GetResourceOption());
break;
}
case SchedulerEventType.NodeUpdate:
{
NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)@event;
NodeUpdate(nodeUpdatedEvent.GetRMNode());
break;
}
case SchedulerEventType.AppAdded:
{
AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)@event;
AddApplication(appAddedEvent.GetApplicationId(), appAddedEvent.GetQueue(), appAddedEvent
.GetUser(), appAddedEvent.GetIsAppRecovering());
break;
}
case SchedulerEventType.AppRemoved:
{
AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)@event;
DoneApplication(appRemovedEvent.GetApplicationID(), appRemovedEvent.GetFinalState
());
break;
}
case SchedulerEventType.AppAttemptAdded:
{
AppAttemptAddedSchedulerEvent appAttemptAddedEvent = (AppAttemptAddedSchedulerEvent
)@event;
AddApplicationAttempt(appAttemptAddedEvent.GetApplicationAttemptId(), appAttemptAddedEvent
.GetTransferStateFromPreviousAttempt(), appAttemptAddedEvent.GetIsAttemptRecovering
());
break;
}
case SchedulerEventType.AppAttemptRemoved:
{
AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = (AppAttemptRemovedSchedulerEvent
)@event;
try
{
DoneApplicationAttempt(appAttemptRemovedEvent.GetApplicationAttemptID(), appAttemptRemovedEvent
.GetFinalAttemptState(), appAttemptRemovedEvent.GetKeepContainersAcrossAppAttempts
());
}
catch (IOException ie)
{
Log.Error("Unable to remove application " + appAttemptRemovedEvent.GetApplicationAttemptID
(), ie);
}
break;
}
case SchedulerEventType.ContainerExpired:
{
ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent
)@event;
ContainerId containerid = containerExpiredEvent.GetContainerId();
CompletedContainer(GetRMContainer(containerid), SchedulerUtils.CreateAbnormalContainerStatus
(containerid, SchedulerUtils.ExpiredContainer), RMContainerEventType.Expire);
break;
}
case SchedulerEventType.ContainerRescheduled:
{
ContainerRescheduledEvent containerRescheduledEvent = (ContainerRescheduledEvent)
@event;
RMContainer container = containerRescheduledEvent.GetContainer();
RecoverResourceRequestForContainer(container);
break;
}
default:
{
Log.Error("Invalid eventtype " + @event.GetType() + ". Ignoring!");
break;
}
}
}
/// <exception cref="System.IO.IOException"/>
public static Token CreateContainerToken(ContainerId cId, long rmIdentifier, NodeId
nodeId, string user, NMContainerTokenSecretManager containerTokenSecretManager)
{
return(CreateContainerToken(cId, rmIdentifier, nodeId, user, containerTokenSecretManager
, null));
}
public virtual void TestMultipleContainersStopAndGetStatus()
{
containerManager.Start();
IList <StartContainerRequest> startRequest = new AList <StartContainerRequest>();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
IList <ContainerId> containerIds = new AList <ContainerId>();
for (int i = 0; i < 10; i++)
{
ContainerId cId = CreateContainerId(i);
string user = null;
if ((i & 1) == 0)
{
// container with even id fail
user = "******";
}
else
{
user = "******";
}
Token containerToken = CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId
(), user, context.GetContainerTokenSecretManager());
StartContainerRequest request = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
startRequest.AddItem(request);
containerIds.AddItem(cId);
}
// start containers
StartContainersRequest requestList = StartContainersRequest.NewInstance(startRequest
);
containerManager.StartContainers(requestList);
// Get container statuses
GetContainerStatusesRequest statusRequest = GetContainerStatusesRequest.NewInstance
(containerIds);
GetContainerStatusesResponse statusResponse = containerManager.GetContainerStatuses
(statusRequest);
NUnit.Framework.Assert.AreEqual(5, statusResponse.GetContainerStatuses().Count);
foreach (ContainerStatus status in statusResponse.GetContainerStatuses())
{
// Containers with odd id should succeed
NUnit.Framework.Assert.AreEqual(1, status.GetContainerId().GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, statusResponse.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry in statusResponse.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry.Value.GetMessage().Contains("Reject this container"
));
}
// stop containers
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
StopContainersResponse stopResponse = containerManager.StopContainers(stopRequest
);
NUnit.Framework.Assert.AreEqual(5, stopResponse.GetSuccessfullyStoppedContainers(
).Count);
foreach (ContainerId id in stopResponse.GetSuccessfullyStoppedContainers())
{
// Containers with odd id should succeed.
NUnit.Framework.Assert.AreEqual(1, id.GetContainerId() & 1);
}
NUnit.Framework.Assert.AreEqual(5, stopResponse.GetFailedRequests().Count);
foreach (KeyValuePair <ContainerId, SerializedException> entry_1 in stopResponse.GetFailedRequests
())
{
// Containers with even id should fail.
NUnit.Framework.Assert.AreEqual(0, entry_1.Key.GetContainerId() & 1);
NUnit.Framework.Assert.IsTrue(entry_1.Value.GetMessage().Contains("Reject this container"
));
}
}
public virtual void TestContainerLaunchFromPreviousRM()
{
containerManager.Start();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ContainerId cId1 = CreateContainerId(0);
ContainerId cId2 = CreateContainerId(0);
containerLaunchContext.SetLocalResources(new Dictionary <string, LocalResource>());
// Construct the Container with Invalid RMIdentifier
StartContainerRequest startRequest1 = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId1, ResourceManagerConstants.RmInvalidIdentifier, context
.GetNodeId(), user, context.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(startRequest1);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
bool catchException = false;
try
{
StartContainersResponse response = containerManager.StartContainers(allRequests);
if (response.GetFailedRequests().Contains(cId1))
{
throw response.GetFailedRequests()[cId1].DeSerialize();
}
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
catchException = true;
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Container " + cId1 + " rejected as it is allocated by a previous RM"
));
NUnit.Framework.Assert.IsTrue(Sharpen.Runtime.EqualsIgnoreCase(e.GetType().FullName
, typeof(InvalidContainerException).FullName));
}
// Verify that startContainer fail because of invalid container request
NUnit.Framework.Assert.IsTrue(catchException);
// Construct the Container with a RMIdentifier within current RM
StartContainerRequest startRequest2 = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId2, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list2 = new AList <StartContainerRequest>();
list.AddItem(startRequest2);
StartContainersRequest allRequests2 = StartContainersRequest.NewInstance(list2);
containerManager.StartContainers(allRequests2);
bool noException = true;
try
{
containerManager.StartContainers(allRequests2);
}
catch (YarnException)
{
noException = false;
}
// Verify that startContainer get no YarnException
NUnit.Framework.Assert.IsTrue(noException);
}
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public static void WaitForContainerState(ContainerManagementProtocol containerManager
, ContainerId containerID, ContainerState finalState)
{
WaitForContainerState(containerManager, containerID, finalState, 20);
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
await this.LoginAsync(context, this.ContainerSource);
try
{
var containerSource = (ContainerSource)SecureResource.Create(this.ContainerSource, (IResourceResolutionContext)context);
var containerId = new ContainerId(this.ContainerSource, containerSource?.RegistryPrefix, this.RepositoryName, this.Tag);
if (!string.IsNullOrEmpty(this.ContainerSource))
{
containerId = await this.PullAsync(context, containerId);
}
var containerConfigArgs = await this.GetContainerConfigText(context);
if (containerConfigArgs == null)
{
return;
}
var escapeArg = GetEscapeArg(context);
var args = new StringBuilder("run ");
args.Append(containerConfigArgs);
args.Append(' ');
if (this.RunInBackground)
{
args.Append("-d ");
}
else if (string.IsNullOrWhiteSpace(this.ContainerName))
{
args.Append("--rm ");
}
if (!string.IsNullOrWhiteSpace(this.ContainerName))
{
args.Append($"--name {escapeArg(this.ContainerName)} ");
}
if (!string.IsNullOrWhiteSpace(AdditionalArguments))
{
args.Append($"{this.AdditionalArguments} ");
}
args.Append(escapeArg(containerId.FullName));
var argsText = args.ToString();
this.LogDebug($"Executing docker {argsText}...");
int result = await this.ExecuteCommandLineAsync(
context,
new RemoteProcessStartInfo
{
FileName = this.DockerExePath,
Arguments = argsText
}
);
this.Log(result == 0 ? MessageLevel.Debug : MessageLevel.Error, "Docker exited with code " + result);
}
finally
{
await this.LogoutAsync(context, this.ContainerSource);
}
}
private YarnProtos.ContainerIdProto ConvertToProtoFormat(ContainerId containerId)
{
return(((ContainerIdPBImpl)containerId).GetProto());
}
public override async Task ExecuteAsync(IOperationExecutionContext context)
{
await this.LoginAsync(context, this.ContainerSource);
try
{
var procExec = await context.Agent.GetServiceAsync <IRemoteProcessExecuter>();
var fileOps = await context.Agent.GetServiceAsync <IFileOperationsExecuter>();
await fileOps.CreateDirectoryAsync(context.WorkingDirectory);
var sourcePath = context.ResolvePath(this.SourceDirectory);
await fileOps.CreateDirectoryAsync(sourcePath);
var dockerfilePath = fileOps.CombinePath(sourcePath, "Dockerfile");
if (!string.IsNullOrWhiteSpace(this.DockerfileTemplate))
{
var item = SDK.GetRaftItem(RaftItemType.TextTemplate, this.DockerfileTemplate, context);
if (item == null)
{
this.LogError($"Text template \"{this.DockerfileTemplate}\" not found.");
return;
}
var dockerfileText = await context.ApplyTextTemplateAsync(item.Content, this.TemplateArguments != null?new Dictionary <string, RuntimeValue>(this.TemplateArguments) : null);
await fileOps.WriteAllTextAsync(dockerfilePath, dockerfileText, InedoLib.UTF8Encoding);
}
var containerSource = (ContainerSource)SecureResource.Create(this.ContainerSource, (IResourceResolutionContext)context);
var containerId = new ContainerId(this.ContainerSource, containerSource?.RegistryPrefix, this.RepositoryName, this.Tag);
var escapeArg = GetEscapeArg(context);
var args = $"build --force-rm --progress=plain --tag={escapeArg(containerId.FullName)} {this.AdditionalArguments} {escapeArg(sourcePath)}";
this.LogDebug("Executing docker " + args);
var startInfo = new RemoteProcessStartInfo
{
FileName = this.DockerExePath,
Arguments = args,
WorkingDirectory = sourcePath
};
using (var process = procExec.CreateProcess(startInfo))
{
process.OutputDataReceived += (s, e) => this.LogInformation(e.Data);
process.ErrorDataReceived += (s, e) => this.LogBuildError(context, e.Data);
process.Start();
await process.WaitAsync(context.CancellationToken);
if (process.ExitCode != 0)
{
this.LogError($"exit code: {process.ExitCode ?? -1}");
return;
}
}
var digest = await this.ExecuteGetDigest(context, containerId.FullName);
containerId = containerId.WithDigest(digest);
if (!string.IsNullOrEmpty(this.ContainerSource))
{
await this.PushAsync(context, containerId);
if (this.RemoveAfterPush)
{
this.LogDebug("Removing local image after successful push...");
await this.RemoveAsync(context, containerId);
}
}
if (this.AttachToBuild)
{
await this.AttachToBuildAsync(context, containerId);
}
}
finally
{
await this.LogoutAsync(context, this.ContainerSource);
}
}
public virtual RMContainer GetRMContainer(ContainerId containerId)
{
SchedulerApplicationAttempt attempt = GetCurrentAttemptForContainer(containerId);
return((attempt == null) ? null : attempt.GetRMContainer(containerId));
}
public ResourceReleaseEvent(LocalResourceRequest rsrc, ContainerId container)
: base(rsrc, ResourceEventType.Release)
{
this.container = container;
}
/// <exception cref="System.Exception"/>
private static void UploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi
, Configuration configuration, IList <string> rootLogDirs, NodeId nodeId, ContainerId
containerId, Path appDir, FileSystem fs)
{
Path path = new Path(appDir, LogAggregationUtils.GetNodeString(nodeId) + Runtime.
CurrentTimeMillis());
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, path, ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
DataOutputStream @out = writer.GetWriter().PrepareAppendKey(-1);
new AggregatedLogFormat.LogKey(containerId).Write(@out);
@out.Close();
@out = writer.GetWriter().PrepareAppendValue(-1);
new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser
().GetShortUserName()).Write(@out, new HashSet <FilePath>());
@out.Close();
writer.Close();
}
protected internal override void Render(HtmlBlock.Block html)
{
ContainerId containerId = VerifyAndGetContainerId(html);
NodeId nodeId = VerifyAndGetNodeId(html);
string appOwner = VerifyAndGetAppOwner(html);
AggregatedLogsBlock.LogLimits logLimits = VerifyAndGetLogLimits(html);
if (containerId == null || nodeId == null || appOwner == null || appOwner.IsEmpty
() || logLimits == null)
{
return;
}
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
string logEntity = $(YarnWebParams.EntityString);
if (logEntity == null || logEntity.IsEmpty())
{
logEntity = containerId.ToString();
}
if (!conf.GetBoolean(YarnConfiguration.LogAggregationEnabled, YarnConfiguration.DefaultLogAggregationEnabled
))
{
html.H1().("Aggregation is not enabled. Try the nodemanager at " + nodeId).();
return;
}
Path remoteRootLogDir = new Path(conf.Get(YarnConfiguration.NmRemoteAppLogDir, YarnConfiguration
.DefaultNmRemoteAppLogDir));
Path remoteAppDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, applicationId
, appOwner, LogAggregationUtils.GetRemoteNodeLogDirSuffix(conf));
RemoteIterator <FileStatus> nodeFiles;
try
{
Path qualifiedLogDir = FileContext.GetFileContext(conf).MakeQualified(remoteAppDir
);
nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), conf).ListStatus(
remoteAppDir);
}
catch (FileNotFoundException)
{
html.H1().("Logs not available for " + logEntity + ". Aggregation may not be complete, "
+ "Check back later or try the nodemanager at " + nodeId).();
return;
}
catch (Exception)
{
html.H1().("Error getting logs at " + nodeId).();
return;
}
bool foundLog = false;
string desiredLogType = $(YarnWebParams.ContainerLogType);
try
{
while (nodeFiles.HasNext())
{
AggregatedLogFormat.LogReader reader = null;
try
{
FileStatus thisNodeFile = nodeFiles.Next();
if (!thisNodeFile.GetPath().GetName().Contains(LogAggregationUtils.GetNodeString(
nodeId)) || thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix
))
{
continue;
}
long logUploadedTime = thisNodeFile.GetModificationTime();
reader = new AggregatedLogFormat.LogReader(conf, thisNodeFile.GetPath());
string owner = null;
IDictionary <ApplicationAccessType, string> appAcls = null;
try
{
owner = reader.GetApplicationOwner();
appAcls = reader.GetApplicationAcls();
}
catch (IOException e)
{
Log.Error("Error getting logs for " + logEntity, e);
continue;
}
ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf);
aclsManager.AddApplication(applicationId, appAcls);
string remoteUser = Request().GetRemoteUser();
UserGroupInformation callerUGI = null;
if (remoteUser != null)
{
callerUGI = UserGroupInformation.CreateRemoteUser(remoteUser);
}
if (callerUGI != null && !aclsManager.CheckAccess(callerUGI, ApplicationAccessType
.ViewApp, owner, applicationId))
{
html.H1().("User [" + remoteUser + "] is not authorized to view the logs for " +
logEntity + " in log file [" + thisNodeFile.GetPath().GetName() + "]").();
Log.Error("User [" + remoteUser + "] is not authorized to view the logs for " + logEntity
);
continue;
}
AggregatedLogFormat.ContainerLogsReader logReader = reader.GetContainerLogsReader
(containerId);
if (logReader == null)
{
continue;
}
foundLog = ReadContainerLogs(html, logReader, logLimits, desiredLogType, logUploadedTime
);
}
catch (IOException ex)
{
Log.Error("Error getting logs for " + logEntity, ex);
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
}
if (!foundLog)
{
if (desiredLogType.IsEmpty())
{
html.H1("No logs available for container " + containerId.ToString());
}
else
{
html.H1("Unable to locate '" + desiredLogType + "' log for container " + containerId
.ToString());
}
}
}
catch (IOException e)
{
html.H1().("Error getting logs for " + logEntity).();
Log.Error("Error getting logs for " + logEntity, e);
}
}
public abstract void StopContainerAsync(ContainerId containerId, NodeId nodeId);
public ContainerEvent(ContainerId cID, ContainerEventType eventType)
: base(eventType)
{
this.containerID = cID;
}
/// <exception cref="System.Exception"/>
public virtual void TestUsageWithMultipleContainersAndRMRestart()
{
// Set max attempts to 1 so that when the first attempt fails, the app
// won't try to start a new one.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm0 = new MockRM(conf, memStore);
rm0.Start();
MockNM nm = new MockNM("127.0.0.1:1234", 65536, rm0.GetResourceTrackerService());
nm.RegisterNode();
RMApp app0 = rm0.SubmitApp(200);
rm0.WaitForState(app0.GetApplicationId(), RMAppState.Accepted);
RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();
ApplicationAttemptId attemptId0 = attempt0.GetAppAttemptId();
rm0.WaitForState(attemptId0, RMAppAttemptState.Scheduled);
nm.NodeHeartbeat(true);
rm0.WaitForState(attemptId0, RMAppAttemptState.Allocated);
MockAM am0 = rm0.SendAMLaunched(attempt0.GetAppAttemptId());
am0.RegisterAppAttempt();
int NumContainers = 2;
am0.Allocate("127.0.0.1", 1000, NumContainers, new AList <ContainerId>());
nm.NodeHeartbeat(true);
IList <Container> conts = am0.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
while (conts.Count != NumContainers)
{
nm.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am0.Allocate(new AList <ResourceRequest>(), new
AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(500);
}
// launch the 2nd and 3rd containers.
foreach (Container c in conts)
{
nm.NodeHeartbeat(attempt0.GetAppAttemptId(), c.GetId().GetContainerId(), ContainerState
.Running);
rm0.WaitForState(nm, c.GetId(), RMContainerState.Running);
}
// Get the RMContainers for all of the live containers, to be used later
// for metrics calculations and comparisons.
ICollection <RMContainer> rmContainers = rm0.scheduler.GetSchedulerAppInfo(attempt0
.GetAppAttemptId()).GetLiveContainers();
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
while (app0.GetRMAppMetrics().GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000
)
{
Sharpen.Thread.Sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
// Stop all non-AM containers
foreach (Container c_1 in conts)
{
if (c_1.GetId().GetContainerId() == 1)
{
continue;
}
nm.NodeHeartbeat(attempt0.GetAppAttemptId(), c_1.GetId().GetContainerId(), ContainerState
.Complete);
rm0.WaitForState(nm, c_1.GetId(), RMContainerState.Completed);
}
// After all other containers have completed, manually complete the master
// container in order to trigger a save to the state store of the resource
// usage metrics. This will cause the attempt to fail, and, since the max
// attempt retries is 1, the app will also fail. This is intentional so
// that all containers will complete prior to saving.
ContainerId cId = ContainerId.NewContainerId(attempt0.GetAppAttemptId(), 1);
nm.NodeHeartbeat(attempt0.GetAppAttemptId(), cId.GetContainerId(), ContainerState
.Complete);
rm0.WaitForState(nm, cId, RMContainerState.Completed);
// Check that the container metrics match those from the app usage report.
long memorySeconds = 0;
long vcoreSeconds = 0;
foreach (RMContainer c_2 in rmContainers)
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c_2);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
RMAppMetrics metricsBefore = app0.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", memorySeconds,
metricsBefore.GetMemorySeconds());
NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", vcoreSeconds, metricsBefore
.GetVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
RMApp app0After = rm1.GetRMContext().GetRMApps()[app0.GetApplicationId()];
// Compare container resource usage metrics from before and after restart.
RMAppMetrics metricsAfter = app0After.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Vcore seconds were not the same after RM Restart"
, metricsBefore.GetVcoreSeconds(), metricsAfter.GetVcoreSeconds());
NUnit.Framework.Assert.AreEqual("Memory seconds were not the same after RM Restart"
, metricsBefore.GetMemorySeconds(), metricsAfter.GetMemorySeconds());
rm0.Stop();
rm0.Close();
rm1.Stop();
rm1.Close();
}
public _PrivilegedExceptionAction_302(TestApplicationHistoryManagerOnTimelineStore
_enclosing, ContainerId containerId)
{
this._enclosing = _enclosing;
this.containerId = containerId;
}
public void PrimaryBoilerCalculateEEI_CorrecrOverallResultWContainer(PackagedSolutionId packId, ContainerId id, float expected)
{
var package = new PackageFactory().GetPackage(packId);
var calculation = new BoilerAsPrimary();
AssignUsageProperties(package);
//package.Appliances.Add(new ApplianceFactory().GetBoiler(id) ?? new Appliance());
if (id == ContainerId.SM500)
{
package.ApplianceInstances.Add(new ApplianceInstance()
{
Appliance = new ApplianceFactory().GetContainer(id)
});
}
var result = new EEICalculationResult();
result = calculation.CalculateEEI(package);
var EEI = Math.Round(result.EEI);
Assert.IsTrue(expected <= EEI + 1f && expected >= EEI - 1f);
}
// do nothing
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
protected internal override void AuthorizeGetAndStopContainerRequest(ContainerId
containerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
container, bool stopRequest, NMTokenIdentifier identifier)
{
}
public ContainerContext(string user, ContainerId containerId, Resource resource)
{
this.user = user;
this.containerId = containerId;
this.resource = resource;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void TestContainerLaunchAndExit(int exitCode)
{
FilePath scriptFile = Shell.AppendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
FilePath processStartFile = new FilePath(tmpDir, "start_file.txt").GetAbsoluteFile
();
// ////// Construct the Container-id
ContainerId cId = CreateContainerId(0);
if (Shell.Windows)
{
fileWriter.WriteLine("@echo Hello World!> " + processStartFile);
fileWriter.WriteLine("@echo " + cId + ">> " + processStartFile);
if (exitCode != 0)
{
fileWriter.WriteLine("@exit " + exitCode);
}
}
else
{
fileWriter.Write("\numask 0");
// So that start file is readable by the test
fileWriter.Write("\necho Hello World! > " + processStartFile);
fileWriter.Write("\necho $$ >> " + processStartFile);
// Have script throw an exit code at the end
if (exitCode != 0)
{
fileWriter.Write("\nexit " + exitCode);
}
}
fileWriter.Close();
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
URL resource_alpha = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.NewRecordInstance <LocalResource>();
rsrc_alpha.SetResource(resource_alpha);
rsrc_alpha.SetSize(-1);
rsrc_alpha.SetVisibility(LocalResourceVisibility.Application);
rsrc_alpha.SetType(LocalResourceType.File);
rsrc_alpha.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = rsrc_alpha;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, CreateContainerToken(cId, DummyRmIdentifier, context.GetNodeId(), user, context
.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cId, ContainerState
.Complete);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.NewInstance(
containerIds);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(gcsRequest
).GetContainerStatuses()[0];
// Verify exit status matches exit state of script
NUnit.Framework.Assert.AreEqual(exitCode, containerStatus.GetExitStatus());
}
