protected internal virtual ApplicationAttemptId CreateSchedulingRequest(int memory
, int vcores, string queueId, string userId, int numContainers, int priority)
{
ApplicationAttemptId id = CreateAppAttemptId(this.AppId++, this.AttemptId++);
scheduler.AddApplication(id.GetApplicationId(), queueId, userId, false);
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
if (scheduler.GetSchedulerApplications().Contains(id.GetApplicationId()))
{
scheduler.AddApplicationAttempt(id, false, false);
}
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest request = CreateResourceRequest(memory, vcores, ResourceRequest.Any
, priority, numContainers, true);
ask.AddItem(request);
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttempt rmAppAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(rmApp.GetCurrentAppAttempt()).ThenReturn(rmAppAttempt);
Org.Mockito.Mockito.When(rmAppAttempt.GetRMAppAttemptMetrics()).ThenReturn(new RMAppAttemptMetrics
(id, resourceManager.GetRMContext()));
resourceManager.GetRMContext().GetRMApps()[id.GetApplicationId()] = rmApp;
scheduler.Allocate(id, ask, new AList <ContainerId>(), null, null);
return(id);
}
// resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
// is called.
// newlyAllocatedContainers.add(rmContainer);
// schedulingOpportunities
// lastScheduledContainer
public virtual void IncNumAllocatedContainers(NodeType containerType, NodeType requestType
)
{
RMAppAttempt attempt = rmContext.GetRMApps()[attemptId.GetApplicationId()].GetCurrentAppAttempt
();
if (attempt != null)
{
attempt.GetRMAppAttemptMetrics().IncNumAllocatedContainers(containerType, requestType
);
}
}
public virtual void TestBasic()
{
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
OutputCommitter mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>();
Clock mockClock = Org.Mockito.Mockito.Mock <Clock>();
CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter
, new TestCommitterEventHandler.TestingRMHeartbeatHandler());
YarnConfiguration conf = new YarnConfiguration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
JobContext mockJobContext = Org.Mockito.Mockito.Mock <JobContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId
()));
TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler
();
Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler
);
Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock);
handler.Init(conf);
handler.Start();
try
{
handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext));
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path startCommitFile = MRApps.GetStartJobCommitFile(conf, user, jobId);
Path endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId);
Path endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId);
Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent();
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile
));
NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile
));
NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile
));
Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>());
}
finally
{
handler.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestSchedulingDelay()
{
// Add one node
string host = "127.0.0.1";
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4096, 4), 1, host
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.Handle(nodeUpdateEvent);
// Create one application and submit one each of node-local, rack-local
// and ANY requests
ApplicationAttemptId appAttemptId = CreateAppAttemptId(this.AppId++, this.AttemptId
++);
CreateMockRMApp(appAttemptId);
scheduler.AddApplication(appAttemptId.GetApplicationId(), "queue11", "user11", false
);
scheduler.AddApplicationAttempt(appAttemptId, false, false);
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ask.AddItem(CreateResourceRequest(1024, 1, ResourceRequest.Any, 1, 1, true));
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
FSAppAttempt app = scheduler.GetSchedulerApp(appAttemptId);
// Advance time and let continuous scheduling kick in
mockClock.Tick(1);
while (1024 != app.GetCurrentConsumption().GetMemory())
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(1024, app.GetCurrentConsumption().GetMemory());
}
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, string
user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext)
{
// This pendingRelease is used in work-preserving recovery scenario to keep
// track of the AM's outstanding release requests. RM on recovery could
// receive the release request form AM before it receives the container status
// from NM for recovery. In this case, the to-be-recovered containers reported
// by NM should not be recovered.
// Time of the last container scheduled at the current allowed level
Preconditions.CheckNotNull(rmContext, "RMContext should not be null");
this.rmContext = rmContext;
this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue,
activeUsersManager, rmContext.GetEpoch());
this.queue = queue;
this.pendingRelease = new HashSet <ContainerId>();
this.attemptId = applicationAttemptId;
if (rmContext.GetRMApps() != null && rmContext.GetRMApps().Contains(applicationAttemptId
.GetApplicationId()))
{
ApplicationSubmissionContext appSubmissionContext = rmContext.GetRMApps()[applicationAttemptId
.GetApplicationId()].GetApplicationSubmissionContext();
if (appSubmissionContext != null)
{
unmanagedAM = appSubmissionContext.GetUnmanagedAM();
this.logAggregationContext = appSubmissionContext.GetLogAggregationContext();
}
}
}
public virtual void AddApplicationAttempt(ApplicationAttemptId appAttemptId, bool
transferStateFromPreviousAttempt, bool isAttemptRecovering)
{
lock (this)
{
SchedulerApplication <FiCaSchedulerApp> application = applications[appAttemptId.GetApplicationId
()];
string user = application.GetUser();
// TODO: Fix store
FiCaSchedulerApp schedulerApp = new FiCaSchedulerApp(appAttemptId, user, DefaultQueue
, activeUsersManager, this.rmContext);
if (transferStateFromPreviousAttempt)
{
schedulerApp.TransferStateFromPreviousAttempt(application.GetCurrentAppAttempt());
}
application.SetCurrentAppAttempt(schedulerApp);
metrics.SubmitAppAttempt(user);
Log.Info("Added Application Attempt " + appAttemptId + " to scheduler from user "
+ application.GetUser());
if (isAttemptRecovering)
{
if (Log.IsDebugEnabled())
{
Log.Debug(appAttemptId + " is recovering. Skipping notifying ATTEMPT_ADDED");
}
}
else
{
rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.AttemptAdded));
}
}
}
/// <exception cref="System.IO.IOException"/>
private void DoneApplicationAttempt(ApplicationAttemptId applicationAttemptId, RMAppAttemptState
rmAppAttemptFinalState, bool keepContainers)
{
lock (this)
{
FiCaSchedulerApp attempt = GetApplicationAttempt(applicationAttemptId);
SchedulerApplication <FiCaSchedulerApp> application = applications[applicationAttemptId
.GetApplicationId()];
if (application == null || attempt == null)
{
throw new IOException("Unknown application " + applicationAttemptId + " has completed!"
);
}
// Kill all 'live' containers
foreach (RMContainer container in attempt.GetLiveContainers())
{
if (keepContainers && container.GetState().Equals(RMContainerState.Running))
{
// do not kill the running container in the case of work-preserving AM
// restart.
Log.Info("Skip killing " + container.GetContainerId());
continue;
}
CompletedContainer(container, SchedulerUtils.CreateAbnormalContainerStatus(container
.GetContainerId(), SchedulerUtils.CompletedApplication), RMContainerEventType.Kill
);
}
// Clean up pending requests, metrics etc.
attempt.Stop(rmAppAttemptFinalState);
}
}
public virtual T GetApplicationAttempt(ApplicationAttemptId applicationAttemptId)
{
SchedulerApplication <T> app = applications[applicationAttemptId.GetApplicationId(
)];
return(app == null ? null : app.GetCurrentAppAttempt());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerReport> GetContainers(ApplicationAttemptId
appAttemptId)
{
ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt app = GetApplication
(appAttemptId.GetApplicationId(), ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
.UserAndAcls);
CheckAccess(app);
TimelineEntities entities = timelineDataManager.GetEntities(ContainerMetricsConstants
.EntityType, new NameValuePair(ContainerMetricsConstants.ParentPrimariyFilter, appAttemptId
.ToString()), null, null, null, null, null, long.MaxValue, EnumSet.AllOf <TimelineReader.Field
>(), UserGroupInformation.GetLoginUser());
IDictionary <ContainerId, ContainerReport> containers = new LinkedHashMap <ContainerId
, ContainerReport>();
if (entities != null && entities.GetEntities() != null)
{
foreach (TimelineEntity entity in entities.GetEntities())
{
ContainerReport container = ConvertToContainerReport(entity, serverHttpAddress, app
.appReport.GetUser());
containers[container.GetContainerId()] = container;
}
}
return(containers);
}
public virtual IList <Container> GetTransferredContainers(ApplicationAttemptId currentAttempt
)
{
ApplicationId appId = currentAttempt.GetApplicationId();
SchedulerApplication <T> app = applications[appId];
IList <Container> containerList = new AList <Container>();
RMApp appImpl = this.rmContext.GetRMApps()[appId];
if (appImpl.GetApplicationSubmissionContext().GetUnmanagedAM())
{
return(containerList);
}
if (app == null)
{
return(containerList);
}
ICollection <RMContainer> liveContainers = app.GetCurrentAppAttempt().GetLiveContainers
();
ContainerId amContainerId = rmContext.GetRMApps()[appId].GetCurrentAppAttempt().GetMasterContainer
().GetId();
foreach (RMContainer rmContainer in liveContainers)
{
if (!rmContainer.GetContainerId().Equals(amContainerId))
{
containerList.AddItem(rmContainer.GetContainer());
}
}
return(containerList);
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerReport GetAMContainer(ApplicationAttemptId appAttemptId)
{
ApplicationReport app = GetApplication(appAttemptId.GetApplicationId());
return(ConvertToContainerReport(historyStore.GetAMContainer(appAttemptId), app ==
null ? null : app.GetUser()));
}
protected internal virtual RMApp CreateMockRMApp(ApplicationAttemptId attemptId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(attemptId.GetApplicationId
());
RMAppAttemptImpl attempt = Org.Mockito.Mockito.Mock <RMAppAttemptImpl>();
Org.Mockito.Mockito.When(attempt.GetAppAttemptId()).ThenReturn(attemptId);
RMAppAttemptMetrics attemptMetric = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics>
();
Org.Mockito.Mockito.When(attempt.GetRMAppAttemptMetrics()).ThenReturn(attemptMetric
);
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(attempt);
resourceManager.GetRMContext().GetRMApps()[attemptId.GetApplicationId()] = app;
return(app);
}
protected internal virtual void CreateApplicationWithAMResource(ApplicationAttemptId
attId, string queue, string user, Resource amResource)
{
RMContext rmContext = resourceManager.GetRMContext();
RMApp rmApp = new RMAppImpl(attId.GetApplicationId(), rmContext, conf, null, null
, null, ApplicationSubmissionContext.NewInstance(null, null, null, null, null, false
, false, 0, amResource, null), null, null, 0, null, null, null);
rmContext.GetRMApps()[attId.GetApplicationId()] = rmApp;
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attId.GetApplicationId
(), queue, user);
scheduler.Handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent
(attId, false);
scheduler.Handle(attempAddedEvent);
}
public JobHistoryCopyService(ApplicationAttemptId applicationAttemptId, EventHandler
handler)
: base("JobHistoryCopyService")
{
this.applicationAttemptId = applicationAttemptId;
this.jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(applicationAttemptId.GetApplicationId
()));
this.handler = handler;
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual FinishApplicationMasterResponse FinishApplicationMaster(FinishApplicationMasterRequest
request)
{
ApplicationAttemptId applicationAttemptId = AuthorizeRequest().GetApplicationAttemptId
();
ApplicationId appId = applicationAttemptId.GetApplicationId();
RMApp rmApp = rmContext.GetRMApps()[applicationAttemptId.GetApplicationId()];
// checking whether the app exits in RMStateStore at first not to throw
// ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart.
if (rmApp.IsAppFinalStateStored())
{
Log.Info(rmApp.GetApplicationId() + " unregistered successfully. ");
return(FinishApplicationMasterResponse.NewInstance(true));
}
ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
];
if (Lock == null)
{
ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
}
// Allow only one thread in AM to do finishApp at a time.
lock (Lock)
{
if (!HasApplicationMasterRegistered(applicationAttemptId))
{
string message = "Application Master is trying to unregister before registering for: "
+ appId;
Log.Error(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appId].GetUser(), RMAuditLogger.AuditConstants
.UnregisterAm, string.Empty, "ApplicationMasterService", message, appId, applicationAttemptId
);
throw new ApplicationMasterNotRegisteredException(message);
}
this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptUnregistrationEvent
(applicationAttemptId, request.GetTrackingUrl(), request.GetFinalApplicationStatus
(), request.GetDiagnostics()));
// For UnmanagedAMs, return true so they don't retry
return(FinishApplicationMasterResponse.NewInstance(rmApp.GetApplicationSubmissionContext
().GetUnmanagedAM()));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> GetApplicationAttempts
(ApplicationId appId)
{
IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> historyDataMap =
new Dictionary <ApplicationAttemptId, ApplicationAttemptHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appId);
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ApplicationAttemptPrefix))
{
ApplicationAttemptId appAttemptId = ConverterUtils.ToApplicationAttemptId(entry.key
.id);
if (appAttemptId.GetApplicationId().Equals(appId))
{
ApplicationAttemptHistoryData historyData = historyDataMap[appAttemptId];
if (historyData == null)
{
historyData = ApplicationAttemptHistoryData.NewInstance(appAttemptId, null, -1, null
, null, null, FinalApplicationStatus.Undefined, null);
historyDataMap[appAttemptId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeApplicationAttemptHistoryData(historyData, ParseApplicationAttemptStartData(
entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeApplicationAttemptHistoryData(historyData, ParseApplicationAttemptFinishData
(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all application" + " attempts of application "
+ appId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some application" + " attempts of application "
+ appId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
private RMAppImpl CreateMockRMApp(ApplicationAttemptId attemptId, RMContext context
)
{
RMAppImpl app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(attemptId.GetApplicationId
());
RMAppAttemptImpl attempt = Org.Mockito.Mockito.Mock <RMAppAttemptImpl>();
Org.Mockito.Mockito.When(attempt.GetAppAttemptId()).ThenReturn(attemptId);
RMAppAttemptMetrics attemptMetric = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics>
();
Org.Mockito.Mockito.When(attempt.GetRMAppAttemptMetrics()).ThenReturn(attemptMetric
);
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(attempt);
context.GetRMApps().PutIfAbsent(attemptId.GetApplicationId(), app);
return(app);
}
/// <summary>This will be called by startContainer.</summary>
/// <remarks>
/// This will be called by startContainer. It will add the master key into
/// the cache used for starting this container. This should be called before
/// validating the startContainer request.
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/>
public virtual void AppAttemptStartContainer(NMTokenIdentifier identifier)
{
lock (this)
{
ApplicationAttemptId appAttemptId = identifier.GetApplicationAttemptId();
if (!appToAppAttemptMap.Contains(appAttemptId.GetApplicationId()))
{
// First application attempt for the given application
appToAppAttemptMap[appAttemptId.GetApplicationId()] = new AList <ApplicationAttemptId
>();
}
MasterKeyData oldKey = oldMasterKeys[appAttemptId];
if (oldKey == null)
{
// This is a new application attempt.
appToAppAttemptMap[appAttemptId.GetApplicationId()].AddItem(appAttemptId);
}
if (oldKey == null || oldKey.GetMasterKey().GetKeyId() != identifier.GetKeyId())
{
// Update key only if it is modified.
Log.Debug("NMToken key updated for application attempt : " + identifier.GetApplicationAttemptId
().ToString());
if (identifier.GetKeyId() == currentMasterKey.GetMasterKey().GetKeyId())
{
UpdateAppAttemptKey(appAttemptId, currentMasterKey);
}
else
{
if (previousMasterKey != null && identifier.GetKeyId() == previousMasterKey.GetMasterKey
().GetKeyId())
{
UpdateAppAttemptKey(appAttemptId, previousMasterKey);
}
else
{
throw new SecretManager.InvalidToken("Older NMToken should not be used while starting the container."
);
}
}
}
}
}
private static TimelineEntity CreateAppAttemptEntity(ApplicationAttemptId appAttemptId
)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(AppAttemptMetricsConstants.EntityType);
entity.SetEntityId(appAttemptId.ToString());
entity.AddPrimaryFilter(AppAttemptMetricsConstants.ParentPrimaryFilter, appAttemptId
.GetApplicationId().ToString());
return(entity);
}
public virtual void TestContainerLogs()
{
WebResource r = Resource();
ContainerId containerId = BuilderUtils.NewContainerId(0, 0, 0, 0);
string containerIdStr = BuilderUtils.NewContainerId(0, 0, 0, 0).ToString();
ApplicationAttemptId appAttemptId = containerId.GetApplicationAttemptId();
ApplicationId appId = appAttemptId.GetApplicationId();
string appIdStr = appId.ToString();
string filename = "logfile1";
string logMessage = "log message\n";
nmContext.GetApplications()[appId] = new ApplicationImpl(null, "user", appId, null
, nmContext);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
new Configuration(), "user", appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[containerId] = container;
// write out log file
Path path = dirsHandler.GetLogPathForWrite(ContainerLaunch.GetRelativeContainerLogDir
(appIdStr, containerIdStr) + "/" + filename, false);
FilePath logFile = new FilePath(path.ToUri().GetPath());
logFile.DeleteOnExit();
NUnit.Framework.Assert.IsTrue("Failed to create log dir", logFile.GetParentFile()
.Mkdirs());
PrintWriter pw = new PrintWriter(logFile);
pw.Write(logMessage);
pw.Close();
// ask for it
ClientResponse response = r.Path("ws").Path("v1").Path("node").Path("containerlogs"
).Path(containerIdStr).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse
>();
string responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
// ask for file that doesn't exist
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path("uhhh").Accept(MediaType.TextPlain).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(ClientResponse.Status.NotFound.GetStatusCode(), response
.GetStatus());
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.IsTrue(responseText.Contains("Cannot find this log on the local disk."
));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), containerId);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[containerId]);
response = r.Path("ws").Path("v1").Path("node").Path("containerlogs").Path(containerIdStr
).Path(filename).Accept(MediaType.TextPlain).Get <ClientResponse>();
responseText = response.GetEntity <string>();
NUnit.Framework.Assert.AreEqual(logMessage, responseText);
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerHistoryData> GetContainers(ApplicationAttemptId
appAttemptId)
{
IDictionary <ContainerId, ContainerHistoryData> historyDataMap = new Dictionary <ContainerId
, ContainerHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appAttemptId.GetApplicationId());
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ContainerPrefix))
{
ContainerId containerId = ConverterUtils.ToContainerId(entry.key.id);
if (containerId.GetApplicationAttemptId().Equals(appAttemptId))
{
ContainerHistoryData historyData = historyDataMap[containerId];
if (historyData == null)
{
historyData = ContainerHistoryData.NewInstance(containerId, null, null, null, long.MinValue
, long.MaxValue, null, int.MaxValue, null);
historyDataMap[containerId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerStartData(entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerFinishData(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all conatiners" + " of application attempt "
+ appAttemptId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some containers" + " of application attempt "
+ appAttemptId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
protected internal virtual void ValidateIds(ApplicationId appId, ApplicationAttemptId
appAttemptId, ContainerId containerId)
{
if (!appAttemptId.GetApplicationId().Equals(appId))
{
throw new NotFoundException("appId and appAttemptId don't match");
}
if (containerId != null && !containerId.GetApplicationAttemptId().Equals(appAttemptId
))
{
throw new NotFoundException("appAttemptId and containerId don't match");
}
}
public static ApplicationAttemptId GetMockApplicationAttemptId(int appId, int attemptId
)
{
ApplicationId applicationId = BuilderUtils.NewApplicationId(0l, appId);
ApplicationAttemptId applicationAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
>();
Org.Mockito.Mockito.When(applicationAttemptId.GetApplicationId()).ThenReturn(applicationId
);
Org.Mockito.Mockito.When(applicationAttemptId.GetAttemptId()).ThenReturn(attemptId
);
return(applicationAttemptId);
}
/// <exception cref="System.Exception"/>
protected internal override void ModifyAppState()
{
// imitate appAttemptFile1 is still .new, but old one is deleted
string appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr1
);
Path appDir = fsTester.store.GetAppDir(attemptId1.GetApplicationId().ToString());
Path appAttemptFile1 = new Path(appDir, attemptId1.ToString() + ".new");
FileSystemRMStateStore fileSystemRMStateStore = (FileSystemRMStateStore)fsTester.
GetRMStateStore();
fileSystemRMStateStore.RenameFile(appAttemptFile1, new Path(appAttemptFile1.GetParent
(), appAttemptFile1.GetName() + ".new"));
}
/// <exception cref="System.IO.IOException"/>
public static Path GetPreviousJobHistoryPath(Configuration conf, ApplicationAttemptId
applicationAttemptId)
{
string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString
();
string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf,
jobId);
Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir
));
FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf);
return(fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath, jobId
, (applicationAttemptId.GetAttemptId() - 1))));
}
// from AMLauncher
/// <exception cref="System.Exception"/>
public virtual MockAM SendAMLaunched(ApplicationAttemptId appAttemptId)
{
MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);
am.WaitForState(RMAppAttemptState.Allocated);
//create and set AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = this.rmContext
.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId);
((RMAppAttemptImpl)this.rmContext.GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt
(appAttemptId)).SetAMRMToken(amrmToken);
GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.Launched));
return(am);
}
internal virtual void HandleNMContainerStatus(NMContainerStatus containerStatus,
NodeId nodeId)
{
ApplicationAttemptId appAttemptId = containerStatus.GetContainerId().GetApplicationAttemptId
();
RMApp rmApp = rmContext.GetRMApps()[appAttemptId.GetApplicationId()];
if (rmApp == null)
{
Log.Error("Received finished container : " + containerStatus.GetContainerId() + " for unknown application "
+ appAttemptId.GetApplicationId() + " Skipping.");
return;
}
if (rmApp.GetApplicationSubmissionContext().GetUnmanagedAM())
{
if (Log.IsDebugEnabled())
{
Log.Debug("Ignoring container completion status for unmanaged AM " + rmApp.GetApplicationId
());
}
return;
}
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptId);
Container masterContainer = rmAppAttempt.GetMasterContainer();
if (masterContainer.GetId().Equals(containerStatus.GetContainerId()) && containerStatus
.GetContainerState() == ContainerState.Complete)
{
ContainerStatus status = ContainerStatus.NewInstance(containerStatus.GetContainerId
(), containerStatus.GetContainerState(), containerStatus.GetDiagnostics(), containerStatus
.GetContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt = new RMAppAttemptContainerFinishedEvent(appAttemptId
, status, nodeId);
rmContext.GetDispatcher().GetEventHandler().Handle(evt);
}
}
public AppSchedulingInfo(ApplicationAttemptId appAttemptId, string user, Queue queue
, ActiveUsersManager activeUsersManager, long epoch)
{
// TODO making containerIdCounter long
//private final ApplicationStore store;
/* Allocated by scheduler */
// for app metrics
this.applicationAttemptId = appAttemptId;
this.applicationId = appAttemptId.GetApplicationId();
this.queue = queue;
this.queueName = queue.GetQueueName();
this.user = user;
this.activeUsersManager = activeUsersManager;
this.containerIdCounter = new AtomicLong(epoch << EpochBitShift);
}
private FiCaSchedulerApp GetMockApplication(int appId, string user, Org.Apache.Hadoop.Yarn.Api.Records.Resource
amResource)
{
FiCaSchedulerApp application = Org.Mockito.Mockito.Mock <FiCaSchedulerApp>();
ApplicationAttemptId applicationAttemptId = TestUtils.GetMockApplicationAttemptId
(appId, 0);
Org.Mockito.Mockito.DoReturn(applicationAttemptId.GetApplicationId()).When(application
).GetApplicationId();
Org.Mockito.Mockito.DoReturn(applicationAttemptId).When(application).GetApplicationAttemptId
();
Org.Mockito.Mockito.DoReturn(user).When(application).GetUser();
Org.Mockito.Mockito.DoReturn(amResource).When(application).GetAMResource();
return(application);
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerReport> GetContainers(ApplicationAttemptId
appAttemptId)
{
ApplicationReport app = GetApplication(appAttemptId.GetApplicationId());
IDictionary <ContainerId, ContainerHistoryData> histData = historyStore.GetContainers
(appAttemptId);
Dictionary <ContainerId, ContainerReport> containersReport = new Dictionary <ContainerId
, ContainerReport>();
foreach (KeyValuePair <ContainerId, ContainerHistoryData> entry in histData)
{
containersReport[entry.Key] = ConvertToContainerReport(entry.Value, app == null ?
null : app.GetUser());
}
return(containersReport);
}
