private static void MergeContainerHistoryData(ContainerHistoryData historyData, ContainerFinishData
finishData)
{
historyData.SetFinishTime(finishData.GetFinishTime());
historyData.SetDiagnosticsInfo(finishData.GetDiagnosticsInfo());
historyData.SetContainerExitStatus(finishData.GetContainerExitStatus());
historyData.SetContainerState(finishData.GetContainerState());
}
private static void MergeContainerHistoryData(ContainerHistoryData historyData, ContainerStartData
startData)
{
historyData.SetAllocatedResource(startData.GetAllocatedResource());
historyData.SetAssignedNode(startData.GetAssignedNode());
historyData.SetPriority(startData.GetPriority());
historyData.SetStartTime(startData.GetStartTime());
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerHistoryData GetContainer(ContainerId containerId)
{
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(containerId.GetApplicationAttemptId().GetApplicationId());
try
{
bool readStartData = false;
bool readFinishData = false;
ContainerHistoryData historyData = ContainerHistoryData.NewInstance(containerId,
null, null, null, long.MinValue, long.MaxValue, null, int.MaxValue, null);
while ((!readStartData || !readFinishData) && hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.Equals(containerId.ToString()))
{
if (entry.key.suffix.Equals(StartDataSuffix))
{
ContainerStartData startData = ParseContainerStartData(entry.value);
MergeContainerHistoryData(historyData, startData);
readStartData = true;
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
ContainerFinishData finishData = ParseContainerFinishData(entry.value);
MergeContainerHistoryData(historyData, finishData);
readFinishData = true;
}
}
}
}
if (!readStartData && !readFinishData)
{
return(null);
}
if (!readStartData)
{
Log.Warn("Start information is missing for container " + containerId);
}
if (!readFinishData)
{
Log.Warn("Finish information is missing for container " + containerId);
}
Log.Info("Completed reading history information of container " + containerId);
return(historyData);
}
catch (IOException e)
{
Log.Error("Error when reading history file of container " + containerId, e);
throw;
}
finally
{
hfReader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
)
{
// read application history data
NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count);
for (int i = 1; i <= num; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
ApplicationHistoryData appData = store.GetApplication(appId);
NUnit.Framework.Assert.IsNotNull(appData);
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName());
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo());
// read application attempt history data
NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count);
for (int j = 1; j <= num; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
ApplicationAttemptHistoryData attemptData = store.GetApplicationAttempt(appAttemptId
);
NUnit.Framework.Assert.IsNotNull(attemptData);
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost());
if (missingApplicationAttempt && j == num)
{
NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo());
continue;
}
else
{
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo
());
}
// read container history data
NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count);
for (int k = 1; k <= num; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
ContainerHistoryData containerData = store.GetContainer(containerId);
NUnit.Framework.Assert.IsNotNull(containerData);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData
.GetPriority());
if (missingContainer && k == num)
{
NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo());
}
else
{
NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo
());
}
}
ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
NUnit.Framework.Assert.IsNotNull(masterContainer);
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
.GetContainerId());
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ContainerId, ContainerHistoryData> GetContainers(ApplicationAttemptId
appAttemptId)
{
IDictionary <ContainerId, ContainerHistoryData> historyDataMap = new Dictionary <ContainerId
, ContainerHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appAttemptId.GetApplicationId());
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ContainerPrefix))
{
ContainerId containerId = ConverterUtils.ToContainerId(entry.key.id);
if (containerId.GetApplicationAttemptId().Equals(appAttemptId))
{
ContainerHistoryData historyData = historyDataMap[containerId];
if (historyData == null)
{
historyData = ContainerHistoryData.NewInstance(containerId, null, null, null, long.MinValue
, long.MaxValue, null, int.MaxValue, null);
historyDataMap[containerId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerStartData(entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeContainerHistoryData(historyData, ParseContainerFinishData(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all conatiners" + " of application attempt "
+ appAttemptId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some containers" + " of application attempt "
+ appAttemptId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
private ContainerReport ConvertToContainerReport(ContainerHistoryData containerHistory
, string user)
{
// If the container has the aggregated log, add the server root url
string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, containerHistory
.GetAssignedNode().ToString(), containerHistory.GetContainerId().ToString(), containerHistory
.GetContainerId().ToString(), user);
return(ContainerReport.NewInstance(containerHistory.GetContainerId(), containerHistory
.GetAllocatedResource(), containerHistory.GetAssignedNode(), containerHistory.GetPriority
(), containerHistory.GetStartTime(), containerHistory.GetFinishTime(), containerHistory
.GetDiagnosticsInfo(), logUrl, containerHistory.GetContainerExitStatus(), containerHistory
.GetContainerState(), null));
}
/// <exception cref="System.IO.IOException"/>
public virtual void ContainerStarted(ContainerStartData containerStart)
{
ConcurrentMap <ContainerId, ContainerHistoryData> subMap = GetSubMap(containerStart
.GetContainerId().GetApplicationAttemptId());
ContainerHistoryData oldData = subMap.PutIfAbsent(containerStart.GetContainerId()
, ContainerHistoryData.NewInstance(containerStart.GetContainerId(), containerStart
.GetAllocatedResource(), containerStart.GetAssignedNode(), containerStart.GetPriority
(), containerStart.GetStartTime(), long.MaxValue, null, int.MaxValue, null));
if (oldData != null)
{
throw new IOException("The start information of container " + containerStart.GetContainerId
() + " is already stored.");
}
}
public virtual void TestWriteContainer()
{
RMContainer container = CreateRMContainer(ContainerId.NewContainerId(ApplicationAttemptId
.NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
writer.ContainerStarted(container);
ContainerHistoryData containerHD = null;
for (int i = 0; i < MaxRetries; ++i)
{
containerHD = store.GetContainer(ContainerId.NewContainerId(ApplicationAttemptId.
NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
if (containerHD != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.IsNotNull(containerHD);
NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("test host", -100), containerHD
.GetAssignedNode());
NUnit.Framework.Assert.AreEqual(Resource.NewInstance(-1, -1), containerHD.GetAllocatedResource
());
NUnit.Framework.Assert.AreEqual(Priority.Undefined, containerHD.GetPriority());
NUnit.Framework.Assert.AreEqual(0L, container.GetCreationTime());
writer.ContainerFinished(container);
for (int i_1 = 0; i_1 < MaxRetries; ++i_1)
{
containerHD = store.GetContainer(ContainerId.NewContainerId(ApplicationAttemptId.
NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
if (containerHD.GetContainerState() != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.AreEqual("test diagnostics info", containerHD.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(-1, containerHD.GetContainerExitStatus());
NUnit.Framework.Assert.AreEqual(ContainerState.Complete, containerHD.GetContainerState
());
}
/// <exception cref="System.IO.IOException"/>
public virtual void ContainerFinished(ContainerFinishData containerFinish)
{
ConcurrentMap <ContainerId, ContainerHistoryData> subMap = GetSubMap(containerFinish
.GetContainerId().GetApplicationAttemptId());
ContainerHistoryData data = subMap[containerFinish.GetContainerId()];
if (data == null)
{
throw new IOException("The finish information of container " + containerFinish.GetContainerId
() + " is stored before" + " the start information.");
}
// Make the assumption that ContainerState should not be null if
// the finish information is already recorded
if (data.GetContainerState() != null)
{
throw new IOException("The finish information of container " + containerFinish.GetContainerId
() + " is already stored.");
}
data.SetFinishTime(containerFinish.GetFinishTime());
data.SetDiagnosticsInfo(containerFinish.GetDiagnosticsInfo());
data.SetContainerExitStatus(containerFinish.GetContainerExitStatus());
data.SetContainerState(containerFinish.GetContainerState());
}
public virtual void TestReadWriteContainerHistory()
{
// Out of order
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 1);
try
{
WriteContainerFinishData(containerId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is stored before the start information"
));
}
// Normal
WriteApplicationAttemptStartData(appAttemptId);
int numContainers = 5;
for (int i = 1; i <= numContainers; ++i)
{
containerId = ContainerId.NewContainerId(appAttemptId, i);
WriteContainerStartData(containerId);
WriteContainerFinishData(containerId);
}
NUnit.Framework.Assert.AreEqual(numContainers, store.GetContainers(appAttemptId).
Count);
for (int i_1 = 1; i_1 <= numContainers; ++i_1)
{
containerId = ContainerId.NewContainerId(appAttemptId, i_1);
ContainerHistoryData data = store.GetContainer(containerId);
NUnit.Framework.Assert.IsNotNull(data);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), data.GetPriority
());
NUnit.Framework.Assert.AreEqual(containerId.ToString(), data.GetDiagnosticsInfo()
);
}
ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
NUnit.Framework.Assert.IsNotNull(masterContainer);
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
.GetContainerId());
WriteApplicationAttemptFinishData(appAttemptId);
// Write again
containerId = ContainerId.NewContainerId(appAttemptId, 1);
try
{
WriteContainerStartData(containerId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is already stored"));
}
try
{
WriteContainerFinishData(containerId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is already stored"));
}
}
