private static void MergeApplicationAttemptHistoryData(ApplicationAttemptHistoryData
historyData, ApplicationAttemptStartData startData)
{
historyData.SetHost(startData.GetHost());
historyData.SetRPCPort(startData.GetRPCPort());
historyData.SetMasterContainerId(startData.GetMasterContainerId());
}
/// <exception cref="System.IO.IOException"/>
public virtual void ApplicationAttemptFinished(ApplicationAttemptFinishData appAttemptFinish
)
{
ConcurrentMap <ApplicationAttemptId, ApplicationAttemptHistoryData> subMap = GetSubMap
(appAttemptFinish.GetApplicationAttemptId().GetApplicationId());
ApplicationAttemptHistoryData data = subMap[appAttemptFinish.GetApplicationAttemptId
()];
if (data == null)
{
throw new IOException("The finish information of application attempt " + appAttemptFinish
.GetApplicationAttemptId() + " is stored before" + " the start information.");
}
// Make the assumption that YarnApplicationAttemptState should not be null
// if the finish information is already recorded
if (data.GetYarnApplicationAttemptState() != null)
{
throw new IOException("The finish information of application attempt " + appAttemptFinish
.GetApplicationAttemptId() + " is already stored.");
}
data.SetTrackingURL(appAttemptFinish.GetTrackingURL());
data.SetDiagnosticsInfo(appAttemptFinish.GetDiagnosticsInfo());
data.SetFinalApplicationStatus(appAttemptFinish.GetFinalApplicationStatus());
data.SetYarnApplicationAttemptState(appAttemptFinish.GetYarnApplicationAttemptState
());
}
public virtual void TestReadWriteApplicationAttemptHistory()
{
// Out of order
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
try
{
WriteApplicationAttemptFinishData(appAttemptId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is stored before the start information"
));
}
// Normal
int numAppAttempts = 5;
WriteApplicationStartData(appId);
for (int i = 1; i <= numAppAttempts; ++i)
{
appAttemptId = ApplicationAttemptId.NewInstance(appId, i);
WriteApplicationAttemptStartData(appAttemptId);
WriteApplicationAttemptFinishData(appAttemptId);
}
NUnit.Framework.Assert.AreEqual(numAppAttempts, store.GetApplicationAttempts(appId
).Count);
for (int i_1 = 1; i_1 <= numAppAttempts; ++i_1)
{
appAttemptId = ApplicationAttemptId.NewInstance(appId, i_1);
ApplicationAttemptHistoryData data = store.GetApplicationAttempt(appAttemptId);
NUnit.Framework.Assert.IsNotNull(data);
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), data.GetHost());
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), data.GetDiagnosticsInfo(
));
}
WriteApplicationFinishData(appId);
// Write again
appAttemptId = ApplicationAttemptId.NewInstance(appId, 1);
try
{
WriteApplicationAttemptStartData(appAttemptId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is already stored"));
}
try
{
WriteApplicationAttemptFinishData(appAttemptId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is already stored"));
}
}
private ApplicationAttemptReport ConvertToApplicationAttemptReport(ApplicationAttemptHistoryData
appAttemptHistory)
{
return(ApplicationAttemptReport.NewInstance(appAttemptHistory.GetApplicationAttemptId
(), appAttemptHistory.GetHost(), appAttemptHistory.GetRPCPort(), appAttemptHistory
.GetTrackingURL(), null, appAttemptHistory.GetDiagnosticsInfo(), appAttemptHistory
.GetYarnApplicationAttemptState(), appAttemptHistory.GetMasterContainerId()));
}
private static void MergeApplicationAttemptHistoryData(ApplicationAttemptHistoryData
historyData, ApplicationAttemptFinishData finishData)
{
historyData.SetDiagnosticsInfo(finishData.GetDiagnosticsInfo());
historyData.SetTrackingURL(finishData.GetTrackingURL());
historyData.SetFinalApplicationStatus(finishData.GetFinalApplicationStatus());
historyData.SetYarnApplicationAttemptState(finishData.GetYarnApplicationAttemptState
());
}
/// <exception cref="System.IO.IOException"/>
private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
)
{
// read application history data
NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count);
for (int i = 1; i <= num; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
ApplicationHistoryData appData = store.GetApplication(appId);
NUnit.Framework.Assert.IsNotNull(appData);
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName());
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo());
// read application attempt history data
NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count);
for (int j = 1; j <= num; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
ApplicationAttemptHistoryData attemptData = store.GetApplicationAttempt(appAttemptId
);
NUnit.Framework.Assert.IsNotNull(attemptData);
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost());
if (missingApplicationAttempt && j == num)
{
NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo());
continue;
}
else
{
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo
());
}
// read container history data
NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count);
for (int k = 1; k <= num; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
ContainerHistoryData containerData = store.GetContainer(containerId);
NUnit.Framework.Assert.IsNotNull(containerData);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData
.GetPriority());
if (missingContainer && k == num)
{
NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo());
}
else
{
NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo
());
}
}
ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
NUnit.Framework.Assert.IsNotNull(masterContainer);
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
.GetContainerId());
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> GetApplicationAttempts
(ApplicationId appId)
{
IDictionary <ApplicationAttemptId, ApplicationAttemptHistoryData> historyDataMap =
new Dictionary <ApplicationAttemptId, ApplicationAttemptHistoryData>();
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appId);
try
{
while (hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.StartsWith(ConverterUtils.ApplicationAttemptPrefix))
{
ApplicationAttemptId appAttemptId = ConverterUtils.ToApplicationAttemptId(entry.key
.id);
if (appAttemptId.GetApplicationId().Equals(appId))
{
ApplicationAttemptHistoryData historyData = historyDataMap[appAttemptId];
if (historyData == null)
{
historyData = ApplicationAttemptHistoryData.NewInstance(appAttemptId, null, -1, null
, null, null, FinalApplicationStatus.Undefined, null);
historyDataMap[appAttemptId] = historyData;
}
if (entry.key.suffix.Equals(StartDataSuffix))
{
MergeApplicationAttemptHistoryData(historyData, ParseApplicationAttemptStartData(
entry.value));
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
MergeApplicationAttemptHistoryData(historyData, ParseApplicationAttemptFinishData
(entry.value));
}
}
}
}
}
Log.Info("Completed reading history information of all application" + " attempts of application "
+ appId);
}
catch (IOException)
{
Log.Info("Error when reading history information of some application" + " attempts of application "
+ appId);
}
finally
{
hfReader.Close();
}
return(historyDataMap);
}
/// <exception cref="System.IO.IOException"/>
public virtual ContainerHistoryData GetAMContainer(ApplicationAttemptId appAttemptId
)
{
ApplicationAttemptHistoryData attemptHistoryData = GetApplicationAttempt(appAttemptId
);
if (attemptHistoryData == null || attemptHistoryData.GetMasterContainerId() == null)
{
return(null);
}
return(GetContainer(attemptHistoryData.GetMasterContainerId()));
}
public virtual void TestWriteApplicationAttempt()
{
RMAppAttempt appAttempt = CreateRMAppAttempt(ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(0, 1), 1));
writer.ApplicationAttemptStarted(appAttempt);
ApplicationAttemptHistoryData appAttemptHD = null;
for (int i = 0; i < MaxRetries; ++i)
{
appAttemptHD = store.GetApplicationAttempt(ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(0, 1), 1));
if (appAttemptHD != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.IsNotNull(appAttemptHD);
NUnit.Framework.Assert.AreEqual("test host", appAttemptHD.GetHost());
NUnit.Framework.Assert.AreEqual(-100, appAttemptHD.GetRPCPort());
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(ApplicationAttemptId.NewInstance
(ApplicationId.NewInstance(0, 1), 1), 1), appAttemptHD.GetMasterContainerId());
writer.ApplicationAttemptFinished(appAttempt, RMAppAttemptState.Finished);
for (int i_1 = 0; i_1 < MaxRetries; ++i_1)
{
appAttemptHD = store.GetApplicationAttempt(ApplicationAttemptId.NewInstance(ApplicationId
.NewInstance(0, 1), 1));
if (appAttemptHD.GetYarnApplicationAttemptState() != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.AreEqual("test diagnostics info", appAttemptHD.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual("test url", appAttemptHD.GetTrackingURL());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Undefined, appAttemptHD.GetFinalApplicationStatus
());
NUnit.Framework.Assert.AreEqual(YarnApplicationAttemptState.Finished, appAttemptHD
.GetYarnApplicationAttemptState());
}
/// <exception cref="System.IO.IOException"/>
public virtual void ApplicationAttemptStarted(ApplicationAttemptStartData appAttemptStart
)
{
ConcurrentMap <ApplicationAttemptId, ApplicationAttemptHistoryData> subMap = GetSubMap
(appAttemptStart.GetApplicationAttemptId().GetApplicationId());
ApplicationAttemptHistoryData oldData = subMap.PutIfAbsent(appAttemptStart.GetApplicationAttemptId
(), ApplicationAttemptHistoryData.NewInstance(appAttemptStart.GetApplicationAttemptId
(), appAttemptStart.GetHost(), appAttemptStart.GetRPCPort(), appAttemptStart.GetMasterContainerId
(), null, null, null, null));
if (oldData != null)
{
throw new IOException("The start information of application attempt " + appAttemptStart
.GetApplicationAttemptId() + " is already stored.");
}
}
/// <exception cref="System.IO.IOException"/>
private ApplicationReport ConvertToApplicationReport(ApplicationHistoryData appHistory
)
{
ApplicationAttemptId currentApplicationAttemptId = null;
string trackingUrl = Unavailable;
string host = Unavailable;
int rpcPort = -1;
ApplicationAttemptHistoryData lastAttempt = GetLastAttempt(appHistory.GetApplicationId
());
if (lastAttempt != null)
{
currentApplicationAttemptId = lastAttempt.GetApplicationAttemptId();
trackingUrl = lastAttempt.GetTrackingURL();
host = lastAttempt.GetHost();
rpcPort = lastAttempt.GetRPCPort();
}
return(ApplicationReport.NewInstance(appHistory.GetApplicationId(), currentApplicationAttemptId
, appHistory.GetUser(), appHistory.GetQueue(), appHistory.GetApplicationName(),
host, rpcPort, null, appHistory.GetYarnApplicationState(), appHistory.GetDiagnosticsInfo
(), trackingUrl, appHistory.GetStartTime(), appHistory.GetFinishTime(), appHistory
.GetFinalApplicationStatus(), null, string.Empty, 100, appHistory.GetApplicationType
(), null));
}
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationAttemptHistoryData GetApplicationAttempt(ApplicationAttemptId
appAttemptId)
{
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appAttemptId.GetApplicationId());
try
{
bool readStartData = false;
bool readFinishData = false;
ApplicationAttemptHistoryData historyData = ApplicationAttemptHistoryData.NewInstance
(appAttemptId, null, -1, null, null, null, FinalApplicationStatus.Undefined, null
);
while ((!readStartData || !readFinishData) && hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.Equals(appAttemptId.ToString()))
{
if (entry.key.suffix.Equals(StartDataSuffix))
{
ApplicationAttemptStartData startData = ParseApplicationAttemptStartData(entry.value
);
MergeApplicationAttemptHistoryData(historyData, startData);
readStartData = true;
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
ApplicationAttemptFinishData finishData = ParseApplicationAttemptFinishData(entry
.value);
MergeApplicationAttemptHistoryData(historyData, finishData);
readFinishData = true;
}
}
}
}
if (!readStartData && !readFinishData)
{
return(null);
}
if (!readStartData)
{
Log.Warn("Start information is missing for application attempt " + appAttemptId);
}
if (!readFinishData)
{
Log.Warn("Finish information is missing for application attempt " + appAttemptId);
}
Log.Info("Completed reading history information of application attempt " + appAttemptId
);
return(historyData);
}
catch (IOException e)
{
Log.Error("Error when reading history file of application attempt" + appAttemptId
, e);
throw;
}
finally
{
hfReader.Close();
}
}