/// <exception cref="System.IO.IOException"/>
public virtual IDictionary <ApplicationId, ApplicationHistoryData> GetAllApplications
()
{
IDictionary <ApplicationId, ApplicationHistoryData> historyDataMap = new Dictionary
<ApplicationId, ApplicationHistoryData>();
FileStatus[] files = fs.ListStatus(rootDirPath);
foreach (FileStatus file in files)
{
ApplicationId appId = ConverterUtils.ToApplicationId(file.GetPath().GetName());
try
{
ApplicationHistoryData historyData = GetApplication(appId);
if (historyData != null)
{
historyDataMap[appId] = historyData;
}
}
catch (IOException e)
{
// Eat the exception not to disturb the getting the next
// ApplicationHistoryData
Log.Error("History information of application " + appId + " is not included into the result due to the exception"
, e);
}
}
return(historyDataMap);
}
private static void MergeApplicationHistoryData(ApplicationHistoryData historyData
, ApplicationFinishData finishData)
{
historyData.SetFinishTime(finishData.GetFinishTime());
historyData.SetDiagnosticsInfo(finishData.GetDiagnosticsInfo());
historyData.SetFinalApplicationStatus(finishData.GetFinalApplicationStatus());
historyData.SetYarnApplicationState(finishData.GetYarnApplicationState());
}
/// <exception cref="System.IO.IOException"/>
public virtual ApplicationHistoryData GetApplication(ApplicationId appId)
{
FileSystemApplicationHistoryStore.HistoryFileReader hfReader = GetHistoryFileReader
(appId);
try
{
bool readStartData = false;
bool readFinishData = false;
ApplicationHistoryData historyData = ApplicationHistoryData.NewInstance(appId, null
, null, null, null, long.MinValue, long.MinValue, long.MaxValue, null, FinalApplicationStatus
.Undefined, null);
while ((!readStartData || !readFinishData) && hfReader.HasNext())
{
FileSystemApplicationHistoryStore.HistoryFileReader.Entry entry = hfReader.Next();
if (entry.key.id.Equals(appId.ToString()))
{
if (entry.key.suffix.Equals(StartDataSuffix))
{
ApplicationStartData startData = ParseApplicationStartData(entry.value);
MergeApplicationHistoryData(historyData, startData);
readStartData = true;
}
else
{
if (entry.key.suffix.Equals(FinishDataSuffix))
{
ApplicationFinishData finishData = ParseApplicationFinishData(entry.value);
MergeApplicationHistoryData(historyData, finishData);
readFinishData = true;
}
}
}
}
if (!readStartData && !readFinishData)
{
return(null);
}
if (!readStartData)
{
Log.Warn("Start information is missing for application " + appId);
}
if (!readFinishData)
{
Log.Warn("Finish information is missing for application " + appId);
}
Log.Info("Completed reading history information of application " + appId);
return(historyData);
}
catch (IOException e)
{
Log.Error("Error when reading history file of application " + appId, e);
throw;
}
finally
{
hfReader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void TestReadHistoryData(int num, bool missingContainer, bool missingApplicationAttempt
)
{
// read application history data
NUnit.Framework.Assert.AreEqual(num, store.GetAllApplications().Count);
for (int i = 1; i <= num; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
ApplicationHistoryData appData = store.GetApplication(appId);
NUnit.Framework.Assert.IsNotNull(appData);
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetApplicationName());
NUnit.Framework.Assert.AreEqual(appId.ToString(), appData.GetDiagnosticsInfo());
// read application attempt history data
NUnit.Framework.Assert.AreEqual(num, store.GetApplicationAttempts(appId).Count);
for (int j = 1; j <= num; ++j)
{
ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(appId, j);
ApplicationAttemptHistoryData attemptData = store.GetApplicationAttempt(appAttemptId
);
NUnit.Framework.Assert.IsNotNull(attemptData);
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetHost());
if (missingApplicationAttempt && j == num)
{
NUnit.Framework.Assert.IsNull(attemptData.GetDiagnosticsInfo());
continue;
}
else
{
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), attemptData.GetDiagnosticsInfo
());
}
// read container history data
NUnit.Framework.Assert.AreEqual(num, store.GetContainers(appAttemptId).Count);
for (int k = 1; k <= num; ++k)
{
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, k);
ContainerHistoryData containerData = store.GetContainer(containerId);
NUnit.Framework.Assert.IsNotNull(containerData);
NUnit.Framework.Assert.AreEqual(Priority.NewInstance(containerId.GetId()), containerData
.GetPriority());
if (missingContainer && k == num)
{
NUnit.Framework.Assert.IsNull(containerData.GetDiagnosticsInfo());
}
else
{
NUnit.Framework.Assert.AreEqual(containerId.ToString(), containerData.GetDiagnosticsInfo
());
}
}
ContainerHistoryData masterContainer = store.GetAMContainer(appAttemptId);
NUnit.Framework.Assert.IsNotNull(masterContainer);
NUnit.Framework.Assert.AreEqual(ContainerId.NewContainerId(appAttemptId, 1), masterContainer
.GetContainerId());
}
}
}
private static void MergeApplicationHistoryData(ApplicationHistoryData historyData
, ApplicationStartData startData)
{
historyData.SetApplicationName(startData.GetApplicationName());
historyData.SetApplicationType(startData.GetApplicationType());
historyData.SetQueue(startData.GetQueue());
historyData.SetUser(startData.GetUser());
historyData.SetSubmitTime(startData.GetSubmitTime());
historyData.SetStartTime(startData.GetStartTime());
}
public virtual void TestReadWriteApplicationHistory()
{
// Out of order
ApplicationId appId = ApplicationId.NewInstance(0, 1);
try
{
WriteApplicationFinishData(appId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is stored before the start information"
));
}
// Normal
int numApps = 5;
for (int i = 1; i <= numApps; ++i)
{
appId = ApplicationId.NewInstance(0, i);
WriteApplicationStartData(appId);
WriteApplicationFinishData(appId);
}
NUnit.Framework.Assert.AreEqual(numApps, store.GetAllApplications().Count);
for (int i_1 = 1; i_1 <= numApps; ++i_1)
{
appId = ApplicationId.NewInstance(0, i_1);
ApplicationHistoryData data = store.GetApplication(appId);
NUnit.Framework.Assert.IsNotNull(data);
NUnit.Framework.Assert.AreEqual(appId.ToString(), data.GetApplicationName());
NUnit.Framework.Assert.AreEqual(appId.ToString(), data.GetDiagnosticsInfo());
}
// Write again
appId = ApplicationId.NewInstance(0, 1);
try
{
WriteApplicationStartData(appId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is already stored"));
}
try
{
WriteApplicationFinishData(appId);
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is already stored"));
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ApplicationStarted(ApplicationStartData appStart)
{
ApplicationHistoryData oldData = applicationData.PutIfAbsent(appStart.GetApplicationId
(), ApplicationHistoryData.NewInstance(appStart.GetApplicationId(), appStart.GetApplicationName
(), appStart.GetApplicationType(), appStart.GetQueue(), appStart.GetUser(), appStart
.GetSubmitTime(), appStart.GetStartTime(), long.MaxValue, null, null, null));
if (oldData != null)
{
throw new IOException("The start information of application " + appStart.GetApplicationId
() + " is already stored.");
}
}
public virtual void TestWriteApplication()
{
RMApp app = CreateRMApp(ApplicationId.NewInstance(0, 1));
writer.ApplicationStarted(app);
ApplicationHistoryData appHD = null;
for (int i = 0; i < MaxRetries; ++i)
{
appHD = store.GetApplication(ApplicationId.NewInstance(0, 1));
if (appHD != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.IsNotNull(appHD);
NUnit.Framework.Assert.AreEqual("test app", appHD.GetApplicationName());
NUnit.Framework.Assert.AreEqual("test app type", appHD.GetApplicationType());
NUnit.Framework.Assert.AreEqual("test user", appHD.GetUser());
NUnit.Framework.Assert.AreEqual("test queue", appHD.GetQueue());
NUnit.Framework.Assert.AreEqual(0L, appHD.GetSubmitTime());
NUnit.Framework.Assert.AreEqual(1L, appHD.GetStartTime());
writer.ApplicationFinished(app, RMAppState.Finished);
for (int i_1 = 0; i_1 < MaxRetries; ++i_1)
{
appHD = store.GetApplication(ApplicationId.NewInstance(0, 1));
if (appHD.GetYarnApplicationState() != null)
{
break;
}
else
{
Sharpen.Thread.Sleep(100);
}
}
NUnit.Framework.Assert.AreEqual(2L, appHD.GetFinishTime());
NUnit.Framework.Assert.AreEqual("test diagnostics info", appHD.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Undefined, appHD.GetFinalApplicationStatus
());
NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished, appHD.GetYarnApplicationState
());
}
/// <exception cref="System.IO.IOException"/>
public virtual void ApplicationFinished(ApplicationFinishData appFinish)
{
ApplicationHistoryData data = applicationData[appFinish.GetApplicationId()];
if (data == null)
{
throw new IOException("The finish information of application " + appFinish.GetApplicationId
() + " is stored before the start" + " information.");
}
// Make the assumption that YarnApplicationState should not be null if
// the finish information is already recorded
if (data.GetYarnApplicationState() != null)
{
throw new IOException("The finish information of application " + appFinish.GetApplicationId
() + " is already stored.");
}
data.SetFinishTime(appFinish.GetFinishTime());
data.SetDiagnosticsInfo(appFinish.GetDiagnosticsInfo());
data.SetFinalApplicationStatus(appFinish.GetFinalApplicationStatus());
data.SetYarnApplicationState(appFinish.GetYarnApplicationState());
}
/// <exception cref="System.IO.IOException"/>
private ApplicationReport ConvertToApplicationReport(ApplicationHistoryData appHistory
)
{
ApplicationAttemptId currentApplicationAttemptId = null;
string trackingUrl = Unavailable;
string host = Unavailable;
int rpcPort = -1;
ApplicationAttemptHistoryData lastAttempt = GetLastAttempt(appHistory.GetApplicationId
());
if (lastAttempt != null)
{
currentApplicationAttemptId = lastAttempt.GetApplicationAttemptId();
trackingUrl = lastAttempt.GetTrackingURL();
host = lastAttempt.GetHost();
rpcPort = lastAttempt.GetRPCPort();
}
return(ApplicationReport.NewInstance(appHistory.GetApplicationId(), currentApplicationAttemptId
, appHistory.GetUser(), appHistory.GetQueue(), appHistory.GetApplicationName(),
host, rpcPort, null, appHistory.GetYarnApplicationState(), appHistory.GetDiagnosticsInfo
(), trackingUrl, appHistory.GetStartTime(), appHistory.GetFinishTime(), appHistory
.GetFinalApplicationStatus(), null, string.Empty, 100, appHistory.GetApplicationType
(), null));
}
