public virtual void TestJobHistoryFileNameBackwardsCompatible()
{
JobID oldJobId = JobID.ForName(JobId);
JobId jobId = TypeConverter.ToYarn(oldJobId);
long submitTime = long.Parse(SubmitTime);
long finishTime = long.Parse(FinishTime);
int numMaps = System.Convert.ToInt32(NumMaps);
int numReduces = System.Convert.ToInt32(NumReduces);
string jobHistoryFile = string.Format(OldJobHistoryFileFormatter, JobId, SubmitTime
, UserName, JobName, FinishTime, NumMaps, NumReduces, JobStatus);
JobIndexInfo info = FileNameIndexUtils.GetIndexInfo(jobHistoryFile);
NUnit.Framework.Assert.AreEqual("Job id incorrect after decoding old history file"
, jobId, info.GetJobId());
NUnit.Framework.Assert.AreEqual("Submit time incorrect after decoding old history file"
, submitTime, info.GetSubmitTime());
NUnit.Framework.Assert.AreEqual("User incorrect after decoding old history file",
UserName, info.GetUser());
NUnit.Framework.Assert.AreEqual("Job name incorrect after decoding old history file"
, JobName, info.GetJobName());
NUnit.Framework.Assert.AreEqual("Finish time incorrect after decoding old history file"
, finishTime, info.GetFinishTime());
NUnit.Framework.Assert.AreEqual("Num maps incorrect after decoding old history file"
, numMaps, info.GetNumMaps());
NUnit.Framework.Assert.AreEqual("Num reduces incorrect after decoding old history file"
, numReduces, info.GetNumReduces());
NUnit.Framework.Assert.AreEqual("Job status incorrect after decoding old history file"
, JobStatus, info.GetJobStatus());
NUnit.Framework.Assert.IsNull("Queue name incorrect after decoding old history file"
, info.GetQueueName());
}
/// <summary>Returns the jobId from a job history file name.</summary>
/// <param name="pathString">the path string.</param>
/// <returns>the JobId</returns>
/// <exception cref="System.IO.IOException">if the filename format is invalid.</exception>
public static JobID GetJobIDFromHistoryFilePath(string pathString)
{
string[] parts = pathString.Split(Path.Separator);
string fileNamePart = parts[parts.Length - 1];
JobIndexInfo jobIndexInfo = FileNameIndexUtils.GetIndexInfo(fileNamePart);
return(TypeConverter.FromYarn(jobIndexInfo.GetJobId()));
}
public virtual void TestJobStartTimeBackwardsCompatible()
{
string jobHistoryFile = string.Format(OldFormatBeforeAddStartTime, JobId, SubmitTime
, UserName, JobNameWithDelimiterEscape, FinishTime, NumMaps, NumReduces, JobStatus
, QueueName);
JobIndexInfo info = FileNameIndexUtils.GetIndexInfo(jobHistoryFile);
NUnit.Framework.Assert.AreEqual(info.GetJobStartTime(), info.GetSubmitTime());
}
public virtual void TestQueueNamePercentDecoding()
{
string jobHistoryFile = string.Format(JobHistoryFileFormatter, JobId, SubmitTime,
UserName, JobName, FinishTime, NumMaps, NumReduces, JobStatus, QueueNameWithDelimiterEscape
, JobStartTime);
JobIndexInfo info = FileNameIndexUtils.GetIndexInfo(jobHistoryFile);
NUnit.Framework.Assert.AreEqual("Queue name doesn't match", QueueNameWithDelimiter
, info.GetQueueName());
}
public virtual void TestQueueNamePercentEncoding()
{
JobIndexInfo info = new JobIndexInfo();
JobID oldJobId = JobID.ForName(JobId);
JobId jobId = TypeConverter.ToYarn(oldJobId);
info.SetJobId(jobId);
info.SetSubmitTime(long.Parse(SubmitTime));
info.SetUser(UserName);
info.SetJobName(JobName);
info.SetFinishTime(long.Parse(FinishTime));
info.SetNumMaps(System.Convert.ToInt32(NumMaps));
info.SetNumReduces(System.Convert.ToInt32(NumReduces));
info.SetJobStatus(JobStatus);
info.SetQueueName(QueueNameWithDelimiter);
info.SetJobStartTime(long.Parse(JobStartTime));
string jobHistoryFile = FileNameIndexUtils.GetDoneFileName(info);
NUnit.Framework.Assert.IsTrue("Queue name not encoded correctly into job history file"
, jobHistoryFile.Contains(QueueNameWithDelimiterEscape));
}
public virtual void TestEncodingDecodingEquivalence()
{
JobIndexInfo info = new JobIndexInfo();
JobID oldJobId = JobID.ForName(JobId);
JobId jobId = TypeConverter.ToYarn(oldJobId);
info.SetJobId(jobId);
info.SetSubmitTime(long.Parse(SubmitTime));
info.SetUser(UserName);
info.SetJobName(JobName);
info.SetFinishTime(long.Parse(FinishTime));
info.SetNumMaps(System.Convert.ToInt32(NumMaps));
info.SetNumReduces(System.Convert.ToInt32(NumReduces));
info.SetJobStatus(JobStatus);
info.SetQueueName(QueueName);
info.SetJobStartTime(long.Parse(JobStartTime));
string jobHistoryFile = FileNameIndexUtils.GetDoneFileName(info);
JobIndexInfo parsedInfo = FileNameIndexUtils.GetIndexInfo(jobHistoryFile);
NUnit.Framework.Assert.AreEqual("Job id different after encoding and decoding", info
.GetJobId(), parsedInfo.GetJobId());
NUnit.Framework.Assert.AreEqual("Submit time different after encoding and decoding"
, info.GetSubmitTime(), parsedInfo.GetSubmitTime());
NUnit.Framework.Assert.AreEqual("User different after encoding and decoding", info
.GetUser(), parsedInfo.GetUser());
NUnit.Framework.Assert.AreEqual("Job name different after encoding and decoding",
info.GetJobName(), parsedInfo.GetJobName());
NUnit.Framework.Assert.AreEqual("Finish time different after encoding and decoding"
, info.GetFinishTime(), parsedInfo.GetFinishTime());
NUnit.Framework.Assert.AreEqual("Num maps different after encoding and decoding",
info.GetNumMaps(), parsedInfo.GetNumMaps());
NUnit.Framework.Assert.AreEqual("Num reduces different after encoding and decoding"
, info.GetNumReduces(), parsedInfo.GetNumReduces());
NUnit.Framework.Assert.AreEqual("Job status different after encoding and decoding"
, info.GetJobStatus(), parsedInfo.GetJobStatus());
NUnit.Framework.Assert.AreEqual("Queue name different after encoding and decoding"
, info.GetQueueName(), parsedInfo.GetQueueName());
NUnit.Framework.Assert.AreEqual("Job start time different after encoding and decoding"
, info.GetJobStartTime(), parsedInfo.GetJobStartTime());
}
// Sanitize job history file for predictable parsing
// Job history file names need to be backwards compatible
// Only append new elements to the end of this list
/// <summary>Constructs the job history file name from the JobIndexInfo.</summary>
/// <param name="indexInfo">the index info.</param>
/// <returns>the done job history filename.</returns>
/// <exception cref="System.IO.IOException"/>
public static string GetDoneFileName(JobIndexInfo indexInfo)
{
StringBuilder sb = new StringBuilder();
//JobId
sb.Append(EscapeDelimiters(TypeConverter.FromYarn(indexInfo.GetJobId()).ToString(
)));
sb.Append(Delimiter);
//SubmitTime
sb.Append(indexInfo.GetSubmitTime());
sb.Append(Delimiter);
//UserName
sb.Append(EscapeDelimiters(GetUserName(indexInfo)));
sb.Append(Delimiter);
//JobName
sb.Append(EscapeDelimiters(TrimJobName(GetJobName(indexInfo))));
sb.Append(Delimiter);
//FinishTime
sb.Append(indexInfo.GetFinishTime());
sb.Append(Delimiter);
//NumMaps
sb.Append(indexInfo.GetNumMaps());
sb.Append(Delimiter);
//NumReduces
sb.Append(indexInfo.GetNumReduces());
sb.Append(Delimiter);
//JobStatus
sb.Append(indexInfo.GetJobStatus());
sb.Append(Delimiter);
//QueueName
sb.Append(EscapeDelimiters(GetQueueName(indexInfo)));
sb.Append(Delimiter);
//JobStartTime
sb.Append(indexInfo.GetJobStartTime());
sb.Append(JobHistoryUtils.JobHistoryFileExtension);
return(EncodeJobHistoryFileName(sb.ToString()));
}
/// <summary>
/// Parses the provided job history file name to construct a
/// JobIndexInfo object which is returned.
/// </summary>
/// <param name="jhFileName">the job history filename.</param>
/// <returns>a JobIndexInfo object built from the filename.</returns>
/// <exception cref="System.IO.IOException"/>
public static JobIndexInfo GetIndexInfo(string jhFileName)
{
string fileName = Sharpen.Runtime.Substring(jhFileName, 0, jhFileName.IndexOf(JobHistoryUtils
.JobHistoryFileExtension));
JobIndexInfo indexInfo = new JobIndexInfo();
string[] jobDetails = fileName.Split(Delimiter);
JobID oldJobId = JobID.ForName(DecodeJobHistoryFileName(jobDetails[JobIdIndex]));
JobId jobId = TypeConverter.ToYarn(oldJobId);
indexInfo.SetJobId(jobId);
// Do not fail if there are some minor parse errors
try
{
try
{
indexInfo.SetSubmitTime(long.Parse(DecodeJobHistoryFileName(jobDetails[SubmitTimeIndex
])));
}
catch (FormatException e)
{
Log.Warn("Unable to parse submit time from job history file " + jhFileName + " : "
+ e);
}
indexInfo.SetUser(DecodeJobHistoryFileName(jobDetails[UserIndex]));
indexInfo.SetJobName(DecodeJobHistoryFileName(jobDetails[JobNameIndex]));
try
{
indexInfo.SetFinishTime(long.Parse(DecodeJobHistoryFileName(jobDetails[FinishTimeIndex
])));
}
catch (FormatException e)
{
Log.Warn("Unable to parse finish time from job history file " + jhFileName + " : "
+ e);
}
try
{
indexInfo.SetNumMaps(System.Convert.ToInt32(DecodeJobHistoryFileName(jobDetails[NumMapsIndex
])));
}
catch (FormatException e)
{
Log.Warn("Unable to parse num maps from job history file " + jhFileName + " : " +
e);
}
try
{
indexInfo.SetNumReduces(System.Convert.ToInt32(DecodeJobHistoryFileName(jobDetails
[NumReducesIndex])));
}
catch (FormatException e)
{
Log.Warn("Unable to parse num reduces from job history file " + jhFileName + " : "
+ e);
}
indexInfo.SetJobStatus(DecodeJobHistoryFileName(jobDetails[JobStatusIndex]));
indexInfo.SetQueueName(DecodeJobHistoryFileName(jobDetails[QueueNameIndex]));
try
{
if (jobDetails.Length <= JobStartTimeIndex)
{
indexInfo.SetJobStartTime(indexInfo.GetSubmitTime());
}
else
{
indexInfo.SetJobStartTime(long.Parse(DecodeJobHistoryFileName(jobDetails[JobStartTimeIndex
])));
}
}
catch (FormatException e)
{
Log.Warn("Unable to parse start time from job history file " + jhFileName + " : "
+ e);
}
}
catch (IndexOutOfRangeException)
{
Log.Warn("Parsing job history file with partial data encoded into name: " + jhFileName
);
}
return(indexInfo);
}
private static string GetQueueName(JobIndexInfo indexInfo)
{
return(GetNonEmptyString(indexInfo.GetQueueName()));
}