/// <exception cref="System.IO.IOException"/>
public static void CreateSplitFiles(Path jobSubmitDir, Configuration conf, FileSystem
fs, InputSplit[] splits)
{
FSDataOutputStream @out = CreateFile(fs, JobSubmissionFiles.GetJobSplitFile(jobSubmitDir
), conf);
JobSplit.SplitMetaInfo[] info = WriteOldSplits(splits, @out, conf);
@out.Close();
WriteJobSplitMetaInfo(fs, JobSubmissionFiles.GetJobSplitMetaFile(jobSubmitDir), new
FsPermission(JobSubmissionFiles.JobFilePermission), splitVersion, info);
}
/// <exception cref="System.IO.IOException"/>
public static JobSplit.TaskSplitMetaInfo[] ReadSplitMetaInfo(JobID jobId, FileSystem
fs, Configuration conf, Path jobSubmitDir)
{
long maxMetaInfoSize = conf.GetLong(MRJobConfig.SplitMetainfoMaxsize, MRJobConfig
.DefaultSplitMetainfoMaxsize);
Path metaSplitFile = JobSubmissionFiles.GetJobSplitMetaFile(jobSubmitDir);
string jobSplitFile = JobSubmissionFiles.GetJobSplitFile(jobSubmitDir).ToString();
FileStatus fStatus = fs.GetFileStatus(metaSplitFile);
if (maxMetaInfoSize > 0 && fStatus.GetLen() > maxMetaInfoSize)
{
throw new IOException("Split metadata size exceeded " + maxMetaInfoSize + ". Aborting job "
+ jobId);
}
FSDataInputStream @in = fs.Open(metaSplitFile);
byte[] header = new byte[JobSplit.MetaSplitFileHeader.Length];
@in.ReadFully(header);
if (!Arrays.Equals(JobSplit.MetaSplitFileHeader, header))
{
throw new IOException("Invalid header on split file");
}
int vers = WritableUtils.ReadVInt(@in);
if (vers != JobSplit.MetaSplitVersion)
{
@in.Close();
throw new IOException("Unsupported split version " + vers);
}
int numSplits = WritableUtils.ReadVInt(@in);
//TODO: check for insane values
JobSplit.TaskSplitMetaInfo[] allSplitMetaInfo = new JobSplit.TaskSplitMetaInfo[numSplits
];
for (int i = 0; i < numSplits; i++)
{
JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo();
splitMetaInfo.ReadFields(@in);
JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(jobSplitFile, splitMetaInfo
.GetStartOffset());
allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex, splitMetaInfo.GetLocations
(), splitMetaInfo.GetInputDataLength());
}
@in.Close();
return(allSplitMetaInfo);
}
