//method to write splits for old api mapper.
/// <exception cref="System.IO.IOException"/>
private int WriteOldSplits(JobConf job, Path jobSubmitDir)
{
InputSplit[] splits = job.GetInputFormat().GetSplits(job, job.GetNumMapTasks());
// sort the splits into order based on size, so that the biggest
// go first
Arrays.Sort(splits, new _IComparer_332());
JobSplitWriter.CreateSplitFiles(jobSubmitDir, job, jobSubmitDir.GetFileSystem(job
), splits);
return(splits.Length);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
private int WriteNewSplits <T>(JobContext job, Path jobSubmitDir)
where T : InputSplit
{
Configuration conf = job.GetConfiguration();
InputFormat <object, object> input = ReflectionUtils.NewInstance(job.GetInputFormatClass
(), conf);
IList <InputSplit> splits = input.GetSplits(job);
T[] array = (T[])Sharpen.Collections.ToArray(splits, new InputSplit[splits.Count]
);
// sort the splits into order based on size, so that the biggest
// go first
Arrays.Sort(array, new JobSubmitter.SplitComparator());
JobSplitWriter.CreateSplitFiles(jobSubmitDir, conf, jobSubmitDir.GetFileSystem(conf
), array);
return(array.Length);
}
/// <summary>
/// Validates map phase progress after each record is processed by map task
/// using custom task reporter.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestMapProgress()
{
JobConf job = new JobConf();
fs = FileSystem.GetLocal(job);
Path rootDir = new Path(TestRootDir);
CreateInputFile(rootDir);
job.SetNumReduceTasks(0);
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0"
));
job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof(
OutputFormat));
job.Set(FileInputFormat.InputDir, TestRootDir);
jobId = ((JobID)taskId.GetJobID());
JobContext jContext = new JobContextImpl(job, jobId);
InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass
(), job);
IList <InputSplit> splits = input.GetSplits(jContext);
JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir)
.GetFileSystem(job), splits);
JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, fs, job, new Path(TestRootDir));
job.SetUseNewMapper(true);
// use new api
for (int i = 0; i < splitMetaInfo.Length; i++)
{
// rawSplits.length is 1
map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system"
) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1);
JobConf localConf = new JobConf(job);
map.LocalizeConfiguration(localConf);
map.SetConf(localConf);
map.Run(localConf, fakeUmbilical);
}
// clean up
fs.Delete(rootDir, true);
}
