/// <summary>
/// Run a test with several mappers in parallel, operating at different
/// speeds.
/// </summary>
/// <remarks>
/// Run a test with several mappers in parallel, operating at different
/// speeds. Verify that the correct amount of output is created.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestMultiMaps()
{
Job job = Job.GetInstance();
Path inputPath = CreateMultiMapsInput();
Path outputPath = GetOutputPath();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.GetLocal(conf);
if (fs.Exists(outputPath))
{
fs.Delete(outputPath, true);
}
job.SetMapperClass(typeof(TestLocalRunner.StressMapper));
job.SetReducerClass(typeof(TestLocalRunner.CountingReducer));
job.SetNumReduceTasks(1);
LocalJobRunner.SetLocalMaxRunningMaps(job, 6);
job.GetConfiguration().Set(MRJobConfig.IoSortMb, "25");
FileInputFormat.AddInputPath(job, inputPath);
FileOutputFormat.SetOutputPath(job, outputPath);
Sharpen.Thread toInterrupt = Sharpen.Thread.CurrentThread();
Sharpen.Thread interrupter = new _Thread_311(toInterrupt);
// 2m
Log.Info("Submitting job...");
job.Submit();
Log.Info("Starting thread to interrupt main thread in 2 minutes");
interrupter.Start();
Log.Info("Waiting for job to complete...");
try
{
job.WaitForCompletion(true);
}
catch (Exception ie)
{
Log.Fatal("Interrupted while waiting for job completion", ie);
for (int i = 0; i < 10; i++)
{
Log.Fatal("Dumping stacks");
ReflectionUtils.LogThreadInfo(Log, "multimap threads", 0);
Sharpen.Thread.Sleep(1000);
}
throw;
}
Log.Info("Job completed, stopping interrupter");
interrupter.Interrupt();
try
{
interrupter.Join();
}
catch (Exception)
{
}
// it might interrupt us right as we interrupt it
Log.Info("Verifying output");
VerifyOutput(outputPath);
}
public virtual void TestInvalidMultiMapParallelism()
{
Job job = Job.GetInstance();
Path inputPath = CreateMultiMapsInput();
Path outputPath = GetOutputPath();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.GetLocal(conf);
if (fs.Exists(outputPath))
{
fs.Delete(outputPath, true);
}
job.SetMapperClass(typeof(TestLocalRunner.StressMapper));
job.SetReducerClass(typeof(TestLocalRunner.CountingReducer));
job.SetNumReduceTasks(1);
LocalJobRunner.SetLocalMaxRunningMaps(job, -6);
FileInputFormat.AddInputPath(job, inputPath);
FileOutputFormat.SetOutputPath(job, outputPath);
bool success = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsFalse("Job succeeded somehow", success);
}
/// <summary>
/// Run a test which creates a SequenceMapper / IdentityReducer
/// job over a set of generated number files.
/// </summary>
/// <exception cref="System.Exception"/>
private void DoMultiReducerTest(int numMaps, int numReduces, int parallelMaps, int
parallelReduces)
{
Path @in = GetNumberDirPath();
Path @out = GetOutputPath();
// Clear data from any previous tests.
Configuration conf = new Configuration();
FileSystem fs = FileSystem.GetLocal(conf);
if (fs.Exists(@out))
{
fs.Delete(@out, true);
}
if (fs.Exists(@in))
{
fs.Delete(@in, true);
}
for (int i = 0; i < numMaps; i++)
{
MakeNumberFile(i, 100);
}
Job job = Job.GetInstance();
job.SetNumReduceTasks(numReduces);
job.SetMapperClass(typeof(TestLocalRunner.SequenceMapper));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(NullWritable));
FileInputFormat.AddInputPath(job, @in);
FileOutputFormat.SetOutputPath(job, @out);
LocalJobRunner.SetLocalMaxRunningMaps(job, parallelMaps);
LocalJobRunner.SetLocalMaxRunningReduces(job, parallelReduces);
bool result = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed!!", result);
VerifyNumberJob(numMaps);
}