/// <summary> /// Run a test with several mappers in parallel, operating at different /// speeds. /// </summary> /// <remarks> /// Run a test with several mappers in parallel, operating at different /// speeds. Verify that the correct amount of output is created. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestMultiMaps() { Job job = Job.GetInstance(); Path inputPath = CreateMultiMapsInput(); Path outputPath = GetOutputPath(); Configuration conf = new Configuration(); FileSystem fs = FileSystem.GetLocal(conf); if (fs.Exists(outputPath)) { fs.Delete(outputPath, true); } job.SetMapperClass(typeof(TestLocalRunner.StressMapper)); job.SetReducerClass(typeof(TestLocalRunner.CountingReducer)); job.SetNumReduceTasks(1); LocalJobRunner.SetLocalMaxRunningMaps(job, 6); job.GetConfiguration().Set(MRJobConfig.IoSortMb, "25"); FileInputFormat.AddInputPath(job, inputPath); FileOutputFormat.SetOutputPath(job, outputPath); Sharpen.Thread toInterrupt = Sharpen.Thread.CurrentThread(); Sharpen.Thread interrupter = new _Thread_311(toInterrupt); // 2m Log.Info("Submitting job..."); job.Submit(); Log.Info("Starting thread to interrupt main thread in 2 minutes"); interrupter.Start(); Log.Info("Waiting for job to complete..."); try { job.WaitForCompletion(true); } catch (Exception ie) { Log.Fatal("Interrupted while waiting for job completion", ie); for (int i = 0; i < 10; i++) { Log.Fatal("Dumping stacks"); ReflectionUtils.LogThreadInfo(Log, "multimap threads", 0); Sharpen.Thread.Sleep(1000); } throw; } Log.Info("Job completed, stopping interrupter"); interrupter.Interrupt(); try { interrupter.Join(); } catch (Exception) { } // it might interrupt us right as we interrupt it Log.Info("Verifying output"); VerifyOutput(outputPath); }
public virtual void TestInvalidMultiMapParallelism() { Job job = Job.GetInstance(); Path inputPath = CreateMultiMapsInput(); Path outputPath = GetOutputPath(); Configuration conf = new Configuration(); FileSystem fs = FileSystem.GetLocal(conf); if (fs.Exists(outputPath)) { fs.Delete(outputPath, true); } job.SetMapperClass(typeof(TestLocalRunner.StressMapper)); job.SetReducerClass(typeof(TestLocalRunner.CountingReducer)); job.SetNumReduceTasks(1); LocalJobRunner.SetLocalMaxRunningMaps(job, -6); FileInputFormat.AddInputPath(job, inputPath); FileOutputFormat.SetOutputPath(job, outputPath); bool success = job.WaitForCompletion(true); NUnit.Framework.Assert.IsFalse("Job succeeded somehow", success); }
/// <summary> /// Run a test which creates a SequenceMapper / IdentityReducer /// job over a set of generated number files. /// </summary> /// <exception cref="System.Exception"/> private void DoMultiReducerTest(int numMaps, int numReduces, int parallelMaps, int parallelReduces) { Path @in = GetNumberDirPath(); Path @out = GetOutputPath(); // Clear data from any previous tests. Configuration conf = new Configuration(); FileSystem fs = FileSystem.GetLocal(conf); if (fs.Exists(@out)) { fs.Delete(@out, true); } if (fs.Exists(@in)) { fs.Delete(@in, true); } for (int i = 0; i < numMaps; i++) { MakeNumberFile(i, 100); } Job job = Job.GetInstance(); job.SetNumReduceTasks(numReduces); job.SetMapperClass(typeof(TestLocalRunner.SequenceMapper)); job.SetOutputKeyClass(typeof(Text)); job.SetOutputValueClass(typeof(NullWritable)); FileInputFormat.AddInputPath(job, @in); FileOutputFormat.SetOutputPath(job, @out); LocalJobRunner.SetLocalMaxRunningMaps(job, parallelMaps); LocalJobRunner.SetLocalMaxRunningReduces(job, parallelReduces); bool result = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job failed!!", result); VerifyNumberJob(numMaps); }