public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, RecordWriter <KEYOUT, VALUEOUT> output, OutputCommitter committer, StatusReporter reporter) : base(conf, taskid, reporter) { this.output = output; this.committer = committer; }
public TaskRecoverEvent(TaskId taskID, JobHistoryParser.TaskInfo taskInfo, OutputCommitter committer, bool recoverTaskOutput) : base(taskID, TaskEventType.TRecover) { this.taskInfo = taskInfo; this.committer = committer; this.recoverTaskOutput = recoverTaskOutput; }
public TaskAttemptRecoverEvent(TaskAttemptId id, JobHistoryParser.TaskAttemptInfo taInfo, OutputCommitter committer, bool recoverOutput) : base(id, TaskAttemptEventType.TaRecover) { this.taInfo = taInfo; this.committer = committer; this.recoverAttemptOutput = recoverOutput; }
/// <exception cref="System.IO.IOException"/> public override OutputCommitter GetOutputCommitter(TaskAttemptContext context) { if (committer == null) { Path output = GetOutputPath(context); committer = new FileOutputCommitter(output, context); } return(committer); }
protected internal override EventHandler <CommitterEvent> CreateCommitterEventHandler (AppContext context, OutputCommitter committer) { // create an output committer with the task methods stubbed out OutputCommitter stubbedCommitter = new _OutputCommitter_613(committer); return(new CommitterEventHandler(context, stubbedCommitter, GetRMHeartbeatHandler ())); }
public CommitterEventHandler(AppContext context, OutputCommitter committer, RMHeartbeatHandler rmHeartbeatHandler, ClassLoader jobClassLoader) : base("CommitterEventHandler") { this.context = context; this.committer = committer; this.rmHeartbeatHandler = rmHeartbeatHandler; this.stopped = new AtomicBoolean(false); this.jobClassLoader = jobClassLoader; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public Void Call() { OutputCommitter outputCommitter = tof[taskIdx].GetOutputCommitter(taCtx[taskIdx]); outputCommitter.SetupTask(taCtx[taskIdx]); RecordWriter rw = tof[taskIdx].GetRecordWriter(taCtx[taskIdx]); this._enclosing.WriteOutput(rw, taCtx[taskIdx]); outputCommitter.CommitTask(taCtx[taskIdx]); return(null); }
/// <exception cref="System.IO.IOException"/> public override OutputCommitter GetOutputCommitter(TaskAttemptContext context) { lock (this) { if (committer == null) { Path output = GetOutputPath(context); committer = new TestJobOutputCommitter.CommitterWithCustomAbort(output, context); } return(committer); } }
public virtual void TestBasic() { AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>(); OutputCommitter mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>(); Clock mockClock = Org.Mockito.Mockito.Mock <Clock>(); CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter , new TestCommitterEventHandler.TestingRMHeartbeatHandler()); YarnConfiguration conf = new YarnConfiguration(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); JobContext mockJobContext = Org.Mockito.Mockito.Mock <JobContext>(); ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0" ); JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId ())); TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler (); Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId ()); Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid ); Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler ); Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock); handler.Init(conf); handler.Start(); try { handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext)); string user = UserGroupInformation.GetCurrentUser().GetShortUserName(); Path startCommitFile = MRApps.GetStartJobCommitFile(conf, user, jobId); Path endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId); Path endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId); Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent(); NUnit.Framework.Assert.IsNotNull(e); NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent); FileSystem fs = FileSystem.Get(conf); NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile )); NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile )); NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile )); Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>()); } finally { handler.Stop(); } }
public TestJob(MRApp _enclosing, JobId jobId, ApplicationAttemptId applicationAttemptId , Configuration conf, EventHandler eventHandler, TaskAttemptListener taskAttemptListener , Clock clock, OutputCommitter committer, bool newApiCommitter, string user, AppContext appContext, JobStateInternal forcedState, string diagnostic) : base(jobId, MRApp.GetApplicationAttemptId(MRApp.applicationId, this._enclosing. GetStartCount()), conf, eventHandler, taskAttemptListener, new JobTokenSecretManager (), new Credentials(), clock, this._enclosing.GetCompletedTaskFromPreviousRun(), this._enclosing.metrics, committer, newApiCommitter, user, Runtime.CurrentTimeMillis (), this._enclosing.GetAllAMInfos(), appContext, forcedState, diagnostic) { this._enclosing = _enclosing; // This "this leak" is okay because the retained pointer is in an // instance variable. this.localStateMachine = this.localFactory.Make(this); }
protected internal override EventHandler<CommitterEvent> CreateCommitterEventHandler (AppContext context, OutputCommitter committer) { return mockCommitterEventHandler; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestBinary() { Configuration conf = new Configuration(); Job job = Job.GetInstance(conf); Path outdir = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "outseq"); Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); FileOutputFormat.SetOutputPath(job, outdir); SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputKeyClass(job, typeof(IntWritable )); SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputValueClass(job, typeof(DoubleWritable )); SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true); SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType .Block); BytesWritable bkey = new BytesWritable(); BytesWritable bval = new BytesWritable(); TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job .GetConfiguration()); OutputFormat <BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat (); OutputCommitter committer = outputFormat.GetOutputCommitter(context); committer.SetupJob(job); RecordWriter <BytesWritable, BytesWritable> writer = outputFormat.GetRecordWriter( context); IntWritable iwritable = new IntWritable(); DoubleWritable dwritable = new DoubleWritable(); DataOutputBuffer outbuf = new DataOutputBuffer(); Log.Info("Creating data by SequenceFileAsBinaryOutputFormat"); try { for (int i = 0; i < Records; ++i) { iwritable = new IntWritable(r.Next()); iwritable.Write(outbuf); bkey.Set(outbuf.GetData(), 0, outbuf.GetLength()); outbuf.Reset(); dwritable = new DoubleWritable(r.NextDouble()); dwritable.Write(outbuf); bval.Set(outbuf.GetData(), 0, outbuf.GetLength()); outbuf.Reset(); writer.Write(bkey, bval); } } finally { writer.Close(context); } committer.CommitTask(context); committer.CommitJob(job); InputFormat <IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat <IntWritable , DoubleWritable>(); int count = 0; r.SetSeed(seed); SequenceFileInputFormat.SetInputPaths(job, outdir); Log.Info("Reading data by SequenceFileInputFormat"); foreach (InputSplit split in iformat.GetSplits(job)) { RecordReader <IntWritable, DoubleWritable> reader = iformat.CreateRecordReader(split , context); MapContext <IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext = new MapContextImpl <IntWritable, DoubleWritable, BytesWritable, BytesWritable>(job .GetConfiguration(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil .CreateDummyReporter(), split); reader.Initialize(split, mcontext); try { int sourceInt; double sourceDouble; while (reader.NextKeyValue()) { sourceInt = r.Next(); sourceDouble = r.NextDouble(); iwritable = reader.GetCurrentKey(); dwritable = reader.GetCurrentValue(); NUnit.Framework.Assert.AreEqual("Keys don't match: " + "*" + iwritable.Get() + ":" + sourceInt + "*", sourceInt, iwritable.Get()); NUnit.Framework.Assert.IsTrue("Vals don't match: " + "*" + dwritable.Get() + ":" + sourceDouble + "*", double.Compare(dwritable.Get(), sourceDouble) == 0); ++count; } } finally { reader.Close(); } } NUnit.Framework.Assert.AreEqual("Some records not found", Records, count); }
public CommitterEventHandler(AppContext context, OutputCommitter committer, RMHeartbeatHandler rmHeartbeatHandler) : this(context, committer, rmHeartbeatHandler, null) { }
/// <exception cref="System.Exception"/> /// <exception cref="System.IO.IOException"/> public ReduceContextImpl(Configuration conf, TaskAttemptID taskid, RawKeyValueIterator input, Counter inputKeyCounter, Counter inputValueCounter, RecordWriter <KEYOUT, VALUEOUT> output, OutputCommitter committer, StatusReporter reporter, RawComparator <KEYIN> comparator, Type keyClass, Type valueClass) : base(conf, taskid, output, committer, reporter) { iterable = new ReduceContextImpl.ValueIterable(this); // current key // current value // first value in key // more w/ this key // more in file this.input = input; this.inputKeyCounter = inputKeyCounter; this.inputValueCounter = inputValueCounter; this.comparator = comparator; this.serializationFactory = new SerializationFactory(conf); this.keyDeserializer = serializationFactory.GetDeserializer(keyClass); this.keyDeserializer.Open(buffer); this.valueDeserializer = serializationFactory.GetDeserializer(valueClass); this.valueDeserializer.Open(buffer); hasMore = input.Next(); this.keyClass = keyClass; this.valueClass = valueClass; this.conf = conf; this.taskid = taskid; }
public MapContextImpl(Configuration conf, TaskAttemptID taskid, RecordReader <KEYIN , VALUEIN> reader, RecordWriter <KEYOUT, VALUEOUT> writer, OutputCommitter committer , StatusReporter reporter, InputSplit split) : base(conf, taskid, writer, committer, reporter) { this.reader = reader; this.split = split; }
public virtual void TestCommitWindow() { Configuration conf = new Configuration(); conf.Set(MRJobConfig.MrAmStagingDir, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.Init(conf); dispatcher.Start(); TestCommitterEventHandler.TestingJobEventHandler jeh = new TestCommitterEventHandler.TestingJobEventHandler (); dispatcher.Register(typeof(JobEventType), jeh); SystemClock clock = new SystemClock(); AppContext appContext = Org.Mockito.Mockito.Mock <AppContext>(); ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0" ); Org.Mockito.Mockito.When(appContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId ()); Org.Mockito.Mockito.When(appContext.GetApplicationAttemptId()).ThenReturn(attemptid ); Org.Mockito.Mockito.When(appContext.GetEventHandler()).ThenReturn(dispatcher.GetEventHandler ()); Org.Mockito.Mockito.When(appContext.GetClock()).ThenReturn(clock); OutputCommitter committer = Org.Mockito.Mockito.Mock <OutputCommitter>(); TestCommitterEventHandler.TestingRMHeartbeatHandler rmhh = new TestCommitterEventHandler.TestingRMHeartbeatHandler (); CommitterEventHandler ceh = new CommitterEventHandler(appContext, committer, rmhh ); ceh.Init(conf); ceh.Start(); // verify trying to commit when RM heartbeats are stale does not commit ceh.Handle(new CommitterJobCommitEvent(null, null)); long timeToWaitMs = 5000; while (rmhh.GetNumCallbacks() != 1 && timeToWaitMs > 0) { Sharpen.Thread.Sleep(10); timeToWaitMs -= 10; } NUnit.Framework.Assert.AreEqual("committer did not register a heartbeat callback" , 1, rmhh.GetNumCallbacks()); Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Never()).CommitJob(Matchers.Any <JobContext>()); NUnit.Framework.Assert.AreEqual("committer should not have committed", 0, jeh.numCommitCompletedEvents ); // set a fresh heartbeat and verify commit completes rmhh.SetLastHeartbeatTime(clock.GetTime()); timeToWaitMs = 5000; while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0) { Sharpen.Thread.Sleep(10); timeToWaitMs -= 10; } NUnit.Framework.Assert.AreEqual("committer did not complete commit after RM hearbeat" , 1, jeh.numCommitCompletedEvents); Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(1)).CommitJob(Matchers.Any <JobContext>()); //Clean up so we can try to commit again (Don't do this at home) Cleanup(); // try to commit again and verify it goes through since the heartbeat // is still fresh ceh.Handle(new CommitterJobCommitEvent(null, null)); timeToWaitMs = 5000; while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0) { Sharpen.Thread.Sleep(10); timeToWaitMs -= 10; } NUnit.Framework.Assert.AreEqual("committer did not commit", 2, jeh.numCommitCompletedEvents ); Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(2)).CommitJob(Matchers.Any <JobContext>()); ceh.Stop(); dispatcher.Stop(); }
public _OutputCommitter_613(OutputCommitter committer) { this.committer = committer; }