public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, RecordWriter
<KEYOUT, VALUEOUT> output, OutputCommitter committer, StatusReporter reporter)
: base(conf, taskid, reporter)
{
this.output = output;
this.committer = committer;
}
public TaskRecoverEvent(TaskId taskID, JobHistoryParser.TaskInfo taskInfo, OutputCommitter
committer, bool recoverTaskOutput)
: base(taskID, TaskEventType.TRecover)
{
this.taskInfo = taskInfo;
this.committer = committer;
this.recoverTaskOutput = recoverTaskOutput;
}
public TaskAttemptRecoverEvent(TaskAttemptId id, JobHistoryParser.TaskAttemptInfo
taInfo, OutputCommitter committer, bool recoverOutput)
: base(id, TaskAttemptEventType.TaRecover)
{
this.taInfo = taInfo;
this.committer = committer;
this.recoverAttemptOutput = recoverOutput;
}
/// <exception cref="System.IO.IOException"/>
public override OutputCommitter GetOutputCommitter(TaskAttemptContext context)
{
if (committer == null)
{
Path output = GetOutputPath(context);
committer = new FileOutputCommitter(output, context);
}
return(committer);
}
protected internal override EventHandler <CommitterEvent> CreateCommitterEventHandler
(AppContext context, OutputCommitter committer)
{
// create an output committer with the task methods stubbed out
OutputCommitter stubbedCommitter = new _OutputCommitter_613(committer);
return(new CommitterEventHandler(context, stubbedCommitter, GetRMHeartbeatHandler
()));
}
public CommitterEventHandler(AppContext context, OutputCommitter committer, RMHeartbeatHandler
rmHeartbeatHandler, ClassLoader jobClassLoader)
: base("CommitterEventHandler")
{
this.context = context;
this.committer = committer;
this.rmHeartbeatHandler = rmHeartbeatHandler;
this.stopped = new AtomicBoolean(false);
this.jobClassLoader = jobClassLoader;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public Void Call()
{
OutputCommitter outputCommitter = tof[taskIdx].GetOutputCommitter(taCtx[taskIdx]);
outputCommitter.SetupTask(taCtx[taskIdx]);
RecordWriter rw = tof[taskIdx].GetRecordWriter(taCtx[taskIdx]);
this._enclosing.WriteOutput(rw, taCtx[taskIdx]);
outputCommitter.CommitTask(taCtx[taskIdx]);
return(null);
}
/// <exception cref="System.IO.IOException"/>
public override OutputCommitter GetOutputCommitter(TaskAttemptContext context)
{
lock (this)
{
if (committer == null)
{
Path output = GetOutputPath(context);
committer = new TestJobOutputCommitter.CommitterWithCustomAbort(output, context);
}
return(committer);
}
}
public virtual void TestBasic()
{
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
OutputCommitter mockCommitter = Org.Mockito.Mockito.Mock <OutputCommitter>();
Clock mockClock = Org.Mockito.Mockito.Mock <Clock>();
CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter
, new TestCommitterEventHandler.TestingRMHeartbeatHandler());
YarnConfiguration conf = new YarnConfiguration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
JobContext mockJobContext = Org.Mockito.Mockito.Mock <JobContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
JobId jobId = TypeConverter.ToYarn(TypeConverter.FromYarn(attemptid.GetApplicationId
()));
TestCommitterEventHandler.WaitForItHandler waitForItHandler = new TestCommitterEventHandler.WaitForItHandler
();
Org.Mockito.Mockito.When(mockContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(mockContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(waitForItHandler
);
Org.Mockito.Mockito.When(mockContext.GetClock()).ThenReturn(mockClock);
handler.Init(conf);
handler.Start();
try
{
handler.Handle(new CommitterJobCommitEvent(jobId, mockJobContext));
string user = UserGroupInformation.GetCurrentUser().GetShortUserName();
Path startCommitFile = MRApps.GetStartJobCommitFile(conf, user, jobId);
Path endCommitSuccessFile = MRApps.GetEndJobCommitSuccessFile(conf, user, jobId);
Path endCommitFailureFile = MRApps.GetEndJobCommitFailureFile(conf, user, jobId);
Org.Apache.Hadoop.Yarn.Event.Event e = waitForItHandler.GetAndClearEvent();
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsTrue(e is JobCommitCompletedEvent);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue(startCommitFile.ToString(), fs.Exists(startCommitFile
));
NUnit.Framework.Assert.IsTrue(endCommitSuccessFile.ToString(), fs.Exists(endCommitSuccessFile
));
NUnit.Framework.Assert.IsFalse(endCommitFailureFile.ToString(), fs.Exists(endCommitFailureFile
));
Org.Mockito.Mockito.Verify(mockCommitter).CommitJob(Matchers.Any <JobContext>());
}
finally
{
handler.Stop();
}
}
public TestJob(MRApp _enclosing, JobId jobId, ApplicationAttemptId applicationAttemptId
, Configuration conf, EventHandler eventHandler, TaskAttemptListener taskAttemptListener
, Clock clock, OutputCommitter committer, bool newApiCommitter, string user, AppContext
appContext, JobStateInternal forcedState, string diagnostic)
: base(jobId, MRApp.GetApplicationAttemptId(MRApp.applicationId, this._enclosing.
GetStartCount()), conf, eventHandler, taskAttemptListener, new JobTokenSecretManager
(), new Credentials(), clock, this._enclosing.GetCompletedTaskFromPreviousRun(),
this._enclosing.metrics, committer, newApiCommitter, user, Runtime.CurrentTimeMillis
(), this._enclosing.GetAllAMInfos(), appContext, forcedState, diagnostic)
{
this._enclosing = _enclosing;
// This "this leak" is okay because the retained pointer is in an
// instance variable.
this.localStateMachine = this.localFactory.Make(this);
}
protected internal override EventHandler<CommitterEvent> CreateCommitterEventHandler
(AppContext context, OutputCommitter committer)
{
return mockCommitterEventHandler;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestBinary()
{
Configuration conf = new Configuration();
Job job = Job.GetInstance(conf);
Path outdir = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "outseq");
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
FileOutputFormat.SetOutputPath(job, outdir);
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputKeyClass(job, typeof(IntWritable
));
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputValueClass(job, typeof(DoubleWritable
));
SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true);
SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
.Block);
BytesWritable bkey = new BytesWritable();
BytesWritable bval = new BytesWritable();
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
.GetConfiguration());
OutputFormat <BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat
();
OutputCommitter committer = outputFormat.GetOutputCommitter(context);
committer.SetupJob(job);
RecordWriter <BytesWritable, BytesWritable> writer = outputFormat.GetRecordWriter(
context);
IntWritable iwritable = new IntWritable();
DoubleWritable dwritable = new DoubleWritable();
DataOutputBuffer outbuf = new DataOutputBuffer();
Log.Info("Creating data by SequenceFileAsBinaryOutputFormat");
try
{
for (int i = 0; i < Records; ++i)
{
iwritable = new IntWritable(r.Next());
iwritable.Write(outbuf);
bkey.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
dwritable = new DoubleWritable(r.NextDouble());
dwritable.Write(outbuf);
bval.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
writer.Write(bkey, bval);
}
}
finally
{
writer.Close(context);
}
committer.CommitTask(context);
committer.CommitJob(job);
InputFormat <IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat <IntWritable
, DoubleWritable>();
int count = 0;
r.SetSeed(seed);
SequenceFileInputFormat.SetInputPaths(job, outdir);
Log.Info("Reading data by SequenceFileInputFormat");
foreach (InputSplit split in iformat.GetSplits(job))
{
RecordReader <IntWritable, DoubleWritable> reader = iformat.CreateRecordReader(split
, context);
MapContext <IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext =
new MapContextImpl <IntWritable, DoubleWritable, BytesWritable, BytesWritable>(job
.GetConfiguration(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil
.CreateDummyReporter(), split);
reader.Initialize(split, mcontext);
try
{
int sourceInt;
double sourceDouble;
while (reader.NextKeyValue())
{
sourceInt = r.Next();
sourceDouble = r.NextDouble();
iwritable = reader.GetCurrentKey();
dwritable = reader.GetCurrentValue();
NUnit.Framework.Assert.AreEqual("Keys don't match: " + "*" + iwritable.Get() + ":"
+ sourceInt + "*", sourceInt, iwritable.Get());
NUnit.Framework.Assert.IsTrue("Vals don't match: " + "*" + dwritable.Get() + ":"
+ sourceDouble + "*", double.Compare(dwritable.Get(), sourceDouble) == 0);
++count;
}
}
finally
{
reader.Close();
}
}
NUnit.Framework.Assert.AreEqual("Some records not found", Records, count);
}
public CommitterEventHandler(AppContext context, OutputCommitter committer, RMHeartbeatHandler
rmHeartbeatHandler)
: this(context, committer, rmHeartbeatHandler, null)
{
}
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public ReduceContextImpl(Configuration conf, TaskAttemptID taskid, RawKeyValueIterator
input, Counter inputKeyCounter, Counter inputValueCounter, RecordWriter <KEYOUT,
VALUEOUT> output, OutputCommitter committer, StatusReporter reporter, RawComparator
<KEYIN> comparator, Type keyClass, Type valueClass)
: base(conf, taskid, output, committer, reporter)
{
iterable = new ReduceContextImpl.ValueIterable(this);
// current key
// current value
// first value in key
// more w/ this key
// more in file
this.input = input;
this.inputKeyCounter = inputKeyCounter;
this.inputValueCounter = inputValueCounter;
this.comparator = comparator;
this.serializationFactory = new SerializationFactory(conf);
this.keyDeserializer = serializationFactory.GetDeserializer(keyClass);
this.keyDeserializer.Open(buffer);
this.valueDeserializer = serializationFactory.GetDeserializer(valueClass);
this.valueDeserializer.Open(buffer);
hasMore = input.Next();
this.keyClass = keyClass;
this.valueClass = valueClass;
this.conf = conf;
this.taskid = taskid;
}
public MapContextImpl(Configuration conf, TaskAttemptID taskid, RecordReader <KEYIN
, VALUEIN> reader, RecordWriter <KEYOUT, VALUEOUT> writer, OutputCommitter committer
, StatusReporter reporter, InputSplit split)
: base(conf, taskid, writer, committer, reporter)
{
this.reader = reader;
this.split = split;
}
public virtual void TestCommitWindow()
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.MrAmStagingDir, stagingDir);
AsyncDispatcher dispatcher = new AsyncDispatcher();
dispatcher.Init(conf);
dispatcher.Start();
TestCommitterEventHandler.TestingJobEventHandler jeh = new TestCommitterEventHandler.TestingJobEventHandler
();
dispatcher.Register(typeof(JobEventType), jeh);
SystemClock clock = new SystemClock();
AppContext appContext = Org.Mockito.Mockito.Mock <AppContext>();
ApplicationAttemptId attemptid = ConverterUtils.ToApplicationAttemptId("appattempt_1234567890000_0001_0"
);
Org.Mockito.Mockito.When(appContext.GetApplicationID()).ThenReturn(attemptid.GetApplicationId
());
Org.Mockito.Mockito.When(appContext.GetApplicationAttemptId()).ThenReturn(attemptid
);
Org.Mockito.Mockito.When(appContext.GetEventHandler()).ThenReturn(dispatcher.GetEventHandler
());
Org.Mockito.Mockito.When(appContext.GetClock()).ThenReturn(clock);
OutputCommitter committer = Org.Mockito.Mockito.Mock <OutputCommitter>();
TestCommitterEventHandler.TestingRMHeartbeatHandler rmhh = new TestCommitterEventHandler.TestingRMHeartbeatHandler
();
CommitterEventHandler ceh = new CommitterEventHandler(appContext, committer, rmhh
);
ceh.Init(conf);
ceh.Start();
// verify trying to commit when RM heartbeats are stale does not commit
ceh.Handle(new CommitterJobCommitEvent(null, null));
long timeToWaitMs = 5000;
while (rmhh.GetNumCallbacks() != 1 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not register a heartbeat callback"
, 1, rmhh.GetNumCallbacks());
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Never()).CommitJob(Matchers.Any
<JobContext>());
NUnit.Framework.Assert.AreEqual("committer should not have committed", 0, jeh.numCommitCompletedEvents
);
// set a fresh heartbeat and verify commit completes
rmhh.SetLastHeartbeatTime(clock.GetTime());
timeToWaitMs = 5000;
while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not complete commit after RM hearbeat"
, 1, jeh.numCommitCompletedEvents);
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(1)).CommitJob(Matchers.Any
<JobContext>());
//Clean up so we can try to commit again (Don't do this at home)
Cleanup();
// try to commit again and verify it goes through since the heartbeat
// is still fresh
ceh.Handle(new CommitterJobCommitEvent(null, null));
timeToWaitMs = 5000;
while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0)
{
Sharpen.Thread.Sleep(10);
timeToWaitMs -= 10;
}
NUnit.Framework.Assert.AreEqual("committer did not commit", 2, jeh.numCommitCompletedEvents
);
Org.Mockito.Mockito.Verify(committer, Org.Mockito.Mockito.Times(2)).CommitJob(Matchers.Any
<JobContext>());
ceh.Stop();
dispatcher.Stop();
}
public _OutputCommitter_613(OutputCommitter committer)
{
this.committer = committer;
}
