//////////////////////////////////////////////
// Writable
//////////////////////////////////////////////
/// <exception cref="System.IO.IOException"/>
public virtual void Write(DataOutput @out)
{
taskid.Write(@out);
@out.WriteFloat(progress);
Text.WriteString(@out, state);
@out.WriteLong(startTime);
@out.WriteLong(finishTime);
WritableUtils.WriteStringArray(@out, diagnostics);
counters.Write(@out);
WritableUtils.WriteEnum(@out, currentStatus);
if (currentStatus == TIPStatus.Running)
{
WritableUtils.WriteVInt(@out, runningAttempts.Count);
TaskAttemptID[] t = new TaskAttemptID[0];
t = Sharpen.Collections.ToArray(runningAttempts, t);
for (int i = 0; i < t.Length; i++)
{
t[i].Write(@out);
}
}
else
{
if (currentStatus == TIPStatus.Complete)
{
successfulAttempt.Write(@out);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReadFields(DataInput @in)
{
this.taskid.ReadFields(@in);
this.progress = @in.ReadFloat();
this.state = StringInterner.WeakIntern(Text.ReadString(@in));
this.startTime = @in.ReadLong();
this.finishTime = @in.ReadLong();
diagnostics = WritableUtils.ReadStringArray(@in);
counters = new Counters();
counters.ReadFields(@in);
currentStatus = WritableUtils.ReadEnum <TIPStatus>(@in);
if (currentStatus == TIPStatus.Running)
{
int num = WritableUtils.ReadVInt(@in);
for (int i = 0; i < num; i++)
{
TaskAttemptID t = new TaskAttemptID();
t.ReadFields(@in);
runningAttempts.AddItem(t);
}
}
else
{
if (currentStatus == TIPStatus.Complete)
{
successfulAttempt.ReadFields(@in);
}
}
}
/// <summary>Reads tasklog and returns it as string after trimming it.</summary>
/// <param name="filter">Task log filter; can be STDOUT, STDERR, SYSLOG, DEBUGOUT, PROFILE
/// </param>
/// <param name="taskId">The task id for which the log has to collected</param>
/// <param name="isCleanup">whether the task is a cleanup attempt or not.</param>
/// <returns>task log as string</returns>
/// <exception cref="System.IO.IOException"/>
public static string ReadTaskLog(TaskLog.LogName filter, TaskAttemptID taskId, bool
isCleanup)
{
// string buffer to store task log
StringBuilder result = new StringBuilder();
int res;
// reads the whole tasklog into inputstream
InputStream taskLogReader = new TaskLog.Reader(taskId, filter, 0, -1, isCleanup);
// construct string log from inputstream.
byte[] b = new byte[65536];
while (true)
{
res = taskLogReader.Read(b);
if (res > 0)
{
result.Append(Sharpen.Runtime.GetStringForBytes(b));
}
else
{
break;
}
}
taskLogReader.Close();
// trim the string and return it
string str = result.ToString();
str = str.Trim();
return(str);
}
public static TaskAttemptId ToYarn(TaskAttemptID id)
{
TaskAttemptId taskAttemptId = recordFactory.NewRecordInstance <TaskAttemptId>();
taskAttemptId.SetTaskId(ToYarn(id.GetTaskID()));
taskAttemptId.SetId(id.GetId());
return(taskAttemptId);
}
public static TaskAttemptContext CreateDummyMapTaskAttemptContext(Configuration conf
)
{
TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.Map, 0, 0);
conf.Set(MRJobConfig.TaskAttemptId, tid.ToString());
return(new TaskAttemptContextImpl(conf, tid));
}
/// <exception cref="Org.Apache.Hadoop.FS.FileAlreadyExistsException"/>
/// <exception cref="System.IO.IOException"/>
public override void CheckOutputSpecs(JobContext job)
{
base.CheckOutputSpecs(job);
// creating dummy TaskAttemptID
TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.JobSetup, 0, 0);
GetOutputCommitter(new TaskAttemptContextImpl(job.GetConfiguration(), tid)).SetupJob
(job);
}
/// <summary>Constructor.</summary>
/// <remarks>
/// Constructor. eventId should be created externally and incremented
/// per event for each job.
/// </remarks>
/// <param name="eventId">
/// event id, event id should be unique and assigned in
/// incrementally, starting from 0.
/// </param>
/// <param name="taskId">task id</param>
/// <param name="status">task's status</param>
/// <param name="taskTrackerHttp">task tracker's host:port for http.</param>
public TaskCompletionEvent(int eventId, TaskAttemptID taskId, int idWithinJob, bool
isMap, TaskCompletionEvent.Status status, string taskTrackerHttp)
{
this.taskId = taskId;
this.idWithinJob = idWithinJob;
this.isMap = isMap;
this.eventId = eventId;
this.status = status;
this.taskTrackerHttp = taskTrackerHttp;
}
public virtual void TestConsumerApi()
{
JobConf jobConf = new JobConf();
ShuffleConsumerPlugin <K, V> shuffleConsumerPlugin = new TestShufflePlugin.TestShuffleConsumerPlugin
<K, V>();
//mock creation
ReduceTask mockReduceTask = Org.Mockito.Mockito.Mock <ReduceTask>();
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = jobConf.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
try
{
string[] dirs = jobConf.GetLocalDirs();
// verify that these APIs are available through super class handler
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, jobConf, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
shuffleConsumerPlugin.Init(context);
shuffleConsumerPlugin.Run();
shuffleConsumerPlugin.Close();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue("Threw exception:" + e, false);
}
// verify that these APIs are available for 3rd party plugins
mockReduceTask.GetTaskID();
mockReduceTask.GetJobID();
mockReduceTask.GetNumMaps();
mockReduceTask.GetPartition();
mockReporter.Progress();
}
public virtual void TestCloneMapContext()
{
TaskID taskId = new TaskID(jobId, TaskType.Map, 0);
TaskAttemptID taskAttemptid = new TaskAttemptID(taskId, 0);
MapContext <IntWritable, IntWritable, IntWritable, IntWritable> mapContext = new MapContextImpl
<IntWritable, IntWritable, IntWritable, IntWritable>(conf, taskAttemptid, null,
null, null, null, null);
Mapper.Context mapperContext = new WrappedMapper <IntWritable, IntWritable, IntWritable
, IntWritable>().GetMapContext(mapContext);
ContextFactory.CloneMapContext(mapperContext, conf, null, null);
}
/// <summary>Downgrade a new TaskAttemptID to an old one</summary>
/// <param name="old">the new id</param>
/// <returns>either old or a new TaskAttemptID constructed to match old</returns>
public static Org.Apache.Hadoop.Mapred.TaskAttemptID Downgrade(Org.Apache.Hadoop.Mapreduce.TaskAttemptID
old)
{
if (old is Org.Apache.Hadoop.Mapred.TaskAttemptID)
{
return((Org.Apache.Hadoop.Mapred.TaskAttemptID)old);
}
else
{
return(new Org.Apache.Hadoop.Mapred.TaskAttemptID(TaskID.Downgrade(old.GetTaskID(
)), old.GetId()));
}
}
/// <summary>test a kill task</summary>
/// <exception cref="System.Exception"/>
private void TestKillTask(Configuration conf)
{
Job job = RunJobInBackGround(conf);
CLI jc = CreateJobClient();
TaskID tid = new TaskID(job.GetJobID(), TaskType.Map, 0);
TaskAttemptID taid = new TaskAttemptID(tid, 1);
ByteArrayOutputStream @out = new ByteArrayOutputStream();
// bad parameters
int exitCode = RunTool(conf, jc, new string[] { "-kill-task" }, @out);
NUnit.Framework.Assert.AreEqual("Exit code", -1, exitCode);
RunTool(conf, jc, new string[] { "-kill-task", taid.ToString() }, @out);
string answer = Sharpen.Runtime.GetStringForBytes(@out.ToByteArray(), "UTF-8");
NUnit.Framework.Assert.IsTrue(answer.Contains("Killed task " + taid));
}
/// <summary>set successful attempt ID of the task.</summary>
protected internal virtual void SetSuccessfulAttemptId(TaskAttemptID t)
{
successfulAttempt = t;
}
/// <summary>Get log parameters for the specified jobID or taskAttemptID</summary>
/// <param name="jobID">the job id.</param>
/// <param name="taskAttemptID">the task attempt id. Optional.</param>
/// <returns>the LogParams</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual LogParams GetLogParams(JobID jobID, TaskAttemptID taskAttemptID)
{
return(client.GetLogFileParams(jobID, taskAttemptID));
}
/// <summary>Default constructor for Writable.</summary>
public TaskCompletionEvent()
{
// using int since runtime is the time difference
taskId = new TaskAttemptID();
}
/// <summary>Sets task id.</summary>
/// <param name="taskId"/>
protected internal virtual void SetTaskAttemptId(TaskAttemptID taskId)
{
this.taskId = taskId;
}
