public virtual object GetDatum()
{
if (datum == null)
{
datum = new TaskAttemptUnsuccessfulCompletion();
datum.taskid = new Utf8(attemptId.GetTaskID().ToString());
datum.taskType = new Utf8(taskType.ToString());
datum.attemptId = new Utf8(attemptId.ToString());
datum.finishTime = finishTime;
datum.hostname = new Utf8(hostname);
if (rackName != null)
{
datum.rackname = new Utf8(rackName);
}
datum.port = port;
datum.error = new Utf8(error);
datum.status = new Utf8(status);
datum.counters = EventWriter.ToAvro(counters);
datum.clockSplits = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetWallclockTime
(allSplits));
datum.cpuUsages = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetCPUTime(allSplits
));
datum.vMemKbytes = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetVMemKbytes(allSplits
));
datum.physMemKbytes = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetPhysMemKbytes
(allSplits));
}
return(datum);
}
/// <summary>
/// Tests the class loader set by
/// <see cref="Org.Apache.Hadoop.Conf.Configuration.SetClassLoader(Sharpen.ClassLoader)
/// "/>
/// is inherited by any
/// <see cref="WrappedRecordReader{K, U}"/>
/// s created by
/// <see cref="CompositeRecordReader{K, V, X}"/>
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestClassLoader()
{
Configuration conf = new Configuration();
TestWrappedRRClassloader.Fake_ClassLoader classLoader = new TestWrappedRRClassloader.Fake_ClassLoader
();
conf.SetClassLoader(classLoader);
NUnit.Framework.Assert.IsTrue(conf.GetClassLoader() is TestWrappedRRClassloader.Fake_ClassLoader
);
FileSystem fs = FileSystem.Get(conf);
Path testdir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
(fs);
Path @base = new Path(testdir, "/empty");
Path[] src = new Path[] { new Path(@base, "i0"), new Path("i1"), new Path("i2") };
conf.Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose("outer", typeof(
TestWrappedRRClassloader.IF_ClassLoaderChecker), src));
CompositeInputFormat <NullWritable> inputFormat = new CompositeInputFormat <NullWritable
>();
// create dummy TaskAttemptID
TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.Map, 0, 0);
conf.Set(MRJobConfig.TaskAttemptId, tid.ToString());
inputFormat.CreateRecordReader(inputFormat.GetSplits(Job.GetInstance(conf))[0], new
TaskAttemptContextImpl(conf, tid));
}
/// <exception cref="System.IO.IOException"/>
public virtual void CleanUpPartialOutputForTask(TaskAttemptContext context)
{
// we double check this is never invoked from a non-preemptable subclass.
// This should never happen, since the invoking codes is checking it too,
// but it is safer to double check. Errors handling this would produce
// inconsistent output.
if (!this.GetType().IsAnnotationPresent(typeof(Checkpointable)))
{
throw new InvalidOperationException("Invoking cleanUpPartialOutputForTask() " + "from non @Preemptable class"
);
}
FileSystem fs = FsFor(GetTaskAttemptPath(context), context.GetConfiguration());
Log.Info("cleanUpPartialOutputForTask: removing everything belonging to " + context
.GetTaskAttemptID().GetTaskID() + " in: " + GetCommittedTaskPath(context).GetParent
());
TaskAttemptID taid = context.GetTaskAttemptID();
TaskID tid = taid.GetTaskID();
Path pCommit = GetCommittedTaskPath(context).GetParent();
// remove any committed output
for (int i = 0; i < taid.GetId(); ++i)
{
TaskAttemptID oldId = new TaskAttemptID(tid, i);
Path pTask = new Path(pCommit, oldId.ToString());
if (fs.Exists(pTask) && !fs.Delete(pTask, true))
{
throw new IOException("Failed to delete " + pTask);
}
}
}
internal ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf)
{
this.numCopiers = jobConf.GetInt(MRJobConfig.ShuffleParallelCopies, 5);
MetricsContext metricsContext = MetricsUtil.GetContext("mapred");
this.shuffleMetrics = MetricsUtil.CreateRecord(metricsContext, "shuffleInput");
this.shuffleMetrics.SetTag("user", jobConf.GetUser());
this.shuffleMetrics.SetTag("jobName", jobConf.GetJobName());
this.shuffleMetrics.SetTag("jobId", reduceId.GetJobID().ToString());
this.shuffleMetrics.SetTag("taskId", reduceId.ToString());
this.shuffleMetrics.SetTag("sessionId", jobConf.GetSessionId());
metricsContext.RegisterUpdater(this);
}
public virtual object GetDatum()
{
if (datum == null)
{
datum = new TaskFinished();
datum.taskid = new Utf8(taskid.ToString());
if (successfulAttemptId != null)
{
datum.successfulAttemptId = new Utf8(successfulAttemptId.ToString());
}
datum.finishTime = finishTime;
datum.counters = EventWriter.ToAvro(counters);
datum.taskType = new Utf8(taskType.ToString());
datum.status = new Utf8(status);
}
return(datum);
}
/// <summary>Print all the information about this attempt.</summary>
public virtual void PrintAll()
{
System.Console.Out.WriteLine("ATTEMPT_ID:" + attemptId.ToString());
System.Console.Out.WriteLine("START_TIME: " + startTime);
System.Console.Out.WriteLine("FINISH_TIME:" + finishTime);
System.Console.Out.WriteLine("ERROR:" + error);
System.Console.Out.WriteLine("TASK_STATUS:" + status);
System.Console.Out.WriteLine("STATE:" + state);
System.Console.Out.WriteLine("TASK_TYPE:" + taskType);
System.Console.Out.WriteLine("TRACKER_NAME:" + trackerName);
System.Console.Out.WriteLine("HTTP_PORT:" + httpPort);
System.Console.Out.WriteLine("SHUFFLE_PORT:" + shufflePort);
System.Console.Out.WriteLine("CONTIANER_ID:" + containerId);
if (counters != null)
{
System.Console.Out.WriteLine("COUNTERS:" + counters.ToString());
}
}
public virtual object GetDatum()
{
if (datum == null)
{
datum = new TaskAttemptFinished();
datum.taskid = new Utf8(attemptId.GetTaskID().ToString());
datum.attemptId = new Utf8(attemptId.ToString());
datum.taskType = new Utf8(taskType.ToString());
datum.taskStatus = new Utf8(taskStatus);
datum.finishTime = finishTime;
if (rackName != null)
{
datum.rackname = new Utf8(rackName);
}
datum.hostname = new Utf8(hostname);
datum.state = new Utf8(state);
datum.counters = EventWriter.ToAvro(counters);
}
return(datum);
}
/// <summary>test a getters of TaskAttemptFinishedEvent and TaskAttemptFinished</summary>
/// <exception cref="System.Exception"/>
public virtual void TestTaskAttemptFinishedEvent()
{
JobID jid = new JobID("001", 1);
TaskID tid = new TaskID(jid, TaskType.Reduce, 2);
TaskAttemptID taskAttemptId = new TaskAttemptID(tid, 3);
Counters counters = new Counters();
TaskAttemptFinishedEvent test = new TaskAttemptFinishedEvent(taskAttemptId, TaskType
.Reduce, "TEST", 123L, "RAKNAME", "HOSTNAME", "STATUS", counters);
NUnit.Framework.Assert.AreEqual(test.GetAttemptId().ToString(), taskAttemptId.ToString
());
NUnit.Framework.Assert.AreEqual(test.GetCounters(), counters);
NUnit.Framework.Assert.AreEqual(test.GetFinishTime(), 123L);
NUnit.Framework.Assert.AreEqual(test.GetHostname(), "HOSTNAME");
NUnit.Framework.Assert.AreEqual(test.GetRackName(), "RAKNAME");
NUnit.Framework.Assert.AreEqual(test.GetState(), "STATUS");
NUnit.Framework.Assert.AreEqual(test.GetTaskId(), tid);
NUnit.Framework.Assert.AreEqual(test.GetTaskStatus(), "TEST");
NUnit.Framework.Assert.AreEqual(test.GetTaskType(), TaskType.Reduce);
}
/// <summary>Create an event to record the start of an attempt</summary>
/// <param name="attemptId">Id of the attempt</param>
/// <param name="taskType">Type of task</param>
/// <param name="startTime">Start time of the attempt</param>
/// <param name="trackerName">Name of the Task Tracker where attempt is running</param>
/// <param name="httpPort">The port number of the tracker</param>
/// <param name="shufflePort">The shuffle port number of the container</param>
/// <param name="containerId">The containerId for the task attempt.</param>
/// <param name="locality">The locality of the task attempt</param>
/// <param name="avataar">The avataar of the task attempt</param>
public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType, long startTime
, string trackerName, int httpPort, int shufflePort, ContainerId containerId, string
locality, string avataar)
{
datum.attemptId = new Utf8(attemptId.ToString());
datum.taskid = new Utf8(attemptId.GetTaskID().ToString());
datum.startTime = startTime;
datum.taskType = new Utf8(taskType.ToString());
datum.trackerName = new Utf8(trackerName);
datum.httpPort = httpPort;
datum.shufflePort = shufflePort;
datum.containerId = new Utf8(containerId.ToString());
if (locality != null)
{
datum.locality = new Utf8(locality);
}
if (avataar != null)
{
datum.avataar = new Utf8(avataar);
}
}
public virtual object GetDatum()
{
if (datum == null)
{
datum = new TaskFailed();
datum.taskid = new Utf8(id.ToString());
datum.error = new Utf8(error);
datum.finishTime = finishTime;
datum.taskType = new Utf8(taskType.ToString());
datum.failedDueToAttempt = failedDueToAttempt == null ? null : new Utf8(failedDueToAttempt
.ToString());
datum.status = new Utf8(status);
datum.counters = EventWriter.ToAvro(counters);
}
return(datum);
}