////////////////////////////////////////////// // Writable ////////////////////////////////////////////// /// <exception cref="System.IO.IOException"/> public virtual void Write(DataOutput @out) { taskid.Write(@out); @out.WriteFloat(progress); Text.WriteString(@out, state); @out.WriteLong(startTime); @out.WriteLong(finishTime); WritableUtils.WriteStringArray(@out, diagnostics); counters.Write(@out); WritableUtils.WriteEnum(@out, currentStatus); if (currentStatus == TIPStatus.Running) { WritableUtils.WriteVInt(@out, runningAttempts.Count); TaskAttemptID[] t = new TaskAttemptID[0]; t = Sharpen.Collections.ToArray(runningAttempts, t); for (int i = 0; i < t.Length; i++) { t[i].Write(@out); } } else { if (currentStatus == TIPStatus.Complete) { successfulAttempt.Write(@out); } } }
/// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { this.taskid.ReadFields(@in); this.progress = @in.ReadFloat(); this.state = StringInterner.WeakIntern(Text.ReadString(@in)); this.startTime = @in.ReadLong(); this.finishTime = @in.ReadLong(); diagnostics = WritableUtils.ReadStringArray(@in); counters = new Counters(); counters.ReadFields(@in); currentStatus = WritableUtils.ReadEnum <TIPStatus>(@in); if (currentStatus == TIPStatus.Running) { int num = WritableUtils.ReadVInt(@in); for (int i = 0; i < num; i++) { TaskAttemptID t = new TaskAttemptID(); t.ReadFields(@in); runningAttempts.AddItem(t); } } else { if (currentStatus == TIPStatus.Complete) { successfulAttempt.ReadFields(@in); } } }
/// <summary>Reads tasklog and returns it as string after trimming it.</summary> /// <param name="filter">Task log filter; can be STDOUT, STDERR, SYSLOG, DEBUGOUT, PROFILE /// </param> /// <param name="taskId">The task id for which the log has to collected</param> /// <param name="isCleanup">whether the task is a cleanup attempt or not.</param> /// <returns>task log as string</returns> /// <exception cref="System.IO.IOException"/> public static string ReadTaskLog(TaskLog.LogName filter, TaskAttemptID taskId, bool isCleanup) { // string buffer to store task log StringBuilder result = new StringBuilder(); int res; // reads the whole tasklog into inputstream InputStream taskLogReader = new TaskLog.Reader(taskId, filter, 0, -1, isCleanup); // construct string log from inputstream. byte[] b = new byte[65536]; while (true) { res = taskLogReader.Read(b); if (res > 0) { result.Append(Sharpen.Runtime.GetStringForBytes(b)); } else { break; } } taskLogReader.Close(); // trim the string and return it string str = result.ToString(); str = str.Trim(); return(str); }
public static TaskAttemptId ToYarn(TaskAttemptID id) { TaskAttemptId taskAttemptId = recordFactory.NewRecordInstance <TaskAttemptId>(); taskAttemptId.SetTaskId(ToYarn(id.GetTaskID())); taskAttemptId.SetId(id.GetId()); return(taskAttemptId); }
public static TaskAttemptContext CreateDummyMapTaskAttemptContext(Configuration conf ) { TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.Map, 0, 0); conf.Set(MRJobConfig.TaskAttemptId, tid.ToString()); return(new TaskAttemptContextImpl(conf, tid)); }
/// <exception cref="Org.Apache.Hadoop.FS.FileAlreadyExistsException"/> /// <exception cref="System.IO.IOException"/> public override void CheckOutputSpecs(JobContext job) { base.CheckOutputSpecs(job); // creating dummy TaskAttemptID TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.JobSetup, 0, 0); GetOutputCommitter(new TaskAttemptContextImpl(job.GetConfiguration(), tid)).SetupJob (job); }
/// <summary>Constructor.</summary> /// <remarks> /// Constructor. eventId should be created externally and incremented /// per event for each job. /// </remarks> /// <param name="eventId"> /// event id, event id should be unique and assigned in /// incrementally, starting from 0. /// </param> /// <param name="taskId">task id</param> /// <param name="status">task's status</param> /// <param name="taskTrackerHttp">task tracker's host:port for http.</param> public TaskCompletionEvent(int eventId, TaskAttemptID taskId, int idWithinJob, bool isMap, TaskCompletionEvent.Status status, string taskTrackerHttp) { this.taskId = taskId; this.idWithinJob = idWithinJob; this.isMap = isMap; this.eventId = eventId; this.status = status; this.taskTrackerHttp = taskTrackerHttp; }
public virtual void TestConsumerApi() { JobConf jobConf = new JobConf(); ShuffleConsumerPlugin <K, V> shuffleConsumerPlugin = new TestShufflePlugin.TestShuffleConsumerPlugin <K, V>(); //mock creation ReduceTask mockReduceTask = Org.Mockito.Mockito.Mock <ReduceTask>(); TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol >(); Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>(); FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>(); Type combinerClass = jobConf.GetCombinerClass(); Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector <K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>(); // needed for mock with generic TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>(); LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator >(); CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec >(); Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>(); TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>(); Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>(); MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>(); Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task >(); try { string[] dirs = jobConf.GetLocalDirs(); // verify that these APIs are available through super class handler ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K , V>(mockTaskAttemptID, jobConf, mockFileSystem, mockUmbilical, mockLocalDirAllocator , mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus , mockProgress, mockProgress, mockTask, mockMapOutputFile, null); shuffleConsumerPlugin.Init(context); shuffleConsumerPlugin.Run(); shuffleConsumerPlugin.Close(); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("Threw exception:" + e, false); } // verify that these APIs are available for 3rd party plugins mockReduceTask.GetTaskID(); mockReduceTask.GetJobID(); mockReduceTask.GetNumMaps(); mockReduceTask.GetPartition(); mockReporter.Progress(); }
public virtual void TestCloneMapContext() { TaskID taskId = new TaskID(jobId, TaskType.Map, 0); TaskAttemptID taskAttemptid = new TaskAttemptID(taskId, 0); MapContext <IntWritable, IntWritable, IntWritable, IntWritable> mapContext = new MapContextImpl <IntWritable, IntWritable, IntWritable, IntWritable>(conf, taskAttemptid, null, null, null, null, null); Mapper.Context mapperContext = new WrappedMapper <IntWritable, IntWritable, IntWritable , IntWritable>().GetMapContext(mapContext); ContextFactory.CloneMapContext(mapperContext, conf, null, null); }
/// <summary>Downgrade a new TaskAttemptID to an old one</summary> /// <param name="old">the new id</param> /// <returns>either old or a new TaskAttemptID constructed to match old</returns> public static Org.Apache.Hadoop.Mapred.TaskAttemptID Downgrade(Org.Apache.Hadoop.Mapreduce.TaskAttemptID old) { if (old is Org.Apache.Hadoop.Mapred.TaskAttemptID) { return((Org.Apache.Hadoop.Mapred.TaskAttemptID)old); } else { return(new Org.Apache.Hadoop.Mapred.TaskAttemptID(TaskID.Downgrade(old.GetTaskID( )), old.GetId())); } }
/// <summary>test a kill task</summary> /// <exception cref="System.Exception"/> private void TestKillTask(Configuration conf) { Job job = RunJobInBackGround(conf); CLI jc = CreateJobClient(); TaskID tid = new TaskID(job.GetJobID(), TaskType.Map, 0); TaskAttemptID taid = new TaskAttemptID(tid, 1); ByteArrayOutputStream @out = new ByteArrayOutputStream(); // bad parameters int exitCode = RunTool(conf, jc, new string[] { "-kill-task" }, @out); NUnit.Framework.Assert.AreEqual("Exit code", -1, exitCode); RunTool(conf, jc, new string[] { "-kill-task", taid.ToString() }, @out); string answer = Sharpen.Runtime.GetStringForBytes(@out.ToByteArray(), "UTF-8"); NUnit.Framework.Assert.IsTrue(answer.Contains("Killed task " + taid)); }
/// <summary>set successful attempt ID of the task.</summary> protected internal virtual void SetSuccessfulAttemptId(TaskAttemptID t) { successfulAttempt = t; }
/// <summary>Get log parameters for the specified jobID or taskAttemptID</summary> /// <param name="jobID">the job id.</param> /// <param name="taskAttemptID">the task attempt id. Optional.</param> /// <returns>the LogParams</returns> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual LogParams GetLogParams(JobID jobID, TaskAttemptID taskAttemptID) { return(client.GetLogFileParams(jobID, taskAttemptID)); }
/// <summary>Default constructor for Writable.</summary> public TaskCompletionEvent() { // using int since runtime is the time difference taskId = new TaskAttemptID(); }
/// <summary>Sets task id.</summary> /// <param name="taskId"/> protected internal virtual void SetTaskAttemptId(TaskAttemptID taskId) { this.taskId = taskId; }