Esempio n. 1
0
 public virtual object GetDatum()
 {
     if (datum == null)
     {
         datum            = new TaskAttemptUnsuccessfulCompletion();
         datum.taskid     = new Utf8(attemptId.GetTaskID().ToString());
         datum.taskType   = new Utf8(taskType.ToString());
         datum.attemptId  = new Utf8(attemptId.ToString());
         datum.finishTime = finishTime;
         datum.hostname   = new Utf8(hostname);
         if (rackName != null)
         {
             datum.rackname = new Utf8(rackName);
         }
         datum.port        = port;
         datum.error       = new Utf8(error);
         datum.status      = new Utf8(status);
         datum.counters    = EventWriter.ToAvro(counters);
         datum.clockSplits = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetWallclockTime
                                                       (allSplits));
         datum.cpuUsages = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetCPUTime(allSplits
                                                                                     ));
         datum.vMemKbytes = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetVMemKbytes(allSplits
                                                                                         ));
         datum.physMemKbytes = AvroArrayUtils.ToAvro(ProgressSplitsBlock.ArrayGetPhysMemKbytes
                                                         (allSplits));
     }
     return(datum);
 }
        /// <summary>
        /// Tests the class loader set by
        /// <see cref="Org.Apache.Hadoop.Conf.Configuration.SetClassLoader(Sharpen.ClassLoader)
        ///     "/>
        /// is inherited by any
        /// <see cref="WrappedRecordReader{K, U}"/>
        /// s created by
        /// <see cref="CompositeRecordReader{K, V, X}"/>
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestClassLoader()
        {
            Configuration conf = new Configuration();

            TestWrappedRRClassloader.Fake_ClassLoader classLoader = new TestWrappedRRClassloader.Fake_ClassLoader
                                                                        ();
            conf.SetClassLoader(classLoader);
            NUnit.Framework.Assert.IsTrue(conf.GetClassLoader() is TestWrappedRRClassloader.Fake_ClassLoader
                                          );
            FileSystem fs      = FileSystem.Get(conf);
            Path       testdir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).MakeQualified
                                     (fs);
            Path @base = new Path(testdir, "/empty");

            Path[] src = new Path[] { new Path(@base, "i0"), new Path("i1"), new Path("i2") };
            conf.Set(CompositeInputFormat.JoinExpr, CompositeInputFormat.Compose("outer", typeof(
                                                                                     TestWrappedRRClassloader.IF_ClassLoaderChecker), src));
            CompositeInputFormat <NullWritable> inputFormat = new CompositeInputFormat <NullWritable
                                                                                        >();
            // create dummy TaskAttemptID
            TaskAttemptID tid = new TaskAttemptID("jt", 1, TaskType.Map, 0, 0);

            conf.Set(MRJobConfig.TaskAttemptId, tid.ToString());
            inputFormat.CreateRecordReader(inputFormat.GetSplits(Job.GetInstance(conf))[0], new
                                           TaskAttemptContextImpl(conf, tid));
        }
        /// <exception cref="System.IO.IOException"/>
        public virtual void CleanUpPartialOutputForTask(TaskAttemptContext context)
        {
            // we double check this is never invoked from a non-preemptable subclass.
            // This should never happen, since the invoking codes is checking it too,
            // but it is safer to double check. Errors handling this would produce
            // inconsistent output.
            if (!this.GetType().IsAnnotationPresent(typeof(Checkpointable)))
            {
                throw new InvalidOperationException("Invoking cleanUpPartialOutputForTask() " + "from non @Preemptable class"
                                                    );
            }
            FileSystem fs = FsFor(GetTaskAttemptPath(context), context.GetConfiguration());

            Log.Info("cleanUpPartialOutputForTask: removing everything belonging to " + context
                     .GetTaskAttemptID().GetTaskID() + " in: " + GetCommittedTaskPath(context).GetParent
                         ());
            TaskAttemptID taid    = context.GetTaskAttemptID();
            TaskID        tid     = taid.GetTaskID();
            Path          pCommit = GetCommittedTaskPath(context).GetParent();

            // remove any committed output
            for (int i = 0; i < taid.GetId(); ++i)
            {
                TaskAttemptID oldId = new TaskAttemptID(tid, i);
                Path          pTask = new Path(pCommit, oldId.ToString());
                if (fs.Exists(pTask) && !fs.Delete(pTask, true))
                {
                    throw new IOException("Failed to delete " + pTask);
                }
            }
        }
        internal ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf)
        {
            this.numCopiers = jobConf.GetInt(MRJobConfig.ShuffleParallelCopies, 5);
            MetricsContext metricsContext = MetricsUtil.GetContext("mapred");

            this.shuffleMetrics = MetricsUtil.CreateRecord(metricsContext, "shuffleInput");
            this.shuffleMetrics.SetTag("user", jobConf.GetUser());
            this.shuffleMetrics.SetTag("jobName", jobConf.GetJobName());
            this.shuffleMetrics.SetTag("jobId", reduceId.GetJobID().ToString());
            this.shuffleMetrics.SetTag("taskId", reduceId.ToString());
            this.shuffleMetrics.SetTag("sessionId", jobConf.GetSessionId());
            metricsContext.RegisterUpdater(this);
        }
Esempio n. 5
0
 public virtual object GetDatum()
 {
     if (datum == null)
     {
         datum        = new TaskFinished();
         datum.taskid = new Utf8(taskid.ToString());
         if (successfulAttemptId != null)
         {
             datum.successfulAttemptId = new Utf8(successfulAttemptId.ToString());
         }
         datum.finishTime = finishTime;
         datum.counters   = EventWriter.ToAvro(counters);
         datum.taskType   = new Utf8(taskType.ToString());
         datum.status     = new Utf8(status);
     }
     return(datum);
 }
Esempio n. 6
0
 /// <summary>Print all the information about this attempt.</summary>
 public virtual void PrintAll()
 {
     System.Console.Out.WriteLine("ATTEMPT_ID:" + attemptId.ToString());
     System.Console.Out.WriteLine("START_TIME: " + startTime);
     System.Console.Out.WriteLine("FINISH_TIME:" + finishTime);
     System.Console.Out.WriteLine("ERROR:" + error);
     System.Console.Out.WriteLine("TASK_STATUS:" + status);
     System.Console.Out.WriteLine("STATE:" + state);
     System.Console.Out.WriteLine("TASK_TYPE:" + taskType);
     System.Console.Out.WriteLine("TRACKER_NAME:" + trackerName);
     System.Console.Out.WriteLine("HTTP_PORT:" + httpPort);
     System.Console.Out.WriteLine("SHUFFLE_PORT:" + shufflePort);
     System.Console.Out.WriteLine("CONTIANER_ID:" + containerId);
     if (counters != null)
     {
         System.Console.Out.WriteLine("COUNTERS:" + counters.ToString());
     }
 }
 public virtual object GetDatum()
 {
     if (datum == null)
     {
         datum            = new TaskAttemptFinished();
         datum.taskid     = new Utf8(attemptId.GetTaskID().ToString());
         datum.attemptId  = new Utf8(attemptId.ToString());
         datum.taskType   = new Utf8(taskType.ToString());
         datum.taskStatus = new Utf8(taskStatus);
         datum.finishTime = finishTime;
         if (rackName != null)
         {
             datum.rackname = new Utf8(rackName);
         }
         datum.hostname = new Utf8(hostname);
         datum.state    = new Utf8(state);
         datum.counters = EventWriter.ToAvro(counters);
     }
     return(datum);
 }
Esempio n. 8
0
        /// <summary>test a getters of TaskAttemptFinishedEvent and TaskAttemptFinished</summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestTaskAttemptFinishedEvent()
        {
            JobID                    jid           = new JobID("001", 1);
            TaskID                   tid           = new TaskID(jid, TaskType.Reduce, 2);
            TaskAttemptID            taskAttemptId = new TaskAttemptID(tid, 3);
            Counters                 counters      = new Counters();
            TaskAttemptFinishedEvent test          = new TaskAttemptFinishedEvent(taskAttemptId, TaskType
                                                                                  .Reduce, "TEST", 123L, "RAKNAME", "HOSTNAME", "STATUS", counters);

            NUnit.Framework.Assert.AreEqual(test.GetAttemptId().ToString(), taskAttemptId.ToString
                                                ());
            NUnit.Framework.Assert.AreEqual(test.GetCounters(), counters);
            NUnit.Framework.Assert.AreEqual(test.GetFinishTime(), 123L);
            NUnit.Framework.Assert.AreEqual(test.GetHostname(), "HOSTNAME");
            NUnit.Framework.Assert.AreEqual(test.GetRackName(), "RAKNAME");
            NUnit.Framework.Assert.AreEqual(test.GetState(), "STATUS");
            NUnit.Framework.Assert.AreEqual(test.GetTaskId(), tid);
            NUnit.Framework.Assert.AreEqual(test.GetTaskStatus(), "TEST");
            NUnit.Framework.Assert.AreEqual(test.GetTaskType(), TaskType.Reduce);
        }
 /// <summary>Create an event to record the start of an attempt</summary>
 /// <param name="attemptId">Id of the attempt</param>
 /// <param name="taskType">Type of task</param>
 /// <param name="startTime">Start time of the attempt</param>
 /// <param name="trackerName">Name of the Task Tracker where attempt is running</param>
 /// <param name="httpPort">The port number of the tracker</param>
 /// <param name="shufflePort">The shuffle port number of the container</param>
 /// <param name="containerId">The containerId for the task attempt.</param>
 /// <param name="locality">The locality of the task attempt</param>
 /// <param name="avataar">The avataar of the task attempt</param>
 public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType, long startTime
                                , string trackerName, int httpPort, int shufflePort, ContainerId containerId, string
                                locality, string avataar)
 {
     datum.attemptId   = new Utf8(attemptId.ToString());
     datum.taskid      = new Utf8(attemptId.GetTaskID().ToString());
     datum.startTime   = startTime;
     datum.taskType    = new Utf8(taskType.ToString());
     datum.trackerName = new Utf8(trackerName);
     datum.httpPort    = httpPort;
     datum.shufflePort = shufflePort;
     datum.containerId = new Utf8(containerId.ToString());
     if (locality != null)
     {
         datum.locality = new Utf8(locality);
     }
     if (avataar != null)
     {
         datum.avataar = new Utf8(avataar);
     }
 }
Esempio n. 10
0
 public virtual object GetDatum()
 {
     if (datum == null)
     {
         datum                    = new TaskFailed();
         datum.taskid             = new Utf8(id.ToString());
         datum.error              = new Utf8(error);
         datum.finishTime         = finishTime;
         datum.taskType           = new Utf8(taskType.ToString());
         datum.failedDueToAttempt = failedDueToAttempt == null ? null : new Utf8(failedDueToAttempt
                                                                                 .ToString());
         datum.status   = new Utf8(status);
         datum.counters = EventWriter.ToAvro(counters);
     }
     return(datum);
 }