Exemple #1
0
            public virtual TaskAttemptState GetState()
            {
                if (this.overridingState != null)
                {
                    return(this.overridingState);
                }
                TaskAttemptState result = this.GetProgress() < 1.0F ? TaskAttemptState.Running :
                                          TaskAttemptState.Succeeded;

                if (result == TaskAttemptState.Succeeded)
                {
                    this.overridingState = TaskAttemptState.Succeeded;
                    System.Console.Out.WriteLine("MyTaskAttemptImpl.getState() -- attempt " + this.myAttemptID
                                                 + " finished.");
                    this._enclosing.slotsInUse.AddAndGet(-this._enclosing.TaskTypeSlots(this.myAttemptID
                                                                                        .GetTaskId().GetTaskType()));
                    (this.myAttemptID.GetTaskId().GetTaskType() == TaskType.Map ? this._enclosing.completedMaps
                                                 : this._enclosing.completedReduces).GetAndIncrement();
                    // check for a spectacularly successful speculation
                    TaskId taskID = this.myAttemptID.GetTaskId();
                    Task   task   = this._enclosing.myJob.GetTask(taskID);
                    foreach (TaskAttempt otherAttempt in task.GetAttempts().Values)
                    {
                        if (otherAttempt != this && otherAttempt.GetState() == TaskAttemptState.Running)
                        {
                            // we had two instances running.  Try to determine how much
                            //  we might have saved by speculation
                            if (this.GetID().GetId() > otherAttempt.GetID().GetId())
                            {
                                // the speculation won
                                this._enclosing.successfulSpeculations.GetAndIncrement();
                                float hisProgress  = otherAttempt.GetProgress();
                                long  hisStartTime = ((TestRuntimeEstimators.MyTaskAttemptImpl)otherAttempt).startMockTime;
                                System.Console.Out.WriteLine("TLTRE:A speculation finished at time " + this._enclosing
                                                             .clock.GetTime() + ".  The stalled attempt is at " + (hisProgress * 100.0) + "% progress, and it started at "
                                                             + hisStartTime + ", which is " + (this._enclosing.clock.GetTime() - hisStartTime
                                                                                               ) + " ago.");
                                long originalTaskEndEstimate = (hisStartTime + this._enclosing.estimator.EstimatedRuntime
                                                                    (otherAttempt.GetID()));
                                System.Console.Out.WriteLine("TLTRE: We would have expected the original attempt to take "
                                                             + this._enclosing.estimator.EstimatedRuntime(otherAttempt.GetID()) + ", finishing at "
                                                             + originalTaskEndEstimate);
                                long estimatedSavings = originalTaskEndEstimate - this._enclosing.clock.GetTime();
                                this._enclosing.taskTimeSavedBySpeculation.AddAndGet(estimatedSavings);
                                System.Console.Out.WriteLine("TLTRE: The task is " + task.GetID());
                                this._enclosing.slotsInUse.AddAndGet(-this._enclosing.TaskTypeSlots(this.myAttemptID
                                                                                                    .GetTaskId().GetTaskType()));
                                ((TestRuntimeEstimators.MyTaskAttemptImpl)otherAttempt).overridingState = TaskAttemptState
                                                                                                          .Killed;
                            }
                            else
                            {
                                System.Console.Out.WriteLine("TLTRE: The normal attempt beat the speculation in "
                                                             + task.GetID());
                            }
                        }
                    }
                }
                return(result);
            }
 public virtual void SetTaskAttemptState(TaskAttemptState taskAttemptState)
 {
     MaybeInitBuilder();
     if (taskAttemptState == null)
     {
         builder.ClearTaskAttemptState();
         return;
     }
     builder.SetTaskAttemptState(ConvertToProtoFormat(taskAttemptState));
 }
Exemple #3
0
            internal virtual void StartUp()
            {
                this.startMockTime   = this._enclosing.clock.GetTime();
                this.overridingState = null;
                this._enclosing.slotsInUse.AddAndGet(this._enclosing.TaskTypeSlots(this.myAttemptID
                                                                                   .GetTaskId().GetTaskType()));
                System.Console.Out.WriteLine("TLTRE.MyTaskAttemptImpl.startUp starting " + this.GetID
                                                 ());
                SpeculatorEvent @event = new SpeculatorEvent(this.GetID().GetTaskId(), -1);

                this._enclosing.dispatcher.GetEventHandler().Handle(@event);
            }
Exemple #4
0
        /// <exception cref="System.Exception"/>
        public virtual void WaitForState(TaskAttempt attempt, TaskAttemptState finalState
                                         )
        {
            int timeoutSecs          = 0;
            TaskAttemptReport report = attempt.GetReport();

            while (!finalState.Equals(report.GetTaskAttemptState()) && timeoutSecs++ < 20)
            {
                System.Console.Out.WriteLine("TaskAttempt State is : " + report.GetTaskAttemptState
                                                 () + " Waiting for state : " + finalState + "   progress : " + report.GetProgress
                                                 ());
                report = attempt.GetReport();
                Sharpen.Thread.Sleep(500);
            }
            System.Console.Out.WriteLine("TaskAttempt State is : " + report.GetTaskAttemptState
                                             ());
            NUnit.Framework.Assert.AreEqual("TaskAttempt state is not correct (timedout)", finalState
                                            , report.GetTaskAttemptState());
        }
Exemple #5
0
 internal CompletedTaskAttempt(TaskId taskId, JobHistoryParser.TaskAttemptInfo attemptInfo
                               )
 {
     this.attemptInfo = attemptInfo;
     this.attemptId   = TypeConverter.ToYarn(attemptInfo.GetAttemptId());
     if (attemptInfo.GetTaskStatus() != null)
     {
         this.state = TaskAttemptState.ValueOf(attemptInfo.GetTaskStatus());
     }
     else
     {
         this.state       = TaskAttemptState.Killed;
         localDiagMessage = "Attmpt state missing from History : marked as KILLED";
         diagnostics.AddItem(localDiagMessage);
     }
     if (attemptInfo.GetError() != null)
     {
         diagnostics.AddItem(attemptInfo.GetError());
     }
 }
Exemple #6
0
        public TaskAttemptInfo(TaskAttempt ta, TaskType type, bool isRunning)
        {
            TaskAttemptReport report = ta.GetReport();

            this.type                = type.ToString();
            this.id                  = MRApps.ToString(ta.GetID());
            this.nodeHttpAddress     = ta.GetNodeHttpAddress();
            this.startTime           = report.GetStartTime();
            this.finishTime          = report.GetFinishTime();
            this.assignedContainerId = ConverterUtils.ToString(report.GetContainerId());
            this.assignedContainer   = report.GetContainerId();
            this.progress            = report.GetProgress() * 100;
            this.status              = report.GetStateString();
            this.state               = report.GetTaskAttemptState();
            this.elapsedTime         = Times.Elapsed(this.startTime, this.finishTime, isRunning);
            if (this.elapsedTime == -1)
            {
                this.elapsedTime = 0;
            }
            this.diagnostics = report.GetDiagnosticInfo();
            this.rack        = ta.GetNodeRackName();
        }
Exemple #7
0
 public static TaskAttemptState ConvertFromProtoFormat(MRProtos.TaskAttemptStateProto
                                                       e)
 {
     return(TaskAttemptState.ValueOf(e.ToString().Replace(TaskAttemptStatePrefix, string.Empty
                                                          )));
 }
Exemple #8
0
 /*
  * TaskAttemptState
  */
 public static MRProtos.TaskAttemptStateProto ConvertToProtoFormat(TaskAttemptState
                                                                   e)
 {
     return(MRProtos.TaskAttemptStateProto.ValueOf(TaskAttemptStatePrefix + e.ToString
                                                       ()));
 }
Exemple #9
0
 private void UpdateLastAttemptState(TaskAttemptState s)
 {
     GetLastAttempt().SetState(s);
 }
Exemple #10
0
 public virtual void SetState(TaskAttemptState state)
 {
     this.state = state;
 }
Exemple #11
0
 // The speculative attempt may be not killed before the MR job succeeds.
 private TaskAttemptStatusUpdateEvent.TaskAttemptStatus CreateTaskAttemptStatus(TaskAttemptId
                                                                                id, float progress, TaskAttemptState state)
 {
     TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
                                                                 ();
     status.id        = id;
     status.progress  = progress;
     status.taskState = state;
     return(status);
 }
 private MRProtos.TaskAttemptStateProto ConvertToProtoFormat(TaskAttemptState e)
 {
     return(MRProtoUtils.ConvertToProtoFormat(e));
 }
Exemple #13
0
 public bool CorrespondsTo(TaskAttemptState state)
 {
     return(this.correspondingStates.Contains(state));
 }
Exemple #14
0
        public virtual void TestAttemptsBlock()
        {
            AppContext ctx  = Org.Mockito.Mockito.Mock <AppContext>();
            AppForTest app  = new AppForTest(ctx);
            Task       task = GetTask(0);
            IDictionary <TaskAttemptId, TaskAttempt> attempts = new Dictionary <TaskAttemptId,
                                                                                TaskAttempt>();
            TaskAttempt   attempt = Org.Mockito.Mockito.Mock <TaskAttempt>();
            TaskAttemptId taId    = new TaskAttemptIdPBImpl();

            taId.SetId(0);
            taId.SetTaskId(task.GetID());
            Org.Mockito.Mockito.When(attempt.GetID()).ThenReturn(taId);
            Org.Mockito.Mockito.When(attempt.GetNodeHttpAddress()).ThenReturn("Node address");
            ApplicationId        appId        = ApplicationIdPBImpl.NewInstance(0, 5);
            ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.NewInstance(appId,
                                                                                       1);
            ContainerId containerId = ContainerIdPBImpl.NewContainerId(appAttemptId, 1);

            Org.Mockito.Mockito.When(attempt.GetAssignedContainerID()).ThenReturn(containerId
                                                                                  );
            Org.Mockito.Mockito.When(attempt.GetAssignedContainerMgrAddress()).ThenReturn("assignedContainerMgrAddress"
                                                                                          );
            Org.Mockito.Mockito.When(attempt.GetNodeRackName()).ThenReturn("nodeRackName");
            long             taStartTime         = 100002L;
            long             taFinishTime        = 100012L;
            long             taShuffleFinishTime = 100010L;
            long             taSortFinishTime    = 100011L;
            TaskAttemptState taState             = TaskAttemptState.Succeeded;

            Org.Mockito.Mockito.When(attempt.GetLaunchTime()).ThenReturn(taStartTime);
            Org.Mockito.Mockito.When(attempt.GetFinishTime()).ThenReturn(taFinishTime);
            Org.Mockito.Mockito.When(attempt.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
                                                                                );
            Org.Mockito.Mockito.When(attempt.GetSortFinishTime()).ThenReturn(taSortFinishTime
                                                                             );
            Org.Mockito.Mockito.When(attempt.GetState()).ThenReturn(taState);
            TaskAttemptReport taReport = Org.Mockito.Mockito.Mock <TaskAttemptReport>();

            Org.Mockito.Mockito.When(taReport.GetStartTime()).ThenReturn(taStartTime);
            Org.Mockito.Mockito.When(taReport.GetFinishTime()).ThenReturn(taFinishTime);
            Org.Mockito.Mockito.When(taReport.GetShuffleFinishTime()).ThenReturn(taShuffleFinishTime
                                                                                 );
            Org.Mockito.Mockito.When(taReport.GetSortFinishTime()).ThenReturn(taSortFinishTime
                                                                              );
            Org.Mockito.Mockito.When(taReport.GetContainerId()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(taReport.GetProgress()).ThenReturn(1.0f);
            Org.Mockito.Mockito.When(taReport.GetStateString()).ThenReturn("Processed 128/128 records <p> \n"
                                                                           );
            Org.Mockito.Mockito.When(taReport.GetTaskAttemptState()).ThenReturn(taState);
            Org.Mockito.Mockito.When(taReport.GetDiagnosticInfo()).ThenReturn(string.Empty);
            Org.Mockito.Mockito.When(attempt.GetReport()).ThenReturn(taReport);
            attempts[taId] = attempt;
            Org.Mockito.Mockito.When(task.GetAttempts()).ThenReturn(attempts);
            app.SetTask(task);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
                                                                                       >();
            Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("User");
            app.SetJob(job);
            TestBlocks.AttemptsBlockForTest block = new TestBlocks.AttemptsBlockForTest(this,
                                                                                        app);
            block.AddParameter(AMParams.TaskType, "r");
            PrintWriter pWriter = new PrintWriter(data);

            HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter
                                                    , 0, false);
            block.Render(html);
            pWriter.Flush();
            // should be printed information about attempts
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("0 attempt_0_0001_r_000000_0"
                                                                   ));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED"));
            NUnit.Framework.Assert.IsFalse(data.ToString().Contains("Processed 128/128 records <p> \n"
                                                                    ));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("Processed 128\\/128 records &lt;p&gt; \\n"
                                                                   ));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"******"100002"));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100010"));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100011"));
            NUnit.Framework.Assert.IsTrue(data.ToString().Contains("100012"));
        }