コード例 #1
0
ファイル: JobInfo.cs プロジェクト: orf53975/hadoop.net
        public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job)
        {
            this.id = MRApps.ToString(job.GetID());
            JobReport report = job.GetReport();

            this.mapsTotal        = job.GetTotalMaps();
            this.mapsCompleted    = job.GetCompletedMaps();
            this.reducesTotal     = job.GetTotalReduces();
            this.reducesCompleted = job.GetCompletedReduces();
            this.submitTime       = report.GetSubmitTime();
            this.startTime        = report.GetStartTime();
            this.finishTime       = report.GetFinishTime();
            this.name             = job.GetName().ToString();
            this.queue            = job.GetQueueName();
            this.user             = job.GetUserName();
            this.state            = job.GetState().ToString();
            this.acls             = new AList <ConfEntryInfo>();
            if (job is CompletedJob)
            {
                avgMapTime               = 0l;
                avgReduceTime            = 0l;
                avgShuffleTime           = 0l;
                avgMergeTime             = 0l;
                failedReduceAttempts     = 0;
                killedReduceAttempts     = 0;
                successfulReduceAttempts = 0;
                failedMapAttempts        = 0;
                killedMapAttempts        = 0;
                successfulMapAttempts    = 0;
                CountTasksAndAttempts(job);
                this.uberized    = job.IsUber();
                this.diagnostics = string.Empty;
                IList <string> diagnostics = job.GetDiagnostics();
                if (diagnostics != null && !diagnostics.IsEmpty())
                {
                    StringBuilder b = new StringBuilder();
                    foreach (string diag in diagnostics)
                    {
                        b.Append(diag);
                    }
                    this.diagnostics = b.ToString();
                }
                IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
                if (allacls != null)
                {
                    foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
                    {
                        this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
                                                                ()));
                    }
                }
            }
        }
コード例 #2
0
ファイル: JobInfo.cs プロジェクト: orf53975/hadoop.net
        public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess)
        {
            // ok for any user to see
            // these should only be seen if acls allow
            this.id = MRApps.ToString(job.GetID());
            JobReport report = job.GetReport();

            this.startTime   = report.GetStartTime();
            this.finishTime  = report.GetFinishTime();
            this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime);
            if (this.elapsedTime == -1)
            {
                this.elapsedTime = 0;
            }
            this.name                  = job.GetName().ToString();
            this.user                  = job.GetUserName();
            this.state                 = job.GetState();
            this.mapsTotal             = job.GetTotalMaps();
            this.mapsCompleted         = job.GetCompletedMaps();
            this.mapProgress           = report.GetMapProgress() * 100;
            this.mapProgressPercent    = StringHelper.Percent(report.GetMapProgress());
            this.reducesTotal          = job.GetTotalReduces();
            this.reducesCompleted      = job.GetCompletedReduces();
            this.reduceProgress        = report.GetReduceProgress() * 100;
            this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress());
            this.acls                  = new AList <ConfEntryInfo>();
            if (hasAccess)
            {
                this.diagnostics = string.Empty;
                CountTasksAndAttempts(job);
                this.uberized = job.IsUber();
                IList <string> diagnostics = job.GetDiagnostics();
                if (diagnostics != null && !diagnostics.IsEmpty())
                {
                    StringBuilder b = new StringBuilder();
                    foreach (string diag in diagnostics)
                    {
                        b.Append(diag);
                    }
                    this.diagnostics = b.ToString();
                }
                IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs();
                if (allacls != null)
                {
                    foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls)
                    {
                        this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString
                                                                ()));
                    }
                }
            }
        }
コード例 #3
0
 //Empty
 public override IList <string> GetDiagnostics()
 {
     return(job.GetDiagnostics());
 }
コード例 #4
0
 public virtual IList <string> GetDiagnostics()
 {
     return(mockJob.GetDiagnostics());
 }
コード例 #5
0
ファイル: RMCommunicator.cs プロジェクト: orf53975/hadoop.net
        protected internal virtual void DoUnregistration()
        {
            FinalApplicationStatus finishState = FinalApplicationStatus.Undefined;
            JobImpl jobImpl = (JobImpl)job;

            if (jobImpl.GetInternalState() == JobStateInternal.Succeeded)
            {
                finishState = FinalApplicationStatus.Succeeded;
            }
            else
            {
                if (jobImpl.GetInternalState() == JobStateInternal.Killed || (jobImpl.GetInternalState
                                                                                  () == JobStateInternal.Running && isSignalled))
                {
                    finishState = FinalApplicationStatus.Killed;
                }
                else
                {
                    if (jobImpl.GetInternalState() == JobStateInternal.Failed || jobImpl.GetInternalState
                            () == JobStateInternal.Error)
                    {
                        finishState = FinalApplicationStatus.Failed;
                    }
                }
            }
            StringBuilder sb = new StringBuilder();

            foreach (string s in job.GetDiagnostics())
            {
                sb.Append(s).Append("\n");
            }
            Log.Info("Setting job diagnostics to " + sb.ToString());
            string historyUrl = MRWebAppUtil.GetApplicationWebURLOnJHSWithScheme(GetConfig(),
                                                                                 context.GetApplicationID());

            Log.Info("History url is " + historyUrl);
            FinishApplicationMasterRequest request = FinishApplicationMasterRequest.NewInstance
                                                         (finishState, sb.ToString(), historyUrl);

            try
            {
                while (true)
                {
                    FinishApplicationMasterResponse response = scheduler.FinishApplicationMaster(request
                                                                                                 );
                    if (response.GetIsUnregistered())
                    {
                        // When excepting ClientService, other services are already stopped,
                        // it is safe to let clients know the final states. ClientService
                        // should wait for some time so clients have enough time to know the
                        // final states.
                        MRAppMaster.RunningAppContext raContext = (MRAppMaster.RunningAppContext)context;
                        raContext.MarkSuccessfulUnregistration();
                        break;
                    }
                    Log.Info("Waiting for application to be successfully unregistered.");
                    Sharpen.Thread.Sleep(rmPollInterval);
                }
            }
            catch (ApplicationMasterNotRegisteredException)
            {
                // RM might have restarted or failed over and so lost the fact that AM had
                // registered before.
                Register();
                DoUnregistration();
            }
        }
コード例 #6
0
 /// <exception cref="System.Exception"/>
 public virtual void TestDiagnosticsForKilledJob()
 {
     Log.Info("STARTING testDiagnosticsForKilledJob");
     try
     {
         Configuration conf = new Configuration();
         conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
                       typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
         RackResolver.Init(conf);
         MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithJobKilled(2, 1, true, this
                                                                             .GetType().FullName, true);
         app.Submit(conf);
         Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
                                                          .GetEnumerator().Next();
         JobId jobId = job.GetID();
         app.WaitForState(job, JobState.Killed);
         // make sure all events are flushed
         app.WaitForState(Service.STATE.Stopped);
         JobHistory jobHistory = new JobHistory();
         jobHistory.Init(conf);
         HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
         JobHistoryParser         parser;
         JobHistoryParser.JobInfo jobInfo;
         lock (fileInfo)
         {
             Path historyFilePath  = fileInfo.GetHistoryFile();
             FSDataInputStream @in = null;
             FileContext       fc  = null;
             try
             {
                 fc  = FileContext.GetFileContext(conf);
                 @in = fc.Open(fc.MakeQualified(historyFilePath));
             }
             catch (IOException ioe)
             {
                 Log.Info("Can not open history file: " + historyFilePath, ioe);
                 throw (new Exception("Can not open History File"));
             }
             parser  = new JobHistoryParser(@in);
             jobInfo = parser.Parse();
         }
         Exception parseException = parser.GetParseException();
         NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException
                                       );
         IList <string> originalDiagnostics = job.GetDiagnostics();
         string         historyError        = jobInfo.GetErrorInfo();
         NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics
                                       != null && !originalDiagnostics.IsEmpty());
         NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError
                                          );
         foreach (string diagString in originalDiagnostics)
         {
             NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString));
         }
         NUnit.Framework.Assert.IsTrue("No killed message in diagnostics", historyError.Contains
                                           (JobImpl.JobKilledDiag));
     }
     finally
     {
         Log.Info("FINISHED testDiagnosticsForKilledJob");
     }
 }