internal ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf) { this.numCopiers = jobConf.GetInt(MRJobConfig.ShuffleParallelCopies, 5); MetricsContext metricsContext = MetricsUtil.GetContext("mapred"); this.shuffleMetrics = MetricsUtil.CreateRecord(metricsContext, "shuffleInput"); this.shuffleMetrics.SetTag("user", jobConf.GetUser()); this.shuffleMetrics.SetTag("jobName", jobConf.GetJobName()); this.shuffleMetrics.SetTag("jobId", reduceId.GetJobID().ToString()); this.shuffleMetrics.SetTag("taskId", reduceId.ToString()); this.shuffleMetrics.SetTag("sessionId", jobConf.GetSessionId()); metricsContext.RegisterUpdater(this); }
public virtual void Configure(JobConf jobConf) { int numberOfThreads = jobConf.GetInt(MultithreadedMapper.NumThreads, 10); if (Log.IsDebugEnabled()) { Log.Debug("Configuring jobConf " + jobConf.GetJobName() + " to use " + numberOfThreads + " threads"); } this.job = jobConf; //increment processed counter only if skipping feature is enabled this.incrProcCount = SkipBadRecords.GetMapperMaxSkipRecords(job) > 0 && SkipBadRecords .GetAutoIncrMapperProcCount(job); this.mapper = ReflectionUtils.NewInstance(jobConf.GetMapperClass(), jobConf); // Creating a threadpool of the configured size to execute the Mapper // map method in parallel. executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit .Milliseconds, new MultithreadedMapRunner.BlockingArrayQueue(numberOfThreads)); }
/// <summary>Get the user-specified job name.</summary> /// <remarks> /// Get the user-specified job name. This is only used to identify the /// job to the user. /// </remarks> /// <returns>the job's name, defaulting to "".</returns> public virtual string GetJobName() { return(conf.GetJobName()); }