internal ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf)
{
this.numCopiers = jobConf.GetInt(MRJobConfig.ShuffleParallelCopies, 5);
MetricsContext metricsContext = MetricsUtil.GetContext("mapred");
this.shuffleMetrics = MetricsUtil.CreateRecord(metricsContext, "shuffleInput");
this.shuffleMetrics.SetTag("user", jobConf.GetUser());
this.shuffleMetrics.SetTag("jobName", jobConf.GetJobName());
this.shuffleMetrics.SetTag("jobId", reduceId.GetJobID().ToString());
this.shuffleMetrics.SetTag("taskId", reduceId.ToString());
this.shuffleMetrics.SetTag("sessionId", jobConf.GetSessionId());
metricsContext.RegisterUpdater(this);
}
public virtual void Configure(JobConf jobConf)
{
int numberOfThreads = jobConf.GetInt(MultithreadedMapper.NumThreads, 10);
if (Log.IsDebugEnabled())
{
Log.Debug("Configuring jobConf " + jobConf.GetJobName() + " to use " + numberOfThreads
+ " threads");
}
this.job = jobConf;
//increment processed counter only if skipping feature is enabled
this.incrProcCount = SkipBadRecords.GetMapperMaxSkipRecords(job) > 0 && SkipBadRecords
.GetAutoIncrMapperProcCount(job);
this.mapper = ReflectionUtils.NewInstance(jobConf.GetMapperClass(), jobConf);
// Creating a threadpool of the configured size to execute the Mapper
// map method in parallel.
executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit
.Milliseconds, new MultithreadedMapRunner.BlockingArrayQueue(numberOfThreads));
}
/// <summary>Get the user-specified job name.</summary>
/// <remarks>
/// Get the user-specified job name. This is only used to identify the
/// job to the user.
/// </remarks>
/// <returns>the job's name, defaulting to "".</returns>
public virtual string GetJobName()
{
return(conf.GetJobName());
}