/// <summary>Get the new configuration.</summary>
/// <param name="job">the job's configuration</param>
public override void Configure(JobConf job)
{
this.job = job;
//disable the auto increment of the counter. For pipes, no of processed
//records could be different(equal or less) than the no of records input.
SkipBadRecords.SetAutoIncrMapperProcCount(job, false);
}
public virtual void Configure(JobConf job)
{
this.job = job;
//disable the auto increment of the counter. For pipes, no of processed
//records could be different(equal or less) than the no of records input.
SkipBadRecords.SetAutoIncrReducerProcCount(job, false);
skipping = job.GetBoolean(MRJobConfig.SkipRecords, false);
}
public virtual void Configure(JobConf jobConf)
{
int numberOfThreads = jobConf.GetInt(MultithreadedMapper.NumThreads, 10);
if (Log.IsDebugEnabled())
{
Log.Debug("Configuring jobConf " + jobConf.GetJobName() + " to use " + numberOfThreads
+ " threads");
}
this.job = jobConf;
//increment processed counter only if skipping feature is enabled
this.incrProcCount = SkipBadRecords.GetMapperMaxSkipRecords(job) > 0 && SkipBadRecords
.GetAutoIncrMapperProcCount(job);
this.mapper = ReflectionUtils.NewInstance(jobConf.GetMapperClass(), jobConf);
// Creating a threadpool of the configured size to execute the Mapper
// map method in parallel.
executorService = new ThreadPoolExecutor(numberOfThreads, numberOfThreads, 0L, TimeUnit
.Milliseconds, new MultithreadedMapRunner.BlockingArrayQueue(numberOfThreads));
}