public static void Cleanup()
{
server.Stop();
FileUtil.FullyDelete(new FilePath(Basedir));
KeyStoreTestUtil.CleanupSSLConfig(keystoresDir, sslConfDir);
clientSslFactory.Destroy();
}
internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V>
scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics
, ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id)
{
/* Default read timeout (in milliseconds) */
// Initiative value is 0, which means it hasn't retried yet.
this.jobConf = job;
this.reporter = reporter;
this.scheduler = scheduler;
this.merger = merger;
this.metrics = metrics;
this.exceptionReporter = exceptionReporter;
this.id = id;
this.reduce = reduceId.GetTaskID().GetId();
this.shuffleSecretKey = shuffleKey;
ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString
());
wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength
.ToString());
badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString
());
wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap
.ToString());
connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection
.ToString());
wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce
.ToString());
this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout
);
this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout);
this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig
.DefaultShuffleFetchRetryIntervalMs);
this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout
);
bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled
, YarnConfiguration.DefaultNmRecoveryEnabled);
this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault
);
SetName("fetcher#" + id);
SetDaemon(true);
lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher))
{
sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault
);
if (sslShuffle && sslFactory == null)
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, job);
try
{
sslFactory.Init();
}
catch (Exception ex)
{
sslFactory.Destroy();
throw new RuntimeException(ex);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void ShutDown()
{
this.stopped = true;
Interrupt();
try
{
Join(5000);
}
catch (Exception ie)
{
Log.Warn("Got interrupt while joining " + GetName(), ie);
}
if (sslFactory != null)
{
sslFactory.Destroy();
}
}
/// <summary>Shutdown valueQueue executor threads</summary>
/// <exception cref="System.IO.IOException"/>
public override void Close()
{
try
{
encKeyVersionQueue.Shutdown();
}
catch (Exception e)
{
throw new IOException(e);
}
finally
{
if (sslFactory != null)
{
sslFactory.Destroy();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close()
{
bossGroup.ShutdownGracefully();
workerGroup.ShutdownGracefully();
if (sslFactory != null)
{
sslFactory.Destroy();
}
if (externalHttpChannel != null)
{
externalHttpChannel.Close();
}
try
{
infoServer.Stop();
}
catch (Exception e)
{
throw new IOException(e);
}
}
