internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V>
scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics
, ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id)
{
/* Default read timeout (in milliseconds) */
// Initiative value is 0, which means it hasn't retried yet.
this.jobConf = job;
this.reporter = reporter;
this.scheduler = scheduler;
this.merger = merger;
this.metrics = metrics;
this.exceptionReporter = exceptionReporter;
this.id = id;
this.reduce = reduceId.GetTaskID().GetId();
this.shuffleSecretKey = shuffleKey;
ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString
());
wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength
.ToString());
badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString
());
wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap
.ToString());
connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection
.ToString());
wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce
.ToString());
this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout
);
this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout);
this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig
.DefaultShuffleFetchRetryIntervalMs);
this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout
);
bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled
, YarnConfiguration.DefaultNmRecoveryEnabled);
this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault
);
SetName("fetcher#" + id);
SetDaemon(true);
lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher))
{
sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault
);
if (sslShuffle && sslFactory == null)
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, job);
try
{
sslFactory.Init();
}
catch (Exception ex)
{
sslFactory.Destroy();
throw new RuntimeException(ex);
}
}
}
}
public static void SetUp()
{
Configuration conf = new Configuration();
conf.Set(HttpServer2.FilterInitializerProperty, typeof(TestHttpCookieFlag.DummyFilterInitializer
).FullName);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSSLHttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
Configuration sslConf = new Configuration(false);
sslConf.AddResource("ssl-server.xml");
sslConf.AddResource("ssl-client.xml");
clientSslFactory = new SSLFactory(SSLFactory.Mode.Client, sslConf);
clientSslFactory.Init();
server = new HttpServer2.Builder().SetName("test").AddEndpoint(new URI("http://localhost"
)).AddEndpoint(new URI("https://localhost")).SetConf(conf).KeyPassword(sslConf.Get
("ssl.server.keystore.keypassword")).KeyStore(sslConf.Get("ssl.server.keystore.location"
), sslConf.Get("ssl.server.keystore.password"), sslConf.Get("ssl.server.keystore.type"
, "jks")).TrustStore(sslConf.Get("ssl.server.truststore.location"), sslConf.Get(
"ssl.server.truststore.password"), sslConf.Get("ssl.server.truststore.type", "jks"
)).Build();
server.AddServlet("echo", "/echo", typeof(TestHttpServer.EchoServlet));
server.Start();
}
public static void Setup()
{
conf = new Configuration();
conf.SetInt(HttpServer2.HttpMaxThreads, 10);
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(TestSSLHttpServer));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
Configuration sslConf = new Configuration(false);
sslConf.AddResource("ssl-server.xml");
sslConf.AddResource("ssl-client.xml");
clientSslFactory = new SSLFactory(SSLFactory.Mode.Client, sslConf);
clientSslFactory.Init();
server = new HttpServer2.Builder().SetName("test").AddEndpoint(new URI("https://localhost"
)).SetConf(conf).KeyPassword(sslConf.Get("ssl.server.keystore.keypassword")).KeyStore
(sslConf.Get("ssl.server.keystore.location"), sslConf.Get("ssl.server.keystore.password"
), sslConf.Get("ssl.server.keystore.type", "jks")).TrustStore(sslConf.Get("ssl.server.truststore.location"
), sslConf.Get("ssl.server.truststore.password"), sslConf.Get("ssl.server.truststore.type"
, "jks")).Build();
server.AddServlet("echo", "/echo", typeof(TestHttpServer.EchoServlet));
server.AddServlet("longheader", "/longheader", typeof(HttpServerFunctionalTest.LongHeaderServlet
));
server.Start();
baseUrl = new Uri("https://" + NetUtils.GetHostPortString(server.GetConnectorAddress
(0)));
Log.Info("HTTP server started: " + baseUrl);
}
/// <exception cref="System.IO.IOException"/>
public KMSClientProvider(URI uri, Configuration conf)
: base(conf)
{
kmsUrl = CreateServiceURL(ExtractKMSPath(uri));
if (Runtime.EqualsIgnoreCase("https", new Uri(kmsUrl).Scheme))
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, conf);
try
{
sslFactory.Init();
}
catch (GeneralSecurityException ex)
{
throw new IOException(ex);
}
}
int timeout = conf.GetInt(TimeoutAttr, DefaultTimeout);
authRetry = conf.GetInt(AuthRetry, DefaultAuthRetry);
configurator = new KMSClientProvider.TimeoutConnConfigurator(timeout, sslFactory);
encKeyVersionQueue = new ValueQueue <KeyProviderCryptoExtension.EncryptedKeyVersion
>(conf.GetInt(CommonConfigurationKeysPublic.KmsClientEncKeyCacheSize, CommonConfigurationKeysPublic
.KmsClientEncKeyCacheSizeDefault), conf.GetFloat(CommonConfigurationKeysPublic.KmsClientEncKeyCacheLowWatermark
, CommonConfigurationKeysPublic.KmsClientEncKeyCacheLowWatermarkDefault), conf.GetInt
(CommonConfigurationKeysPublic.KmsClientEncKeyCacheExpiryMs, CommonConfigurationKeysPublic
.KmsClientEncKeyCacheExpiryDefault), conf.GetInt(CommonConfigurationKeysPublic.KmsClientEncKeyCacheNumRefillThreads
, CommonConfigurationKeysPublic.KmsClientEncKeyCacheNumRefillThreadsDefault), new
KMSClientProvider.EncryptedQueueRefiller(this));
authToken = new DelegationTokenAuthenticatedURL.Token();
actualUgi = (UserGroupInformation.GetCurrentUser().GetAuthenticationMethod() == UserGroupInformation.AuthenticationMethod
.Proxy) ? UserGroupInformation.GetCurrentUser().GetRealUser() : UserGroupInformation
.GetCurrentUser();
}
/// <exception cref="System.Exception"/>
public HttpPipelineFactory(ShuffleHandler _enclosing, Configuration conf)
{
this._enclosing = _enclosing;
this.Shuffle = this._enclosing.GetShuffle(conf);
if (conf.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault
))
{
ShuffleHandler.Log.Info("Encrypted shuffle is enabled.");
this.sslFactory = new SSLFactory(SSLFactory.Mode.Server, conf);
this.sslFactory.Init();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.GeneralSecurityException"/>
private static ConnectionConfigurator NewSslConnConfigurator(int timeout, Configuration
conf)
{
SSLFactory factory;
SSLSocketFactory sf;
HostnameVerifier hv;
factory = new SSLFactory(SSLFactory.Mode.Client, conf);
factory.Init();
sf = factory.CreateSSLSocketFactory();
hv = factory.GetHostnameVerifier();
return(new _ConnectionConfigurator_536(sf, hv, timeout));
}
/// <exception cref="System.IO.IOException"/>
public DatanodeHttpServer(Configuration conf, DataNode datanode, ServerSocketChannel
externalHttpChannel)
{
this.conf = conf;
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.SetInt(HttpServer2.HttpMaxThreads, 10);
HttpServer2.Builder builder = new HttpServer2.Builder().SetName("datanode").SetConf
(confForInfoServer).SetACL(new AccessControlList(conf.Get(DFSConfigKeys.DfsAdmin
, " "))).HostName(GetHostnameForSpnegoPrincipal(confForInfoServer)).AddEndpoint(
URI.Create("http://localhost:0")).SetFindPort(true);
this.infoServer = builder.Build();
this.infoServer.AddInternalServlet(null, "/streamFile/*", typeof(StreamFile));
this.infoServer.AddInternalServlet(null, "/getFileChecksum/*", typeof(FileChecksumServlets.GetServlet
));
this.infoServer.SetAttribute("datanode", datanode);
this.infoServer.SetAttribute(JspHelper.CurrentConf, conf);
this.infoServer.AddServlet(null, "/blockScannerReport", typeof(BlockScanner.Servlet
));
this.infoServer.Start();
IPEndPoint jettyAddr = infoServer.GetConnectorAddress(0);
this.confForCreate = new Configuration(conf);
confForCreate.Set(FsPermission.UmaskLabel, "000");
this.bossGroup = new NioEventLoopGroup();
this.workerGroup = new NioEventLoopGroup();
this.externalHttpChannel = externalHttpChannel;
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
if (policy.IsHttpEnabled())
{
this.httpServer = new ServerBootstrap().Group(bossGroup, workerGroup).ChildHandler
(new _ChannelInitializer_117(this, jettyAddr, conf));
if (externalHttpChannel == null)
{
httpServer.Channel(typeof(NioServerSocketChannel));
}
else
{
httpServer.ChannelFactory(new _ChannelFactory_130(externalHttpChannel));
}
}
else
{
// The channel has been bounded externally via JSVC,
// thus bind() becomes a no-op.
this.httpServer = null;
}
if (policy.IsHttpsEnabled())
{
this.sslFactory = new SSLFactory(SSLFactory.Mode.Server, conf);
try
{
sslFactory.Init();
}
catch (GeneralSecurityException e)
{
throw new IOException(e);
}
this.httpsServer = new ServerBootstrap().Group(bossGroup, workerGroup).Channel(typeof(
NioServerSocketChannel)).ChildHandler(new _ChannelInitializer_155(this, jettyAddr
, conf));
}
else
{
this.httpsServer = null;
this.sslFactory = null;
}
}
