示例#1
0
        public static string GetResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf
                                                                       , HttpConfig.Policy httpPolicy)
        {
            IPEndPoint address = null;
            string     rmId    = null;

            if (HAUtil.IsHAEnabled(conf))
            {
                // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
                // the Active RM
                rmId = (string)Sharpen.Collections.ToArray(HAUtil.GetRMHAIds(conf))[0];
            }
            if (httpPolicy == HttpConfig.Policy.HttpsOnly)
            {
                address = conf.GetSocketAddr(rmId == null ? YarnConfiguration.RmWebappHttpsAddress
                                         : HAUtil.AddSuffix(YarnConfiguration.RmWebappHttpsAddress, rmId), YarnConfiguration
                                             .DefaultRmWebappHttpsAddress, YarnConfiguration.DefaultRmWebappHttpsPort);
            }
            else
            {
                address = conf.GetSocketAddr(rmId == null ? YarnConfiguration.RmWebappAddress : HAUtil
                                             .AddSuffix(YarnConfiguration.RmWebappAddress, rmId), YarnConfiguration.DefaultRmWebappAddress
                                             , YarnConfiguration.DefaultRmWebappPort);
            }
            return(GetResolvedAddress(address));
        }
示例#2
0
        protected override HAServiceTarget ResolveTarget(string rmId)
        {
            ICollection <string> rmIds = HAUtil.GetRMHAIds(GetConf());

            if (!rmIds.Contains(rmId))
            {
                StringBuilder msg = new StringBuilder();
                msg.Append(rmId + " is not a valid serviceId. It should be one of ");
                foreach (string id in rmIds)
                {
                    msg.Append(id + " ");
                }
                throw new ArgumentException(msg.ToString());
            }
            try
            {
                YarnConfiguration conf = new YarnConfiguration(GetConf());
                conf.Set(YarnConfiguration.RmHaId, rmId);
                return(new RMHAServiceTarget(conf));
            }
            catch (ArgumentException)
            {
                throw new YarnRuntimeException("Could not connect to " + rmId + "; the configuration for it might be missing"
                                               );
            }
            catch (IOException)
            {
                throw new YarnRuntimeException("Could not connect to RM HA Admin for node " + rmId
                                               );
            }
        }
        public virtual void Init(Configuration configuration, RMProxy <T> rmProxy, Type protocol
                                 )
        {
            this.rmProxy  = rmProxy;
            this.protocol = protocol;
            this.rmProxy.CheckAllowedProtocols(this.protocol);
            this.conf = new YarnConfiguration(configuration);
            ICollection <string> rmIds = HAUtil.GetRMHAIds(conf);

            this.rmServiceIds = Sharpen.Collections.ToArray(rmIds, new string[rmIds.Count]);
            conf.Set(YarnConfiguration.RmHaId, rmServiceIds[currentProxyIndex]);
            conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, conf.GetInt
                            (YarnConfiguration.ClientFailoverRetries, YarnConfiguration.DefaultClientFailoverRetries
                            ));
            conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey
                        , conf.GetInt(YarnConfiguration.ClientFailoverRetriesOnSocketTimeouts, YarnConfiguration
                                      .DefaultClientFailoverRetriesOnSocketTimeouts));
        }
示例#4
0
 public static Text GetTokenService(Configuration conf, string address, string defaultAddr
                                    , int defaultPort)
 {
     if (HAUtil.IsHAEnabled(conf))
     {
         // Build a list of service addresses to form the service name
         AList <string>    services = new AList <string>();
         YarnConfiguration yarnConf = new YarnConfiguration(conf);
         foreach (string rmId in HAUtil.GetRMHAIds(conf))
         {
             // Set RM_ID to get the corresponding RM_ADDRESS
             yarnConf.Set(YarnConfiguration.RmHaId, rmId);
             services.AddItem(SecurityUtil.BuildTokenService(yarnConf.GetSocketAddr(address, defaultAddr
                                                                                    , defaultPort)).ToString());
         }
         return(new Text(Joiner.On(',').Join(services)));
     }
     // Non-HA case - no need to set RM_ID
     return(SecurityUtil.BuildTokenService(conf.GetSocketAddr(address, defaultAddr, defaultPort
                                                              )));
 }
示例#5
0
        private Configuration CreateHARMConf(string rmIds, string rmId, int adminPort)
        {
            Configuration conf = new YarnConfiguration();

            conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
            conf.Set(YarnConfiguration.RmHaIds, rmIds);
            conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
            conf.Set(YarnConfiguration.RmStore, typeof(ZKRMStateStore).FullName);
            conf.Set(YarnConfiguration.RmZkAddress, hostPort);
            conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs);
            conf.Set(YarnConfiguration.RmHaId, rmId);
            conf.Set(YarnConfiguration.RmWebappAddress, "localhost:0");
            foreach (string rpcAddress in YarnConfiguration.GetServiceAddressConfKeys(conf))
            {
                foreach (string id in HAUtil.GetRMHAIds(conf))
                {
                    conf.Set(HAUtil.AddSuffix(rpcAddress, id), "localhost:0");
                }
            }
            conf.Set(HAUtil.AddSuffix(YarnConfiguration.RmAdminAddress, rmId), "localhost:" +
                     adminPort);
            return(conf);
        }
示例#6
0
 /// <exception cref="System.Exception"/>
 protected override void ServiceInit(Configuration conf)
 {
     useFixedPorts = conf.GetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, YarnConfiguration
                                     .DefaultYarnMiniclusterFixedPorts);
     useRpc = conf.GetBoolean(YarnConfiguration.YarnMiniclusterUseRpc, YarnConfiguration
                              .DefaultYarnMiniclusterUseRpc);
     failoverTimeout = conf.GetInt(YarnConfiguration.RmZkTimeoutMs, YarnConfiguration.
                                   DefaultRmZkTimeoutMs);
     if (useRpc && !useFixedPorts)
     {
         throw new YarnRuntimeException("Invalid configuration!" + " Minicluster can use rpc only when configured to use fixed ports"
                                        );
     }
     conf.SetBoolean(YarnConfiguration.IsMiniYarnCluster, true);
     if (resourceManagers.Length > 1)
     {
         conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
         if (conf.Get(YarnConfiguration.RmHaIds) == null)
         {
             StringBuilder rmIds = new StringBuilder();
             for (int i = 0; i < resourceManagers.Length; i++)
             {
                 if (i != 0)
                 {
                     rmIds.Append(",");
                 }
                 rmIds.Append("rm" + i);
             }
             conf.Set(YarnConfiguration.RmHaIds, rmIds.ToString());
         }
         ICollection <string> rmIdsCollection = HAUtil.GetRMHAIds(conf);
         rmIds = Sharpen.Collections.ToArray(rmIdsCollection, new string[rmIdsCollection.Count
                                             ]);
     }
     for (int i_1 = 0; i_1 < resourceManagers.Length; i_1++)
     {
         resourceManagers[i_1] = CreateResourceManager();
         if (!useFixedPorts)
         {
             if (HAUtil.IsHAEnabled(conf))
             {
                 SetHARMConfigurationWithEphemeralPorts(i_1, conf);
             }
             else
             {
                 SetNonHARMConfigurationWithEphemeralPorts(conf);
             }
         }
         AddService(new MiniYARNCluster.ResourceManagerWrapper(this, i_1));
     }
     for (int index = 0; index < nodeManagers.Length; index++)
     {
         nodeManagers[index] = useRpc ? new MiniYARNCluster.CustomNodeManager(this) : new
                               MiniYARNCluster.ShortCircuitedNodeManager(this);
         AddService(new MiniYARNCluster.NodeManagerWrapper(this, index));
     }
     if (conf.GetBoolean(YarnConfiguration.TimelineServiceEnabled, YarnConfiguration.DefaultTimelineServiceEnabled
                         ) || enableAHS)
     {
         AddService(new MiniYARNCluster.ApplicationHistoryServerWrapper(this));
     }
     base.ServiceInit(conf is YarnConfiguration ? conf : new YarnConfiguration(conf));
 }