LaunchAppHistoryServer(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
), args, Log);
Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
appHistoryServer = null;
try
{
appHistoryServer = new Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(appHistoryServer), ShutdownHookPriority);
YarnConfiguration conf = new YarnConfiguration();
new GenericOptionsParser(conf, args);
appHistoryServer.Init(conf);
appHistoryServer.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting ApplicationHistoryServer", t);
ExitUtil.Terminate(-1, "Error starting ApplicationHistoryServer");
}
return(appHistoryServer);
}
//
// The main work loop
//
public virtual void DoWork()
{
//
// Poll the Namenode (once every checkpointCheckPeriod seconds) to find the
// number of transactions in the edit log that haven't yet been checkpointed.
//
long period = checkpointConf.GetCheckPeriod();
int maxRetries = checkpointConf.GetMaxRetriesOnMergeError();
while (shouldRun)
{
try
{
Sharpen.Thread.Sleep(1000 * period);
}
catch (Exception)
{
}
// do nothing
if (!shouldRun)
{
break;
}
try
{
// We may have lost our ticket since last checkpoint, log in again, just in case
if (UserGroupInformation.IsSecurityEnabled())
{
UserGroupInformation.GetCurrentUser().CheckTGTAndReloginFromKeytab();
}
long now = Time.MonotonicNow();
if (ShouldCheckpointBasedOnCount() || now >= lastCheckpointTime + 1000 * checkpointConf
.GetPeriod())
{
DoCheckpoint();
lastCheckpointTime = now;
}
}
catch (IOException e)
{
Log.Error("Exception in doCheckpoint", e);
Sharpen.Runtime.PrintStackTrace(e);
// Prevent a huge number of edits from being created due to
// unrecoverable conditions and endless retries.
if (checkpointImage.GetMergeErrorCount() > maxRetries)
{
Log.Fatal("Merging failed " + checkpointImage.GetMergeErrorCount() + " times.");
ExitUtil.Terminate(1);
}
}
catch (Exception e)
{
Log.Fatal("Throwable Exception in doCheckpoint", e);
Sharpen.Runtime.PrintStackTrace(e);
ExitUtil.Terminate(1, e);
}
}
}
private void DoWork()
{
while (this.shouldRun)
{
try
{
// There's no point in triggering a log roll if the Standby hasn't
// read any more transactions since the last time a roll was
// triggered.
if (this._enclosing.TooLongSinceLastLoad() && this._enclosing.lastRollTriggerTxId
< this._enclosing.lastLoadedTxnId)
{
this._enclosing.TriggerActiveLogRoll();
}
if (!this.shouldRun)
{
break;
}
// Prevent reading of name system while being modified. The full
// name system lock will be acquired to further block even the block
// state updates.
this._enclosing.namesystem.CpLockInterruptibly();
try
{
this._enclosing.DoTailEdits();
}
finally
{
this._enclosing.namesystem.CpUnlock();
}
}
catch (EditLogInputException elie)
{
EditLogTailer.Log.Warn("Error while reading edits from disk. Will try again.", elie
);
}
catch (Exception)
{
// interrupter should have already set shouldRun to false
continue;
}
catch (Exception t)
{
EditLogTailer.Log.Fatal("Unknown error encountered while tailing edits. " + "Shutting down standby NN."
, t);
ExitUtil.Terminate(1, t);
}
try
{
Sharpen.Thread.Sleep(this._enclosing.sleepTimeMs);
}
catch (Exception e)
{
EditLogTailer.Log.Warn("Edit log tailer interrupted", e);
}
}
}
protected override void RemoveStoredMasterKey(DelegationKey key)
{
try
{
Log.Info("removing master key with keyID " + key.GetKeyId());
rmContext.GetStateStore().RemoveRMDTMasterKey(key);
}
catch (Exception e)
{
Log.Error("Error in removing master key with KeyID: " + key.GetKeyId());
ExitUtil.Terminate(1, e);
}
}
protected override void StoreNewMasterKey(DelegationKey newKey)
{
try
{
Log.Info("storing master key with keyID " + newKey.GetKeyId());
rmContext.GetStateStore().StoreRMDTMasterKey(newKey);
}
catch (Exception e)
{
Log.Error("Error in storing master key with KeyID: " + newKey.GetKeyId());
ExitUtil.Terminate(1, e);
}
}
/// <exception cref="System.IO.IOException"/>
protected override void RemoveStoredToken(RMDelegationTokenIdentifier ident)
{
try
{
Log.Info("removing RMDelegation token with sequence number: " + ident.GetSequenceNumber
());
rmContext.GetStateStore().RemoveRMDelegationToken(ident);
}
catch (Exception e)
{
Log.Error("Error in removing RMDelegationToken with sequence number: " + ident.GetSequenceNumber
());
ExitUtil.Terminate(1, e);
}
}
private static void PrintUsage(TextWriter err)
{
err.WriteLine("fetchdt retrieves delegation tokens from the NameNode");
err.WriteLine();
err.WriteLine("fetchdt <opts> <token file>");
err.WriteLine("Options:");
err.WriteLine(" --webservice <url> Url to contact NN on");
err.WriteLine(" --renewer <name> Name of the delegation token renewer");
err.WriteLine(" --cancel Cancel the delegation token");
err.WriteLine(" --renew Renew the delegation token. Delegation " +
"token must have been fetched using the --renewer <name> option.");
err.WriteLine(" --print Print the delegation token");
err.WriteLine();
GenericOptionsParser.PrintGenericCommandUsage(err);
ExitUtil.Terminate(1);
}
protected override void UpdateStoredToken(RMDelegationTokenIdentifier id, long renewDate
)
{
try
{
Log.Info("updating RMDelegation token with sequence number: " + id.GetSequenceNumber
());
rmContext.GetStateStore().UpdateRMDelegationToken(id, renewDate);
}
catch (Exception e)
{
Log.Error("Error in updating persisted RMDelegationToken" + " with sequence number: "
+ id.GetSequenceNumber());
ExitUtil.Terminate(1, e);
}
}
protected override void StoreNewToken(RMDelegationTokenIdentifier identifier, long
renewDate)
{
try
{
Log.Info("storing RMDelegation token with sequence number: " + identifier.GetSequenceNumber
());
rmContext.GetStateStore().StoreRMDelegationToken(identifier, renewDate);
}
catch (Exception e)
{
Log.Error("Error in storing RMDelegationToken with sequence number: " + identifier
.GetSequenceNumber());
ExitUtil.Terminate(1, e);
}
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
int res = -1;
try
{
using (Org.Apache.Hadoop.Registry.Cli.RegistryCli cli = new Org.Apache.Hadoop.Registry.Cli.RegistryCli
(System.Console.Out, System.Console.Error))
{
res = ToolRunner.Run(cli, args);
}
}
catch (Exception e)
{
ExitUtil.Terminate(res, e);
}
ExitUtil.Terminate(res);
}
/// <summary>
/// Apply the given operation across all of the journal managers, disabling
/// any for which the closure throws an IOException.
/// </summary>
/// <param name="closure">
///
/// <see cref="JournalClosure"/>
/// object encapsulating the operation.
/// </param>
/// <param name="status">message used for logging errors (e.g. "opening journal")</param>
/// <exception cref="System.IO.IOException">If the operation fails on all the journals.
/// </exception>
private void MapJournalsAndReportErrors(JournalSet.JournalClosure closure, string
status)
{
IList <JournalSet.JournalAndStream> badJAS = Lists.NewLinkedList();
foreach (JournalSet.JournalAndStream jas in journals)
{
try
{
closure.Apply(jas);
}
catch (Exception t)
{
if (jas.IsRequired())
{
string msg = "Error: " + status + " failed for required journal (" + jas + ")";
Log.Fatal(msg, t);
// If we fail on *any* of the required journals, then we must not
// continue on any of the other journals. Abort them to ensure that
// retry behavior doesn't allow them to keep going in any way.
AbortAllJournals();
// the current policy is to shutdown the NN on errors to shared edits
// dir. There are many code paths to shared edits failures - syncs,
// roll of edits etc. All of them go through this common function
// where the isRequired() check is made. Applying exit policy here
// to catch all code paths.
ExitUtil.Terminate(1, msg);
}
else
{
Log.Error("Error: " + status + " failed for (journal " + jas + ")", t);
badJAS.AddItem(jas);
}
}
}
DisableAndReportErrorOnJournals(badJAS);
if (!NameNodeResourcePolicy.AreResourcesAvailable(journals, minimumRedundantJournals
))
{
string message = status + " failed for too many journals";
Log.Error("Error: " + message);
throw new IOException(message);
}
}
public virtual void UncaughtException(Sharpen.Thread t, Exception e)
{
if (ShutdownHookManager.Get().IsShutdownInProgress())
{
Log.Error("Thread " + t + " threw an Throwable, but we are shutting " + "down, so ignoring this"
, e);
}
else
{
if (e is Error)
{
try
{
Log.Fatal("Thread " + t + " threw an Error. Shutting down now...", e);
}
catch
{
}
//We don't want to not exit because of an issue with logging
if (e is OutOfMemoryException)
{
//After catching an OOM java says it is undefined behavior, so don't
//even try to clean up or we can get stuck on shutdown.
try
{
System.Console.Error.WriteLine("Halting due to Out Of Memory Error...");
}
catch
{
}
//Again we done want to exit because of logging issues.
ExitUtil.Halt(-1);
}
else
{
ExitUtil.Terminate(-1);
}
}
else
{
Log.Error("Thread " + t + " threw an Exception.", e);
}
}
}
/// <summary>main() has some simple utility methods.</summary>
/// <param name="argv">Command line parameters.</param>
/// <exception>
/// Exception
/// if the filesystem does not exist.
/// </exception>
/// <exception cref="System.Exception"/>
public static void Main(string[] argv)
{
SecondaryNameNode.CommandLineOpts opts = Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode
.ParseArgs(argv);
if (opts == null)
{
Log.Fatal("Failed to parse options");
ExitUtil.Terminate(1);
}
else
{
if (opts.ShouldPrintHelp())
{
opts.Usage();
System.Environment.Exit(0);
}
}
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode
), argv, Log);
Configuration tconf = new HdfsConfiguration();
Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode secondary = null;
try
{
secondary = new Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode(tconf, opts
);
}
catch (IOException ioe)
{
Log.Fatal("Failed to start secondary namenode", ioe);
ExitUtil.Terminate(1);
}
if (opts != null && opts.GetCommand() != null)
{
int ret = secondary.ProcessStartupCommand(opts);
ExitUtil.Terminate(ret);
}
if (secondary != null)
{
secondary.StartCheckpointThread();
secondary.Join();
}
}
public virtual void Start(bool register)
{
StartTCPServer();
// Start TCP server
if (register)
{
ShutdownHookManager.Get().AddShutdownHook(new Nfs3Base.NfsShutdownHook(this), ShutdownHookPriority
);
try
{
rpcProgram.Register(PortmapMapping.TransportTcp, nfsBoundPort);
}
catch (Exception e)
{
Log.Fatal("Failed to register the NFSv3 service.", e);
ExitUtil.Terminate(1, e);
}
}
}
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer
), args, Log);
try
{
YarnConfiguration configuration = new YarnConfiguration();
new GenericOptionsParser(configuration, args);
Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer proxyServer = StartServer
(configuration);
proxyServer.proxy.Join();
}
catch (Exception t)
{
ExitUtil.Terminate(-1, t);
}
}
public override void Run()
{
try
{
this._enclosing.Stop();
}
catch (Exception t)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeManager.Log.Error("Error while shutting down NodeManager"
, t);
}
finally
{
if (this._enclosing.shouldExitOnShutdownEvent && !ShutdownHookManager.Get().IsShutdownInProgress
())
{
ExitUtil.Terminate(-1);
}
}
}
public virtual void Start(bool register)
{
StartUDPServer();
StartTCPServer();
if (register)
{
ShutdownHookManager.Get().AddShutdownHook(new MountdBase.Unregister(this), ShutdownHookPriority
);
try
{
rpcProgram.Register(PortmapMapping.TransportUdp, udpBoundPort);
rpcProgram.Register(PortmapMapping.TransportTcp, tcpBoundPort);
}
catch (Exception e)
{
Log.Fatal("Failed to register the MOUNT service.", e);
ExitUtil.Terminate(1, e);
}
}
}
private void StartTCPServer()
{
SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.GetPort(), rpcProgram,
0);
rpcProgram.StartDaemons();
try
{
tcpServer.Run();
}
catch (Exception e)
{
Log.Fatal("Failed to start the TCP server.", e);
if (tcpServer.GetBoundPort() > 0)
{
rpcProgram.Unregister(PortmapMapping.TransportTcp, tcpServer.GetBoundPort());
}
tcpServer.Shutdown();
ExitUtil.Terminate(1, e);
}
nfsBoundPort = tcpServer.GetBoundPort();
}
/* Start UDP server */
private void StartUDPServer()
{
SimpleUdpServer udpServer = new SimpleUdpServer(rpcProgram.GetPort(), rpcProgram,
1);
rpcProgram.StartDaemons();
try
{
udpServer.Run();
}
catch (Exception e)
{
Log.Fatal("Failed to start the UDP server.", e);
if (udpServer.GetBoundPort() > 0)
{
rpcProgram.Unregister(PortmapMapping.TransportUdp, udpServer.GetBoundPort());
}
udpServer.Shutdown();
ExitUtil.Terminate(1, e);
}
udpBoundPort = udpServer.GetBoundPort();
}
internal static JobHistoryServer LaunchJobHistoryServer(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(JobHistoryServer), args, Log);
JobHistoryServer jobHistoryServer = null;
try
{
jobHistoryServer = new JobHistoryServer();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(jobHistoryServer), ShutdownHookPriority);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
new GenericOptionsParser(conf, args);
jobHistoryServer.Init(conf);
jobHistoryServer.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting JobHistoryServer", t);
ExitUtil.Terminate(-1, "Error starting JobHistoryServer");
}
return(jobHistoryServer);
}
public override void Run()
{
long startTimeMs = 0;
Sharpen.Thread.CurrentThread().SetName("CacheReplicationMonitor(" + Runtime.IdentityHashCode
(this) + ")");
Log.Info("Starting CacheReplicationMonitor with interval " + intervalMs + " milliseconds"
);
try
{
long curTimeMs = Time.MonotonicNow();
while (true)
{
Lock.Lock();
try
{
while (true)
{
if (shutdown)
{
Log.Debug("Shutting down CacheReplicationMonitor");
return;
}
if (completedScanCount < neededScanCount)
{
Log.Debug("Rescanning because of pending operations");
break;
}
long delta = (startTimeMs + intervalMs) - curTimeMs;
if (delta <= 0)
{
Log.Debug("Rescanning after {} milliseconds", (curTimeMs - startTimeMs));
break;
}
doRescan.Await(delta, TimeUnit.Milliseconds);
curTimeMs = Time.MonotonicNow();
}
}
finally
{
Lock.Unlock();
}
startTimeMs = curTimeMs;
mark = !mark;
Rescan();
curTimeMs = Time.MonotonicNow();
// Update synchronization-related variables.
Lock.Lock();
try
{
completedScanCount = curScanCount;
curScanCount = -1;
scanFinished.SignalAll();
}
finally
{
Lock.Unlock();
}
Log.Debug("Scanned {} directive(s) and {} block(s) in {} millisecond(s).", scannedDirectives
, scannedBlocks, (curTimeMs - startTimeMs));
}
}
catch (Exception)
{
Log.Info("Shutting down CacheReplicationMonitor.");
return;
}
catch (Exception t)
{
Log.Error("Thread exiting", t);
ExitUtil.Terminate(1, t);
}
}
private void RunTask(ContainerRemoteLaunchEvent launchEv, IDictionary <TaskAttemptID
, MapOutputFile> localMapFiles)
{
TaskAttemptId attemptID = launchEv.GetTaskAttemptID();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = this._enclosing.context.GetAllJobs
()[attemptID.GetTaskId().GetJobId()];
int numMapTasks = job.GetTotalMaps();
int numReduceTasks = job.GetTotalReduces();
// YARN (tracking) Task:
Task ytask = job.GetTask(attemptID.GetTaskId());
// classic mapred Task:
Task remoteTask = launchEv.GetRemoteTask();
// after "launching," send launched event to task attempt to move
// state from ASSIGNED to RUNNING (also nukes "remoteTask", so must
// do getRemoteTask() call first)
//There is no port number because we are not really talking to a task
// tracker. The shuffle is just done through local files. So the
// port number is set to -1 in this case.
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptContainerLaunchedEvent
(attemptID, -1));
if (numMapTasks == 0)
{
this.doneWithMaps = true;
}
try
{
if (remoteTask.IsMapOrReduce())
{
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.GetTaskId().GetJobId
());
jce.AddCounterUpdate(JobCounter.TotalLaunchedUbertasks, 1);
if (remoteTask.IsMapTask())
{
jce.AddCounterUpdate(JobCounter.NumUberSubmaps, 1);
}
else
{
jce.AddCounterUpdate(JobCounter.NumUberSubreduces, 1);
}
this._enclosing.context.GetEventHandler().Handle(jce);
}
this.RunSubtask(remoteTask, ytask.GetType(), attemptID, numMapTasks, (numReduceTasks
> 0), localMapFiles);
}
catch (RuntimeException)
{
JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptID.GetTaskId().GetJobId
());
jce.AddCounterUpdate(JobCounter.NumFailedUbertasks, 1);
this._enclosing.context.GetEventHandler().Handle(jce);
// this is our signal that the subtask failed in some way, so
// simulate a failed JVM/container and send a container-completed
// event to task attempt (i.e., move state machine from RUNNING
// to FAIL_CONTAINER_CLEANUP [and ultimately to FAILED])
this._enclosing.context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID,
TaskAttemptEventType.TaContainerCompleted));
}
catch (IOException ioe)
{
// if umbilical itself barfs (in error-handler of runSubMap()),
// we're pretty much hosed, so do what YarnChild main() does
// (i.e., exit clumsily--but can never happen, so no worries!)
LocalContainerLauncher.Log.Fatal("oopsie... this can never happen: " + StringUtils
.StringifyException(ioe));
ExitUtil.Terminate(-1);
}
finally
{
// remove my future
if (Sharpen.Collections.Remove(this.futures, attemptID) != null)
{
LocalContainerLauncher.Log.Info("removed attempt " + attemptID + " from the futures to keep track of"
);
}
}
}
public static void Main(string[] args)
{
int res = -1;
// parse arguments
Options opts = new Options();
CommandLine commandLine = null;
try
{
commandLine = ParseArgs(opts, args);
}
catch (ArgumentException)
{
commandLine = null;
}
if (commandLine == null)
{
// invalid arguments
Err("Invalid args");
PrintUsage(opts);
ExitUtil.Terminate(-1);
}
Org.Apache.Hadoop.Hdfs.Tools.JMXGet jm = new Org.Apache.Hadoop.Hdfs.Tools.JMXGet(
);
if (commandLine.HasOption("port"))
{
jm.SetPort(commandLine.GetOptionValue("port"));
}
if (commandLine.HasOption("service"))
{
jm.SetService(commandLine.GetOptionValue("service"));
}
if (commandLine.HasOption("server"))
{
jm.SetServer(commandLine.GetOptionValue("server"));
}
if (commandLine.HasOption("localVM"))
{
// from the file /tmp/hsperfdata*
jm.SetLocalVMUrl(commandLine.GetOptionValue("localVM"));
}
if (commandLine.HasOption("help"))
{
PrintUsage(opts);
ExitUtil.Terminate(0);
}
// rest of args
args = commandLine.GetArgs();
try
{
jm.Init();
if (args.Length == 0)
{
jm.PrintAllValues();
}
else
{
foreach (string key in args)
{
Err("key = " + key);
string val = jm.GetValue(key);
if (val != null)
{
System.Console.Out.Format(Org.Apache.Hadoop.Hdfs.Tools.JMXGet.format, key, val);
}
}
}
res = 0;
}
catch (Exception re)
{
Sharpen.Runtime.PrintStackTrace(re);
res = -1;
}
ExitUtil.Terminate(res);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] argv)
{
int res = ToolRunner.Run(new CLI(), argv);
ExitUtil.Terminate(res);
}
/// <summary>Submit a pipes job based on the command line arguments.</summary>
/// <param name="args"/>
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
int exitCode = new Submitter().Run(args);
ExitUtil.Terminate(exitCode);
}