LaunchAppHistoryServer(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
), args, Log);
Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
appHistoryServer = null;
try
{
appHistoryServer = new Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(appHistoryServer), ShutdownHookPriority);
YarnConfiguration conf = new YarnConfiguration();
new GenericOptionsParser(conf, args);
appHistoryServer.Init(conf);
appHistoryServer.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting ApplicationHistoryServer", t);
ExitUtil.Terminate(-1, "Error starting ApplicationHistoryServer");
}
return(appHistoryServer);
}
// ATTENTION: Use ConsoleWriter instead of Console.WriteLine to use Colors
/// <summary>
/// Starts the backup program initiating all required actions like profile choosing, parsing and running
/// the backup.
/// </summary>
/// <param name="args">unused command line arguments</param>
static void Main(string[] args)
{
// change to English for testing
//Thread.CurrentThread.CurrentUICulture = CultureInfo.GetCultureInfo("en");
try
{
// init profile selector (might throw BackupException if "../backup_profiles" does not exist)
BackupProfileSelector profileSelector = new BackupProfileSelector();
profileSelector.InitProfilePaths();
// create necessary instances
BackupProfileConverter profileConverter = new BackupProfileConverter();
ExcludeUtil excludeUtil = new ExcludeUtil();
// execute main loop (might forward BackupException if the backup cannot be processed)
Start starter = new Start();
starter.DoMainLoop(profileSelector, profileConverter, excludeUtil);
}
catch (BackupException e)
{
// all BackupExceptions are forwarded here to provide a single exit point when an error occurs
ExitUtil.ExitAfterError(e);
}
// backup was successful or regularly cancelled by user, so exit the application in the default way
ExitUtil.ExitWithoutError();
}
/// <summary>
/// Exits the client, terminating the game process.
/// </summary>
public void Exit()
{
this.LogTrace("ExitClient()");
if (_clientProvider.Get().GetEditorModeEnabled())
{
ExitUtil.ExitEditor();
return;
}
// Fire onExit event
if (_onExit != null)
{
_onExit();
}
if (((_clientProvider.Get().GetPlatform() == Platform.iOS) && _configProvider.Get().GetClientCleanExitIosEnabled()) ||
((_clientProvider.Get().GetPlatform() == Platform.Android) && _configProvider.Get().GetClientCleanExitAndroidEnabled()))
{
// If clean exit is enabled, allow Unity to cleanly shut down.
ExitUtil.CleanExit();
return;
}
// On fall-through, we hard exit by killing the process
ExitUtil.HardExit();
}
//test launch method with -D arguments
/// <exception cref="System.Exception"/>
public virtual void TestLaunchWithArguments()
{
ExitUtil.DisableSystemExit();
ApplicationHistoryServer historyServer = null;
try
{
// Not able to modify the config of this test case,
// but others have been customized to avoid conflicts
string[] args = new string[2];
args[0] = "-D" + YarnConfiguration.TimelineServiceLeveldbTtlIntervalMs + "=4000";
args[1] = "-D" + YarnConfiguration.TimelineServiceTtlMs + "=200";
historyServer = ApplicationHistoryServer.LaunchAppHistoryServer(args);
Configuration conf = historyServer.GetConfig();
NUnit.Framework.Assert.AreEqual("4000", conf.Get(YarnConfiguration.TimelineServiceLeveldbTtlIntervalMs
));
NUnit.Framework.Assert.AreEqual("200", conf.Get(YarnConfiguration.TimelineServiceTtlMs
));
}
catch (ExitUtil.ExitException e)
{
NUnit.Framework.Assert.AreEqual(0, e.status);
ExitUtil.ResetFirstExitException();
NUnit.Framework.Assert.Fail();
}
finally
{
if (historyServer != null)
{
historyServer.Stop();
}
}
}
// test launch method
/// <exception cref="System.Exception"/>
public virtual void TestLaunch()
{
ExitUtil.DisableSystemExit();
ApplicationHistoryServer historyServer = null;
try
{
// Not able to modify the config of this test case,
// but others have been customized to avoid conflicts
historyServer = ApplicationHistoryServer.LaunchAppHistoryServer(new string[0]);
}
catch (ExitUtil.ExitException e)
{
NUnit.Framework.Assert.AreEqual(0, e.status);
ExitUtil.ResetFirstExitException();
NUnit.Framework.Assert.Fail();
}
finally
{
if (historyServer != null)
{
historyServer.Stop();
}
}
}
//
// The main work loop
//
public virtual void DoWork()
{
//
// Poll the Namenode (once every checkpointCheckPeriod seconds) to find the
// number of transactions in the edit log that haven't yet been checkpointed.
//
long period = checkpointConf.GetCheckPeriod();
int maxRetries = checkpointConf.GetMaxRetriesOnMergeError();
while (shouldRun)
{
try
{
Sharpen.Thread.Sleep(1000 * period);
}
catch (Exception)
{
}
// do nothing
if (!shouldRun)
{
break;
}
try
{
// We may have lost our ticket since last checkpoint, log in again, just in case
if (UserGroupInformation.IsSecurityEnabled())
{
UserGroupInformation.GetCurrentUser().CheckTGTAndReloginFromKeytab();
}
long now = Time.MonotonicNow();
if (ShouldCheckpointBasedOnCount() || now >= lastCheckpointTime + 1000 * checkpointConf
.GetPeriod())
{
DoCheckpoint();
lastCheckpointTime = now;
}
}
catch (IOException e)
{
Log.Error("Exception in doCheckpoint", e);
Sharpen.Runtime.PrintStackTrace(e);
// Prevent a huge number of edits from being created due to
// unrecoverable conditions and endless retries.
if (checkpointImage.GetMergeErrorCount() > maxRetries)
{
Log.Fatal("Merging failed " + checkpointImage.GetMergeErrorCount() + " times.");
ExitUtil.Terminate(1);
}
}
catch (Exception e)
{
Log.Fatal("Throwable Exception in doCheckpoint", e);
Sharpen.Runtime.PrintStackTrace(e);
ExitUtil.Terminate(1, e);
}
}
}
private void DoWork()
{
while (this.shouldRun)
{
try
{
// There's no point in triggering a log roll if the Standby hasn't
// read any more transactions since the last time a roll was
// triggered.
if (this._enclosing.TooLongSinceLastLoad() && this._enclosing.lastRollTriggerTxId
< this._enclosing.lastLoadedTxnId)
{
this._enclosing.TriggerActiveLogRoll();
}
if (!this.shouldRun)
{
break;
}
// Prevent reading of name system while being modified. The full
// name system lock will be acquired to further block even the block
// state updates.
this._enclosing.namesystem.CpLockInterruptibly();
try
{
this._enclosing.DoTailEdits();
}
finally
{
this._enclosing.namesystem.CpUnlock();
}
}
catch (EditLogInputException elie)
{
EditLogTailer.Log.Warn("Error while reading edits from disk. Will try again.", elie
);
}
catch (Exception)
{
// interrupter should have already set shouldRun to false
continue;
}
catch (Exception t)
{
EditLogTailer.Log.Fatal("Unknown error encountered while tailing edits. " + "Shutting down standby NN."
, t);
ExitUtil.Terminate(1, t);
}
try
{
Sharpen.Thread.Sleep(this._enclosing.sleepTimeMs);
}
catch (Exception e)
{
EditLogTailer.Log.Warn("Edit log tailer interrupted", e);
}
}
}
public virtual void Setup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
ExitUtil.DisableSystemExit();
conf = new YarnConfiguration();
UserGroupInformation.SetConfiguration(conf);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
conf.Set(YarnConfiguration.RmScheduler, typeof(FairScheduler).FullName);
}
protected override void RemoveStoredMasterKey(DelegationKey key)
{
try
{
Log.Info("removing master key with keyID " + key.GetKeyId());
rmContext.GetStateStore().RemoveRMDTMasterKey(key);
}
catch (Exception e)
{
Log.Error("Error in removing master key with KeyID: " + key.GetKeyId());
ExitUtil.Terminate(1, e);
}
}
protected override void StoreNewMasterKey(DelegationKey newKey)
{
try
{
Log.Info("storing master key with keyID " + newKey.GetKeyId());
rmContext.GetStateStore().StoreRMDTMasterKey(newKey);
}
catch (Exception e)
{
Log.Error("Error in storing master key with KeyID: " + newKey.GetKeyId());
ExitUtil.Terminate(1, e);
}
}
public virtual void TestUncaughtExceptionHandlerWithError()
{
ExitUtil.DisableSystemExit();
YarnUncaughtExceptionHandler spyErrorHandler = Org.Mockito.Mockito.Spy(exHandler);
Error error = new Error("test-error");
Sharpen.Thread errorThread = new Sharpen.Thread(new _Runnable_75(error));
errorThread.SetUncaughtExceptionHandler(spyErrorHandler);
NUnit.Framework.Assert.AreSame(spyErrorHandler, errorThread.GetUncaughtExceptionHandler
());
errorThread.Start();
errorThread.Join();
Org.Mockito.Mockito.Verify(spyErrorHandler).UncaughtException(errorThread, error);
}
// test launch method
/// <exception cref="System.Exception"/>
public virtual void TestLaunch()
{
ExitUtil.DisableSystemExit();
try
{
historyServer = JobHistoryServer.LaunchJobHistoryServer(new string[0]);
}
catch (ExitUtil.ExitException e)
{
NUnit.Framework.Assert.AreEqual(0, e.status);
ExitUtil.ResetFirstExitException();
NUnit.Framework.Assert.Fail();
}
}
public virtual void TestUncaughtExceptionHandlerWithOutOfMemoryError()
{
ExitUtil.DisableSystemHalt();
YarnUncaughtExceptionHandler spyOomHandler = Org.Mockito.Mockito.Spy(exHandler);
OutOfMemoryException oomError = new OutOfMemoryException("out-of-memory-error");
Sharpen.Thread oomThread = new Sharpen.Thread(new _Runnable_104(oomError));
oomThread.SetUncaughtExceptionHandler(spyOomHandler);
NUnit.Framework.Assert.AreSame(spyOomHandler, oomThread.GetUncaughtExceptionHandler
());
oomThread.Start();
oomThread.Join();
Org.Mockito.Mockito.Verify(spyOomHandler).UncaughtException(oomThread, oomError);
}
/// <exception cref="System.IO.IOException"/>
protected override void RemoveStoredToken(RMDelegationTokenIdentifier ident)
{
try
{
Log.Info("removing RMDelegation token with sequence number: " + ident.GetSequenceNumber
());
rmContext.GetStateStore().RemoveRMDelegationToken(ident);
}
catch (Exception e)
{
Log.Error("Error in removing RMDelegationToken with sequence number: " + ident.GetSequenceNumber
());
ExitUtil.Terminate(1, e);
}
}
protected override void UpdateStoredToken(RMDelegationTokenIdentifier id, long renewDate
)
{
try
{
Log.Info("updating RMDelegation token with sequence number: " + id.GetSequenceNumber
());
rmContext.GetStateStore().UpdateRMDelegationToken(id, renewDate);
}
catch (Exception e)
{
Log.Error("Error in updating persisted RMDelegationToken" + " with sequence number: "
+ id.GetSequenceNumber());
ExitUtil.Terminate(1, e);
}
}
private static void PrintUsage(TextWriter err)
{
err.WriteLine("fetchdt retrieves delegation tokens from the NameNode");
err.WriteLine();
err.WriteLine("fetchdt <opts> <token file>");
err.WriteLine("Options:");
err.WriteLine(" --webservice <url> Url to contact NN on");
err.WriteLine(" --renewer <name> Name of the delegation token renewer");
err.WriteLine(" --cancel Cancel the delegation token");
err.WriteLine(" --renew Renew the delegation token. Delegation " +
"token must have been fetched using the --renewer <name> option.");
err.WriteLine(" --print Print the delegation token");
err.WriteLine();
GenericOptionsParser.PrintGenericCommandUsage(err);
ExitUtil.Terminate(1);
}
protected override void StoreNewToken(RMDelegationTokenIdentifier identifier, long
renewDate)
{
try
{
Log.Info("storing RMDelegation token with sequence number: " + identifier.GetSequenceNumber
());
rmContext.GetStateStore().StoreRMDelegationToken(identifier, renewDate);
}
catch (Exception e)
{
Log.Error("Error in storing RMDelegationToken with sequence number: " + identifier
.GetSequenceNumber());
ExitUtil.Terminate(1, e);
}
}
private static void ExpectJMXGetExit(string[] args)
{
try
{
JMXGet.Main(args);
NUnit.Framework.Assert.Fail("should call exit");
}
catch (ExitUtil.ExitException)
{
ExitUtil.ResetFirstExitException();
}
catch (Exception ex)
{
NUnit.Framework.Assert.Fail("expectJMXGetExit ex error " + ex);
}
}
public virtual void SetUp()
{
ExitUtil.DisableSystemExit();
string baseDir = PathUtils.GetTestDirName(GetType());
hdfsDir = new FilePath(baseDir, "dfs/name");
if (hdfsDir.Exists() && !FileUtil.FullyDelete(hdfsDir))
{
throw new IOException("Could not delete test directory '" + hdfsDir + "'");
}
Log.Info("hdfsdir is " + hdfsDir.GetAbsolutePath());
// as some tests might change these values we reset them to defaults before
// every test
HdfsServerConstants.StartupOption.Format.SetForceFormat(false);
HdfsServerConstants.StartupOption.Format.SetInteractiveFormat(true);
config = new Configuration();
config.Set(DFSConfigKeys.DfsNamenodeNameDirKey, hdfsDir.GetPath());
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
int res = -1;
try
{
using (Org.Apache.Hadoop.Registry.Cli.RegistryCli cli = new Org.Apache.Hadoop.Registry.Cli.RegistryCli
(System.Console.Out, System.Console.Error))
{
res = ToolRunner.Run(cli, args);
}
}
catch (Exception e)
{
ExitUtil.Terminate(res, e);
}
ExitUtil.Terminate(res);
}
/// <summary>
/// Apply the given operation across all of the journal managers, disabling
/// any for which the closure throws an IOException.
/// </summary>
/// <param name="closure">
///
/// <see cref="JournalClosure"/>
/// object encapsulating the operation.
/// </param>
/// <param name="status">message used for logging errors (e.g. "opening journal")</param>
/// <exception cref="System.IO.IOException">If the operation fails on all the journals.
/// </exception>
private void MapJournalsAndReportErrors(JournalSet.JournalClosure closure, string
status)
{
IList <JournalSet.JournalAndStream> badJAS = Lists.NewLinkedList();
foreach (JournalSet.JournalAndStream jas in journals)
{
try
{
closure.Apply(jas);
}
catch (Exception t)
{
if (jas.IsRequired())
{
string msg = "Error: " + status + " failed for required journal (" + jas + ")";
Log.Fatal(msg, t);
// If we fail on *any* of the required journals, then we must not
// continue on any of the other journals. Abort them to ensure that
// retry behavior doesn't allow them to keep going in any way.
AbortAllJournals();
// the current policy is to shutdown the NN on errors to shared edits
// dir. There are many code paths to shared edits failures - syncs,
// roll of edits etc. All of them go through this common function
// where the isRequired() check is made. Applying exit policy here
// to catch all code paths.
ExitUtil.Terminate(1, msg);
}
else
{
Log.Error("Error: " + status + " failed for (journal " + jas + ")", t);
badJAS.AddItem(jas);
}
}
}
DisableAndReportErrorOnJournals(badJAS);
if (!NameNodeResourcePolicy.AreResourcesAvailable(journals, minimumRedundantJournals
))
{
string message = status + " failed for too many journals";
Log.Error("Error: " + message);
throw new IOException(message);
}
}
public virtual void UncaughtException(Sharpen.Thread t, Exception e)
{
if (ShutdownHookManager.Get().IsShutdownInProgress())
{
Log.Error("Thread " + t + " threw an Throwable, but we are shutting " + "down, so ignoring this"
, e);
}
else
{
if (e is Error)
{
try
{
Log.Fatal("Thread " + t + " threw an Error. Shutting down now...", e);
}
catch
{
}
//We don't want to not exit because of an issue with logging
if (e is OutOfMemoryException)
{
//After catching an OOM java says it is undefined behavior, so don't
//even try to clean up or we can get stuck on shutdown.
try
{
System.Console.Error.WriteLine("Halting due to Out Of Memory Error...");
}
catch
{
}
//Again we done want to exit because of logging issues.
ExitUtil.Halt(-1);
}
else
{
ExitUtil.Terminate(-1);
}
}
else
{
Log.Error("Thread " + t + " threw an Exception.", e);
}
}
}
/// <summary>main() has some simple utility methods.</summary>
/// <param name="argv">Command line parameters.</param>
/// <exception>
/// Exception
/// if the filesystem does not exist.
/// </exception>
/// <exception cref="System.Exception"/>
public static void Main(string[] argv)
{
SecondaryNameNode.CommandLineOpts opts = Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode
.ParseArgs(argv);
if (opts == null)
{
Log.Fatal("Failed to parse options");
ExitUtil.Terminate(1);
}
else
{
if (opts.ShouldPrintHelp())
{
opts.Usage();
System.Environment.Exit(0);
}
}
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode
), argv, Log);
Configuration tconf = new HdfsConfiguration();
Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode secondary = null;
try
{
secondary = new Org.Apache.Hadoop.Hdfs.Server.Namenode.SecondaryNameNode(tconf, opts
);
}
catch (IOException ioe)
{
Log.Fatal("Failed to start secondary namenode", ioe);
ExitUtil.Terminate(1);
}
if (opts != null && opts.GetCommand() != null)
{
int ret = secondary.ProcessStartupCommand(opts);
ExitUtil.Terminate(ret);
}
if (secondary != null)
{
secondary.StartCheckpointThread();
secondary.Join();
}
}
public virtual void Start(bool register)
{
StartTCPServer();
// Start TCP server
if (register)
{
ShutdownHookManager.Get().AddShutdownHook(new Nfs3Base.NfsShutdownHook(this), ShutdownHookPriority
);
try
{
rpcProgram.Register(PortmapMapping.TransportTcp, nfsBoundPort);
}
catch (Exception e)
{
Log.Fatal("Failed to register the NFSv3 service.", e);
ExitUtil.Terminate(1, e);
}
}
}
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer
), args, Log);
try
{
YarnConfiguration configuration = new YarnConfiguration();
new GenericOptionsParser(configuration, args);
Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer proxyServer = StartServer
(configuration);
proxyServer.proxy.Join();
}
catch (Exception t)
{
ExitUtil.Terminate(-1, t);
}
}
public virtual void Start(bool register)
{
StartUDPServer();
StartTCPServer();
if (register)
{
ShutdownHookManager.Get().AddShutdownHook(new MountdBase.Unregister(this), ShutdownHookPriority
);
try
{
rpcProgram.Register(PortmapMapping.TransportUdp, udpBoundPort);
rpcProgram.Register(PortmapMapping.TransportTcp, tcpBoundPort);
}
catch (Exception e)
{
Log.Fatal("Failed to register the MOUNT service.", e);
ExitUtil.Terminate(1, e);
}
}
}
public override void Run()
{
try
{
this._enclosing.Stop();
}
catch (Exception t)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeManager.Log.Error("Error while shutting down NodeManager"
, t);
}
finally
{
if (this._enclosing.shouldExitOnShutdownEvent && !ShutdownHookManager.Get().IsShutdownInProgress
())
{
ExitUtil.Terminate(-1);
}
}
}
/// <summary>test start form console command without options</summary>
private void StartStop()
{
ByteArrayOutputStream data = new ByteArrayOutputStream();
TextWriter error = System.Console.Error;
Runtime.SetErr(new TextWriter(data));
ExitUtil.DisableSystemExit();
try
{
CLI.Main(new string[0]);
Fail(" CLI.main should call System.exit");
}
catch (ExitUtil.ExitException e)
{
ExitUtil.ResetFirstExitException();
NUnit.Framework.Assert.AreEqual(-1, e.status);
}
catch (Exception)
{
}
finally
{
Runtime.SetErr(error);
}
// in console should be written help text
string s = Sharpen.Runtime.GetStringForBytes(data.ToByteArray());
NUnit.Framework.Assert.IsTrue(s.Contains("-submit"));
NUnit.Framework.Assert.IsTrue(s.Contains("-status"));
NUnit.Framework.Assert.IsTrue(s.Contains("-kill"));
NUnit.Framework.Assert.IsTrue(s.Contains("-set-priority"));
NUnit.Framework.Assert.IsTrue(s.Contains("-events"));
NUnit.Framework.Assert.IsTrue(s.Contains("-history"));
NUnit.Framework.Assert.IsTrue(s.Contains("-list"));
NUnit.Framework.Assert.IsTrue(s.Contains("-list-active-trackers"));
NUnit.Framework.Assert.IsTrue(s.Contains("-list-blacklisted-trackers"));
NUnit.Framework.Assert.IsTrue(s.Contains("-list-attempt-ids"));
NUnit.Framework.Assert.IsTrue(s.Contains("-kill-task"));
NUnit.Framework.Assert.IsTrue(s.Contains("-fail-task"));
NUnit.Framework.Assert.IsTrue(s.Contains("-logs"));
}
private void StartTCPServer()
{
SimpleTcpServer tcpServer = new SimpleTcpServer(rpcProgram.GetPort(), rpcProgram,
0);
rpcProgram.StartDaemons();
try
{
tcpServer.Run();
}
catch (Exception e)
{
Log.Fatal("Failed to start the TCP server.", e);
if (tcpServer.GetBoundPort() > 0)
{
rpcProgram.Unregister(PortmapMapping.TransportTcp, tcpServer.GetBoundPort());
}
tcpServer.Shutdown();
ExitUtil.Terminate(1, e);
}
nfsBoundPort = tcpServer.GetBoundPort();
}
/* Start UDP server */
private void StartUDPServer()
{
SimpleUdpServer udpServer = new SimpleUdpServer(rpcProgram.GetPort(), rpcProgram,
1);
rpcProgram.StartDaemons();
try
{
udpServer.Run();
}
catch (Exception e)
{
Log.Fatal("Failed to start the UDP server.", e);
if (udpServer.GetBoundPort() > 0)
{
rpcProgram.Unregister(PortmapMapping.TransportUdp, udpServer.GetBoundPort());
}
udpServer.Shutdown();
ExitUtil.Terminate(1, e);
}
udpBoundPort = udpServer.GetBoundPort();
}
