public virtual void TestDeleteOnExit()
{
// Create deleteOnExit entries
Path file1 = helper.GetTestRootPath(fc, "file1");
FileContextTestHelper.CreateFile(fc, file1, numBlocks, blockSize);
fc.DeleteOnExit(file1);
CheckDeleteOnExitData(1, fc, file1);
// Ensure shutdown hook is added
Assert.True(ShutdownHookManager.Get().HasShutdownHook(FileContext
.Finalizer));
Path file2 = helper.GetTestRootPath(fc, "dir1/file2");
FileContextTestHelper.CreateFile(fc, file2, numBlocks, blockSize);
fc.DeleteOnExit(file2);
CheckDeleteOnExitData(1, fc, file1, file2);
Path dir = helper.GetTestRootPath(fc, "dir3/dir4/dir5/dir6");
FileContextTestHelper.CreateFile(fc, dir, numBlocks, blockSize);
fc.DeleteOnExit(dir);
CheckDeleteOnExitData(1, fc, file1, file2, dir);
// trigger deleteOnExit and ensure the registered
// paths are cleaned up
FileContext.Finalizer.Run();
CheckDeleteOnExitData(0, fc, new Path[0]);
NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, file1));
NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, file2));
NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, dir));
}
LaunchAppHistoryServer(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
), args, Log);
Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
appHistoryServer = null;
try
{
appHistoryServer = new Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(appHistoryServer), ShutdownHookPriority);
YarnConfiguration conf = new YarnConfiguration();
new GenericOptionsParser(conf, args);
appHistoryServer.Init(conf);
appHistoryServer.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting ApplicationHistoryServer", t);
ExitUtil.Terminate(-1, "Error starting ApplicationHistoryServer");
}
return(appHistoryServer);
}
public static ScheduledExecutorService CreateLogSyncer()
{
ScheduledExecutorService scheduler = Executors.NewSingleThreadScheduledExecutor(new
_ThreadFactory_331());
ShutdownHookManager.Get().AddShutdownHook(new _Runnable_340(scheduler), 50);
scheduler.ScheduleWithFixedDelay(new _Runnable_347(), 0L, 5L, TimeUnit.Seconds);
return(scheduler);
}
StartServer(Configuration configuration)
{
Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer proxy = new Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer
();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(proxy), ShutdownHookPriority);
proxy.Init(configuration);
proxy.Start();
return(proxy);
}
internal DFSClientCache(NfsConfiguration config, int clientCache)
{
this.config = config;
this.clientCache = CacheBuilder.NewBuilder().MaximumSize(clientCache).RemovalListener
(ClientRemovalListener()).Build(ClientLoader());
this.inputstreamCache = CacheBuilder.NewBuilder().MaximumSize(DefaultDfsInputstreamCacheSize
).ExpireAfterAccess(DefaultDfsInputstreamCacheTtl, TimeUnit.Seconds).RemovalListener
(InputStreamRemovalListener()).Build(InputStreamLoader());
ShutdownHookManager.Get().AddShutdownHook(new DFSClientCache.CacheFinalizer(this)
, ShutdownHookPriority);
}
public static Org.Apache.Hadoop.Tracing.SpanReceiverHost Get(Configuration conf,
string confPrefix)
{
lock (typeof(Org.Apache.Hadoop.Tracing.SpanReceiverHost))
{
Org.Apache.Hadoop.Tracing.SpanReceiverHost host = hosts[confPrefix];
if (host != null)
{
return(host);
}
Org.Apache.Hadoop.Tracing.SpanReceiverHost newHost = new Org.Apache.Hadoop.Tracing.SpanReceiverHost
(confPrefix);
newHost.LoadSpanReceivers(conf);
ShutdownHookManager.Get().AddShutdownHook(new _Runnable_79(newHost), 0);
hosts[confPrefix] = newHost;
return(newHost);
}
}
public virtual void UncaughtException(Sharpen.Thread t, Exception e)
{
if (ShutdownHookManager.Get().IsShutdownInProgress())
{
Log.Error("Thread " + t + " threw an Throwable, but we are shutting " + "down, so ignoring this"
, e);
}
else
{
if (e is Error)
{
try
{
Log.Fatal("Thread " + t + " threw an Error. Shutting down now...", e);
}
catch
{
}
//We don't want to not exit because of an issue with logging
if (e is OutOfMemoryException)
{
//After catching an OOM java says it is undefined behavior, so don't
//even try to clean up or we can get stuck on shutdown.
try
{
System.Console.Error.WriteLine("Halting due to Out Of Memory Error...");
}
catch
{
}
//Again we done want to exit because of logging issues.
ExitUtil.Halt(-1);
}
else
{
ExitUtil.Terminate(-1);
}
}
else
{
Log.Error("Thread " + t + " threw an Exception.", e);
}
}
}
public virtual void Start(bool register)
{
StartTCPServer();
// Start TCP server
if (register)
{
ShutdownHookManager.Get().AddShutdownHook(new Nfs3Base.NfsShutdownHook(this), ShutdownHookPriority
);
try
{
rpcProgram.Register(PortmapMapping.TransportTcp, nfsBoundPort);
}
catch (Exception e)
{
Log.Fatal("Failed to register the NFSv3 service.", e);
ExitUtil.Terminate(1, e);
}
}
}
public override void Run()
{
try
{
this._enclosing.Stop();
}
catch (Exception t)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeManager.Log.Error("Error while shutting down NodeManager"
, t);
}
finally
{
if (this._enclosing.shouldExitOnShutdownEvent && !ShutdownHookManager.Get().IsShutdownInProgress
())
{
ExitUtil.Terminate(-1);
}
}
}
public virtual void Start(bool register)
{
StartUDPServer();
StartTCPServer();
if (register)
{
ShutdownHookManager.Get().AddShutdownHook(new MountdBase.Unregister(this), ShutdownHookPriority
);
try
{
rpcProgram.Register(PortmapMapping.TransportUdp, udpBoundPort);
rpcProgram.Register(PortmapMapping.TransportTcp, tcpBoundPort);
}
catch (Exception e)
{
Log.Fatal("Failed to register the MOUNT service.", e);
ExitUtil.Terminate(1, e);
}
}
}
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheManager
), args, Log);
try
{
Configuration conf = new YarnConfiguration();
Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheManager sharedCacheManager
= new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheManager();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(sharedCacheManager), ShutdownHookPriority);
sharedCacheManager.Init(conf);
sharedCacheManager.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting SharedCacheManager", t);
System.Environment.Exit(-1);
}
}
protected internal virtual void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event
)
{
//all events go thru this loop
if (Log.IsDebugEnabled())
{
Log.Debug("Dispatching the event " + @event.GetType().FullName + "." + @event.ToString
());
}
Type type = @event.GetType().GetDeclaringClass();
try
{
EventHandler handler = eventDispatchers[type];
if (handler != null)
{
handler.Handle(@event);
}
else
{
throw new Exception("No handler for registered for " + type);
}
}
catch (Exception t)
{
//TODO Maybe log the state of the queue
Log.Fatal("Error in dispatcher thread", t);
// If serviceStop is called, we should exit this thread gracefully.
if (exitOnDispatchException && (ShutdownHookManager.Get().IsShutdownInProgress())
== false && stopped == false)
{
Sharpen.Thread shutDownThread = new Sharpen.Thread(CreateShutDownThread());
shutDownThread.SetName("AsyncDispatcher ShutDown handler");
shutDownThread.Start();
}
}
}
private void InitAndStartNodeManager(Configuration conf, bool hasToReboot)
{
try
{
// Remove the old hook if we are rebooting.
if (hasToReboot && null != nodeManagerShutdownHook)
{
ShutdownHookManager.Get().RemoveShutdownHook(nodeManagerShutdownHook);
}
nodeManagerShutdownHook = new CompositeService.CompositeServiceShutdownHook(this);
ShutdownHookManager.Get().AddShutdownHook(nodeManagerShutdownHook, ShutdownHookPriority
);
// System exit should be called only when NodeManager is instantiated from
// main() funtion
this.shouldExitOnShutdownEvent = true;
this.Init(conf);
this.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting NodeManager", t);
System.Environment.Exit(-1);
}
}
internal static JobHistoryServer LaunchJobHistoryServer(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
StringUtils.StartupShutdownMessage(typeof(JobHistoryServer), args, Log);
JobHistoryServer jobHistoryServer = null;
try
{
jobHistoryServer = new JobHistoryServer();
ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
(jobHistoryServer), ShutdownHookPriority);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
new GenericOptionsParser(conf, args);
jobHistoryServer.Init(conf);
jobHistoryServer.Start();
}
catch (Exception t)
{
Log.Fatal("Error starting JobHistoryServer", t);
ExitUtil.Terminate(-1, "Error starting JobHistoryServer");
}
return(jobHistoryServer);
}
/// <exception cref="Sharpen.RuntimeException"/>
/// <exception cref="System.IO.IOException"/>
private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int
numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles
)
{
TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID);
try
{
JobConf conf = new JobConf(this._enclosing.GetConfig());
conf.Set(JobContext.TaskId, task.GetTaskID().ToString());
conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString());
conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map));
conf.SetInt(JobContext.TaskPartition, task.GetPartition());
conf.Set(JobContext.Id, task.GetJobID().ToString());
// Use the AM's local dir env to generate the intermediate step
// output files
string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment
.LocalDirs.ToString()));
conf.SetStrings(MRConfig.LocalDir, localSysDirs);
LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get
(MRConfig.LocalDir));
// mark this as an uberized subtask so it can set task counter
// (longer-term/FIXME: could redefine as job counter and send
// "JobCounterEvent" to JobImpl on [successful] completion of subtask;
// will need new Job state-machine transition and JobImpl jobCounters
// map to handle)
conf.SetBoolean("mapreduce.task.uberized", true);
// Check and handle Encrypted spill key
task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey);
YarnChild.SetEncryptedSpillKeyIfRequired(task);
// META-FIXME: do we want the extra sanity-checking (doneWithMaps,
// etc.), or just assume/hope the state machine(s) and uber-AM work
// as expected?
if (taskType == TaskType.Map)
{
if (this.doneWithMaps)
{
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task ("
+ attemptID + "), but should be finished with maps");
throw new RuntimeException();
}
MapTask map = (MapTask)task;
map.SetConf(conf);
map.Run(conf, this._enclosing.umbilical);
if (renameOutputs)
{
MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID
, map.GetMapOutputFile());
localMapFiles[classicAttemptID] = renamed;
}
this.Relocalize();
if (++this.finishedSubMaps == numMapTasks)
{
this.doneWithMaps = true;
}
}
else
{
/* TaskType.REDUCE */
if (!this.doneWithMaps)
{
// check if event-queue empty? whole idea of counting maps vs.
// checking event queue is a tad wacky...but could enforce ordering
// (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):
// doesn't send reduce event until maps all done]
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
+ attemptID + "), but not yet finished with maps");
throw new RuntimeException();
}
// a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
// set framework name to local to make task local
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
conf.Set(MRConfig.MasterAddress, "local");
// bypass shuffle
ReduceTask reduce = (ReduceTask)task;
reduce.SetLocalMapFiles(localMapFiles);
reduce.SetConf(conf);
reduce.Run(conf, this._enclosing.umbilical);
this.Relocalize();
}
}
catch (FSError e)
{
LocalContainerLauncher.Log.Fatal("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
this._enclosing.umbilical.FsError(classicAttemptID, e.Message);
}
throw new RuntimeException();
}
catch (Exception exception)
{
LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " +
StringUtils.StringifyException(exception));
try
{
if (task != null)
{
// do cleanup for the task
task.TaskCleanup(this._enclosing.umbilical);
}
}
catch (Exception e)
{
LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException
(e));
}
// Report back any failures, for diagnostic purposes
this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException
(exception));
throw new RuntimeException();
}
catch (Exception throwable)
{
LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils
.StringifyException(throwable));
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = (tCause == null) ? throwable.Message : StringUtils.StringifyException
(tCause);
this._enclosing.umbilical.FatalError(classicAttemptID, cause);
}
throw new RuntimeException();
}
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
Log.Debug("Child starting");
JobConf job = new JobConf(MRJobConfig.JobConfFile);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.Init(job);
UserGroupInformation.SetConfiguration(job);
string host = args[0];
int port = System.Convert.ToInt32(args[1]);
IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port);
TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
long jvmIdLong = long.Parse(args[3]);
JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
) == TaskType.Map, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();
Log.Info("Executing with tokens:");
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
())
{
Log.Info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
.GetJobID()).ToString());
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
(credentials);
SecurityUtil.SetTokenService(jt, address);
taskOwner.AddToken(jt);
TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
(address, job));
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try
{
int idleLoopCount = 0;
JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle)
{
long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
);
TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
myTask = umbilical.GetTask(context);
}
if (myTask.ShouldDie())
{
return;
}
task = myTask.GetTask();
YarnChild.taskid = task.GetTaskID();
// Create the job-conf and set credentials
ConfigureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
.User.ToString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.AddCredentials(credentials);
// set job classloader if configured before invoking the task
MRApps.SetJobClassLoader(job);
logSyncer = TaskLog.CreateLogSyncer();
// Create a final reference to the task for the doAs block
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
}
catch (FSError e)
{
// use job-specified working directory
// run the task
Log.Fatal("FSError from child", e);
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FsError(taskid, e.Message);
}
}
catch (Exception exception)
{
Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
);
try
{
if (task != null)
{
// do cleanup for the task
if (childUGI == null)
{
// no need to job into doAs block
task.TaskCleanup(umbilical);
}
else
{
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
}
}
}
catch (Exception e)
{
Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
}
}
}
catch (Exception throwable)
{
Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = tCause == null ? throwable.Message : StringUtils.StringifyException
(tCause);
umbilical.FatalError(taskid, cause);
}
}
}
finally
{
RPC.StopProxy(umbilical);
DefaultMetricsSystem.Shutdown();
TaskLog.SyncLogsShutdown(logSyncer);
}
}
/// <summary>Create a blook pool slice</summary>
/// <param name="bpid">Block pool Id</param>
/// <param name="volume">
///
/// <see cref="FsVolumeImpl"/>
/// to which this BlockPool belongs to
/// </param>
/// <param name="bpDir">directory corresponding to the BlockPool</param>
/// <param name="conf">configuration</param>
/// <exception cref="System.IO.IOException"/>
internal BlockPoolSlice(string bpid, FsVolumeImpl volume, FilePath bpDir, Configuration
conf)
{
// volume to which this BlockPool belongs to
// StorageDirectory/current/bpid/current
// directory where finalized replicas are stored
// directory store RBW replica
// directory store Temporary replica
// TODO:FEDERATION scalability issue - a thread per DU is needed
this.bpid = bpid;
this.volume = volume;
this.currentDir = new FilePath(bpDir, DataStorage.StorageDirCurrent);
this.finalizedDir = new FilePath(currentDir, DataStorage.StorageDirFinalized);
this.lazypersistDir = new FilePath(currentDir, DataStorage.StorageDirLazyPersist);
if (!this.finalizedDir.Exists())
{
if (!this.finalizedDir.Mkdirs())
{
throw new IOException("Failed to mkdirs " + this.finalizedDir);
}
}
this.deleteDuplicateReplicas = conf.GetBoolean(DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletion
, DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletionDefault);
// Files that were being written when the datanode was last shutdown
// are now moved back to the data directory. It is possible that
// in the future, we might want to do some sort of datanode-local
// recovery for these blocks. For example, crc validation.
//
this.tmpDir = new FilePath(bpDir, DataStorage.StorageDirTmp);
if (tmpDir.Exists())
{
FileUtil.FullyDelete(tmpDir);
}
this.rbwDir = new FilePath(currentDir, DataStorage.StorageDirRbw);
bool supportAppends = conf.GetBoolean(DFSConfigKeys.DfsSupportAppendKey, DFSConfigKeys
.DfsSupportAppendDefault);
if (rbwDir.Exists() && !supportAppends)
{
FileUtil.FullyDelete(rbwDir);
}
if (!rbwDir.Mkdirs())
{
// create rbw directory if not exist
if (!rbwDir.IsDirectory())
{
throw new IOException("Mkdirs failed to create " + rbwDir.ToString());
}
}
if (!tmpDir.Mkdirs())
{
if (!tmpDir.IsDirectory())
{
throw new IOException("Mkdirs failed to create " + tmpDir.ToString());
}
}
// Use cached value initially if available. Or the following call will
// block until the initial du command completes.
this.dfsUsage = new DU(bpDir, conf, LoadDfsUsed());
this.dfsUsage.Start();
// Make the dfs usage to be saved during shutdown.
ShutdownHookManager.Get().AddShutdownHook(new _Runnable_145(this), ShutdownHookPriority
);
}