public _ContainerManagerImpl_438(TestNodeManager2 _enclosing, Context baseArg1, ContainerExecutor
baseArg2, DeletionService baseArg3, NodeStatusUpdater baseArg4, NodeManagerMetrics
baseArg5, ApplicationACLsManager baseArg6, LocalDirsHandlerService baseArg7)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7)
{
this._enclosing = _enclosing;
}
public DummyContainerManager(Context context, ContainerExecutor exec, DeletionService
deletionContext, NodeStatusUpdater nodeStatusUpdater, NodeManagerMetrics metrics
, ApplicationACLsManager applicationACLsManager, LocalDirsHandlerService dirsHandler
)
: base(context, exec, deletionContext, nodeStatusUpdater, metrics, applicationACLsManager
, dirsHandler)
{
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
conf.SetBoolean(Dispatcher.DispatcherExitOnErrorKey, true);
rmWorkPreservingRestartEnabled = conf.GetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled
, YarnConfiguration.DefaultRmWorkPreservingRecoveryEnabled);
InitAndStartRecoveryStore(conf);
NMContainerTokenSecretManager containerTokenSecretManager = new NMContainerTokenSecretManager
(conf, nmStore);
NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore
);
RecoverTokens(nmTokenSecretManager, containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.NewInstance(conf.GetClass <ContainerExecutor
>(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor)), conf
);
try
{
exec.Init();
}
catch (IOException e)
{
throw new YarnRuntimeException("Failed to initialize container executor", e);
}
DeletionService del = CreateDeletionService(exec);
AddService(del);
// NodeManager level dispatcher
this.dispatcher = new AsyncDispatcher();
nodeHealthChecker = new NodeHealthCheckerService();
AddService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.GetDiskHandler();
this.context = CreateNMContext(containerTokenSecretManager, nmTokenSecretManager,
nmStore);
nodeStatusUpdater = CreateNodeStatusUpdater(context, dispatcher, nodeHealthChecker
);
NodeResourceMonitor nodeResourceMonitor = CreateNodeResourceMonitor();
AddService(nodeResourceMonitor);
containerManager = CreateContainerManager(context, exec, del, nodeStatusUpdater,
this.aclsManager, dirsHandler);
AddService(containerManager);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
WebServer webServer = CreateWebServer(context, containerManager.GetContainersMonitor
(), this.aclsManager, dirsHandler);
AddService(webServer);
((NodeManager.NMContext)context).SetWebServer(webServer);
dispatcher.Register(typeof(ContainerManagerEventType), containerManager);
dispatcher.Register(typeof(NodeManagerEventType), this);
AddService(dispatcher);
DefaultMetricsSystem.Initialize("NodeManager");
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
AddService(nodeStatusUpdater);
base.ServiceInit(conf);
}
public virtual void TestRelativeDelete()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
IList <Path> baseDirs = BuildDirs(r, @base, 4);
CreateDirs(new Path("."), baseDirs);
IList <Path> content = BuildDirs(r, new Path("."), 10);
foreach (Path b in baseDirs)
{
CreateDirs(b, content);
}
DeletionService del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor
());
try
{
del.Init(new Configuration());
del.Start();
foreach (Path p in content)
{
NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(new Path(baseDirs[0], p)));
del.Delete((long.Parse(p.GetName()) % 2) == 0 ? null : "dingo", p, Sharpen.Collections.ToArray
(baseDirs, new Path[4]));
}
int msecToWait = 20 * 1000;
foreach (Path p_1 in baseDirs)
{
foreach (Path q in content)
{
Path fp = new Path(p_1, q);
while (msecToWait > 0 && lfs.Util().Exists(fp))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsFalse(lfs.Util().Exists(fp));
}
}
}
finally
{
del.Stop();
}
}
private FileDeletionTask(int taskId, DeletionService delService, string user, Path
subDir, IList <Path> baseDirs)
{
// By default all tasks will start as success=true; however if any of
// the dependent task fails then it will be marked as false in
// fileDeletionTaskFinished().
this.taskId = taskId;
this.delService = delService;
this.user = user;
this.subDir = subDir;
this.baseDirs = baseDirs;
this.successorTaskSet = new HashSet <DeletionService.FileDeletionTask>();
this.numberOfPendingPredecessorTasks = new AtomicInteger(0);
success = true;
}
public virtual void TestStopWithDelayedTasks()
{
DeletionService del = new DeletionService(Org.Mockito.Mockito.Mock <ContainerExecutor
>());
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.DebugNmDeleteDelaySec, 60);
try
{
del.Init(conf);
del.Start();
del.Delete("dingo", new Path("/does/not/exist"));
}
finally
{
del.Stop();
}
NUnit.Framework.Assert.IsTrue(del.IsTerminated());
}
public virtual void TestNoDelete()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
IList <Path> dirs = BuildDirs(r, @base, 20);
CreateDirs(new Path("."), dirs);
TestDeletionService.FakeDefaultContainerExecutor exec = new TestDeletionService.FakeDefaultContainerExecutor
();
Configuration conf = new Configuration();
conf.SetInt(YarnConfiguration.DebugNmDeleteDelaySec, -1);
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
try
{
del.Init(conf);
del.Start();
foreach (Path p in dirs)
{
del.Delete((long.Parse(p.GetName()) % 2) == 0 ? null : "dingo", p, null);
}
int msecToWait = 20 * 1000;
foreach (Path p_1 in dirs)
{
while (msecToWait > 0 && lfs.Util().Exists(p_1))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(p_1));
}
}
finally
{
del.Stop();
}
}
protected internal virtual ContainerManagerImpl CreateContainerManager(Context context
, ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater
, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler)
{
return(new ContainerManagerImpl(context, exec, del, nodeStatusUpdater, metrics, aclsManager
, dirsHandler));
}
protected internal override ContainerManagerImpl CreateContainerManager(Context context
, ContainerExecutor exec, DeletionService del, NodeStatusUpdater nodeStatusUpdater
, ApplicationACLsManager aclsManager, LocalDirsHandlerService dirsHandler)
{
return(new _ContainerManagerImpl_438(this, context, exec, del, nodeStatusUpdater,
this.metrics, aclsManager, dirsHandler));
}
private FileDeletionTask(DeletionService delService, string user, Path subDir, IList
<Path> baseDirs)
: this(InvalidTaskId, delService, user, subDir, baseDirs)
{
}
public virtual void TestRecovery()
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
IList <Path> baseDirs = BuildDirs(r, @base, 4);
CreateDirs(new Path("."), baseDirs);
IList <Path> content = BuildDirs(r, new Path("."), 10);
foreach (Path b in baseDirs)
{
CreateDirs(b, content);
}
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
conf.SetInt(YarnConfiguration.DebugNmDeleteDelaySec, 1);
NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.Init(conf);
stateStore.Start();
DeletionService del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor
(), stateStore);
try
{
del.Init(conf);
del.Start();
foreach (Path p in content)
{
NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(new Path(baseDirs[0], p)));
del.Delete((long.Parse(p.GetName()) % 2) == 0 ? null : "dingo", p, Sharpen.Collections.ToArray
(baseDirs, new Path[4]));
}
// restart the deletion service
del.Stop();
del = new DeletionService(new TestDeletionService.FakeDefaultContainerExecutor(),
stateStore);
del.Init(conf);
del.Start();
// verify paths are still eventually deleted
int msecToWait = 10 * 1000;
foreach (Path p_1 in baseDirs)
{
foreach (Path q in content)
{
Path fp = new Path(p_1, q);
while (msecToWait > 0 && lfs.Util().Exists(fp))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsFalse(lfs.Util().Exists(fp));
}
}
}
finally
{
del.Close();
stateStore.Close();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFileDeletionTaskDependency()
{
TestDeletionService.FakeDefaultContainerExecutor exec = new TestDeletionService.FakeDefaultContainerExecutor
();
Configuration conf = new Configuration();
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
del.Init(conf);
del.Start();
try
{
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
System.Console.Out.WriteLine("SEED: " + seed);
IList <Path> dirs = BuildDirs(r, @base, 2);
CreateDirs(new Path("."), dirs);
// first we will try to delete sub directories which are present. This
// should then trigger parent directory to be deleted.
IList <Path> subDirs = BuildDirs(r, dirs[0], 2);
DeletionService.FileDeletionTask dependentDeletionTask = del.CreateFileDeletionTask
(null, dirs[0], new Path[] { });
IList <DeletionService.FileDeletionTask> deletionTasks = new AList <DeletionService.FileDeletionTask
>();
foreach (Path subDir in subDirs)
{
DeletionService.FileDeletionTask deletionTask = del.CreateFileDeletionTask(null,
null, new Path[] { subDir });
deletionTask.AddFileDeletionTaskDependency(dependentDeletionTask);
deletionTasks.AddItem(deletionTask);
}
foreach (DeletionService.FileDeletionTask task in deletionTasks)
{
del.ScheduleFileDeletionTask(task);
}
int msecToWait = 20 * 1000;
while (msecToWait > 0 && (lfs.Util().Exists(dirs[0])))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsFalse(lfs.Util().Exists(dirs[0]));
// Now we will try to delete sub directories; one of the deletion task we
// will mark as failure and then parent directory should not be deleted.
subDirs = BuildDirs(r, dirs[1], 2);
subDirs.AddItem(new Path(dirs[1], "absentFile"));
dependentDeletionTask = del.CreateFileDeletionTask(null, dirs[1], new Path[] { }
);
deletionTasks = new AList <DeletionService.FileDeletionTask>();
foreach (Path subDir_1 in subDirs)
{
DeletionService.FileDeletionTask deletionTask = del.CreateFileDeletionTask(null,
null, new Path[] { subDir_1 });
deletionTask.AddFileDeletionTaskDependency(dependentDeletionTask);
deletionTasks.AddItem(deletionTask);
}
// marking one of the tasks as a failure.
deletionTasks[2].SetSuccess(false);
foreach (DeletionService.FileDeletionTask task_1 in deletionTasks)
{
del.ScheduleFileDeletionTask(task_1);
}
msecToWait = 20 * 1000;
while (msecToWait > 0 && (lfs.Util().Exists(subDirs[0]) || lfs.Util().Exists(subDirs
[1])))
{
Sharpen.Thread.Sleep(100);
msecToWait -= 100;
}
NUnit.Framework.Assert.IsTrue(lfs.Util().Exists(dirs[1]));
}
finally
{
del.Stop();
}
}
public _ResourceLocalizationService_78(Dispatcher baseArg1, ContainerExecutor baseArg2
, DeletionService baseArg3, LocalDirsHandlerService baseArg4, Context baseArg5)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5)
{
}
protected internal override ResourceLocalizationService CreateResourceLocalizationService
(ContainerExecutor exec, DeletionService deletionContext, Context context)
{
return(new _ResourceLocalizationService_78(base.dispatcher, exec, deletionContext
, base.dirsHandler, context));
}
protected internal override LogHandler CreateLogHandler(Configuration conf, Context
context, DeletionService deletionService)
{
return(new _LogHandler_173());
}
public virtual void TestSuccessfulContainerLaunch()
{
FileContext localFS = FileContext.GetLocalFSFileContext();
localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
localDir.Mkdir();
localLogDir.Mkdir();
remoteLogDir.Mkdir();
YarnConfiguration conf = new YarnConfiguration();
Context context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new
NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService());
conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
ContainerExecutor exec = new DefaultContainerExecutor();
exec.SetConf(conf);
DeletionService del = new DeletionService(exec);
Dispatcher dispatcher = new AsyncDispatcher();
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.Create();
NodeStatusUpdater nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher
, healthChecker, metrics);
// Don't start any updating thread.
DummyContainerManager containerManager = new DummyContainerManager(context, exec,
del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler);
nodeStatusUpdater.Init(conf);
((NodeManager.NMContext)context).SetContainerManager(containerManager);
nodeStatusUpdater.Start();
containerManager.Init(conf);
containerManager.Start();
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId cID = ContainerId.NewContainerId(applicationAttemptId, 0);
string user = "******";
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context.
GetNodeId(), user, context.GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Running);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cID);
StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
);
containerManager.StopContainers(stopRequest);
BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
.Complete);
containerManager.Stop();
}
