public virtual void TestNMWebApp()
{
Context nmContext = new NodeManager.NMContext(null, null, null, null, null);
ResourceView resourceView = new _ResourceView_147();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
(conf), dirsHandler);
server.Init(conf);
server.Start();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
nmContext.GetApplications()[appId] = app;
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 1);
NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
NMStateStoreService stateStore = new NMNullStateStoreService();
foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
{
// TODO: Use builder utils
ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
long currentTime = Runtime.CurrentTimeMillis();
Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
, user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
("password"), currentTime);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
, BuilderUtils.NewContainerTokenIdentifier(containerToken));
nmContext.GetContainers()[containerId] = container;
//TODO: Gross hack. Fix in code.
ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
();
nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
WriteContainerLogs(nmContext, containerId, dirsHandler);
}
}
// Time interval for the move thread.
//History job cleaner interval
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
Log.Info("JobHistory Init");
this.conf = conf;
this.appID = ApplicationId.NewInstance(0, 0);
this.appAttemptID = RecordFactoryProvider.GetRecordFactory(conf).NewRecordInstance
<ApplicationAttemptId>();
moveThreadInterval = conf.GetLong(JHAdminConfig.MrHistoryMoveIntervalMs, JHAdminConfig
.DefaultMrHistoryMoveIntervalMs);
hsManager = CreateHistoryFileManager();
hsManager.Init(conf);
try
{
hsManager.InitExisting();
}
catch (IOException e)
{
throw new YarnRuntimeException("Failed to intialize existing directories", e);
}
storage = CreateHistoryStorage();
if (storage is Org.Apache.Hadoop.Service.Service)
{
((Org.Apache.Hadoop.Service.Service)storage).Init(conf);
}
storage.SetHistoryFileManager(hsManager);
base.ServiceInit(conf);
}
public static Priority Create(int prio)
{
Priority priority = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<Priority>();
priority.SetPriority(prio);
return(priority);
}
public virtual void TestJobIDtoString()
{
JobId jid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId>
();
jid.SetAppId(ApplicationId.NewInstance(0, 0));
NUnit.Framework.Assert.AreEqual("job_0_0000", MRApps.ToString(jid));
}
protected override ResourceTracker GetRMClient()
{
ResourceTrackerService rt = this._enclosing._enclosing.GetResourceManager().GetResourceTrackerService
();
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
// For in-process communication without RPC
return(new _ResourceTracker_610(rt));
}
public PartialJob(JobIndexInfo jobIndexInfo, JobId jobId)
{
this.jobIndexInfo = jobIndexInfo;
this.jobId = jobId;
jobReport = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobReport
>();
jobReport.SetSubmitTime(jobIndexInfo.GetSubmitTime());
jobReport.SetStartTime(jobIndexInfo.GetJobStartTime());
jobReport.SetFinishTime(jobIndexInfo.GetFinishTime());
jobReport.SetJobState(GetState());
}
public MockApp(string user, long clusterTimeStamp, int uniqId)
: base()
{
this.user = user;
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(new Configuration
());
this.appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp, uniqId
);
appState = ApplicationState.New;
}
//TODO_get.set
public virtual void TestTaskAttemptIDtoString()
{
TaskAttemptId taid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<TaskAttemptId>();
taid.SetTaskId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <TaskId
>());
taid.GetTaskId().SetTaskType(TaskType.Map);
taid.GetTaskId().SetJobId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<JobId>());
taid.GetTaskId().GetJobId().SetAppId(ApplicationId.NewInstance(0, 0));
NUnit.Framework.Assert.AreEqual("attempt_0_0000_m_000000_0", MRApps.ToString(taid
));
}
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalResource GetYarnResource(Path p, long size, long timestamp,
LocalResourceType type, LocalResourceVisibility state, string pattern)
{
LocalResource ret = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromURI(p.ToUri()));
ret.SetSize(size);
ret.SetTimestamp(timestamp);
ret.SetType(type);
ret.SetVisibility(state);
ret.SetPattern(pattern);
return(ret);
}
/// <exception cref="System.IO.IOException"/>
public MockContainer(ApplicationAttemptId appAttemptId, Dispatcher dispatcher, Configuration
conf, string user, ApplicationId appId, int uniqId)
{
this.user = user;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
this.id = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId, uniqId);
this.launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext>();
long currentTime = Runtime.CurrentTimeMillis();
this.containerTokenIdentifier = BuilderUtils.NewContainerTokenIdentifier(BuilderUtils
.NewContainerToken(id, "127.0.0.1", 1234, user, BuilderUtils.NewResource(1024, 1
), currentTime + 10000, 123, Sharpen.Runtime.GetBytesForString("password"), currentTime
));
this.state = ContainerState.New;
}
public static NodeStatus CreateNodeStatus(NodeId nodeId, IList <ContainerStatus> containers
)
{
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();
nodeStatus.SetNodeId(nodeId);
nodeStatus.SetContainersStatuses(containers);
NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
>();
nodeHealthStatus.SetIsNodeHealthy(true);
nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
return(nodeStatus);
}
private static TaskAttemptCompletionEvent CreateTce(int eventId, bool isMap, TaskAttemptCompletionEventStatus
status)
{
JobId jid = MRBuilderUtils.NewJobId(12345, 1, 1);
TaskId tid = MRBuilderUtils.NewTaskId(jid, 0, isMap ? TaskType.Map : TaskType.Reduce
);
TaskAttemptId attemptId = MRBuilderUtils.NewTaskAttemptId(tid, 0);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
TaskAttemptCompletionEvent tce = recordFactory.NewRecordInstance <TaskAttemptCompletionEvent
>();
tce.SetEventId(eventId);
tce.SetAttemptId(attemptId);
tce.SetStatus(status);
return(tce);
}
/// <param name="resource">the local resource that contains the original remote path</param>
/// <param name="localPath">
/// the path in the local filesystem where the resource is
/// localized
/// </param>
/// <param name="fs">the filesystem of the shared cache</param>
/// <param name="localFs">the local filesystem</param>
public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration
conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs)
{
this.resource = resource;
this.localPath = localPath;
this.user = user;
this.conf = conf;
this.scmClient = scmClient;
this.fs = fs;
this.sharedCacheRootDir = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration
.DefaultSharedCacheRoot);
this.nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
this.checksum = SharedCacheChecksumFactory.GetChecksum(conf);
this.localFs = localFs;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(null);
}
public static URL GetYarnUrlFromURI(URI uri)
{
URL url = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <URL>();
if (uri.GetHost() != null)
{
url.SetHost(uri.GetHost());
}
if (uri.GetUserInfo() != null)
{
url.SetUserInfo(uri.GetUserInfo());
}
url.SetPort(uri.GetPort());
url.SetScheme(uri.GetScheme());
url.SetFile(uri.GetPath());
return(url);
}
//TODO_get.set
public virtual void TestTaskIDtoString()
{
TaskId tid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <TaskId
>();
tid.SetJobId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId
>());
tid.GetJobId().SetAppId(ApplicationId.NewInstance(0, 0));
tid.SetTaskType(TaskType.Map);
TaskType type = tid.GetTaskType();
System.Console.Error.WriteLine(type);
type = TaskType.Reduce;
System.Console.Error.WriteLine(type);
System.Console.Error.WriteLine(tid.GetTaskType());
NUnit.Framework.Assert.AreEqual("task_0_0000_m_000000", MRApps.ToString(tid));
tid.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("task_0_0000_r_000000", MRApps.ToString(tid));
}
public virtual void SetUp()
{
long now = Runtime.CurrentTimeMillis();
rmContext = MockRMContext(1, now - 10);
ResourceScheduler scheduler = MockResourceScheduler();
Configuration conf = new Configuration();
ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
scheduler);
appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM
(), scheduler, masterService, new ApplicationACLsManager(conf), conf);
appId = MockApps.NewAppID(1);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>();
asContext.SetApplicationId(appId);
asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory));
asContext.SetResource(MockResource());
SetupDispatcher(rmContext, conf);
}
static TypeConverter()
{
recordFactory = RecordFactoryProvider.GetRecordFactory(null);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] argv)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
// usage: $0 user appId locId host port app_log_dir user_dir [user_dir]*
// let $x = $x/usercache for $local.dir
// MKDIR $x/$user/appcache/$appid
// MKDIR $x/$user/appcache/$appid/output
// MKDIR $x/$user/appcache/$appid/filecache
// LOAD $x/$user/appcache/$appid/appTokens
try
{
string user = argv[0];
string appId = argv[1];
string locId = argv[2];
IPEndPoint nmAddr = new IPEndPoint(argv[3], System.Convert.ToInt32(argv[4]));
string[] sLocaldirs = Arrays.CopyOfRange(argv, 5, argv.Length);
AList <Path> localDirs = new AList <Path>(sLocaldirs.Length);
foreach (string sLocaldir in sLocaldirs)
{
localDirs.AddItem(new Path(sLocaldir));
}
string uid = UserGroupInformation.GetCurrentUser().GetShortUserName();
if (!user.Equals(uid))
{
// TODO: fail localization
Log.Warn("Localization running as " + uid + " not " + user);
}
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Localizer.ContainerLocalizer
localizer = new Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Localizer.ContainerLocalizer
(FileContext.GetLocalFSFileContext(), user, appId, locId, localDirs, RecordFactoryProvider
.GetRecordFactory(null));
int nRet = localizer.RunLocalization(nmAddr);
if (Log.IsDebugEnabled())
{
Log.Debug(string.Format("nRet: %d", nRet));
}
System.Environment.Exit(nRet);
}
catch (Exception e)
{
// Print error to stdout so that LCE can use it.
Sharpen.Runtime.PrintStackTrace(e, System.Console.Out);
Log.Error("Exception in main:", e);
throw;
}
}
public virtual void TestHeadroom()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25);
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(100 * 16 * Gb);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils
.spyHook);
// Manipulate queue 'a'
LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]);
string host_0 = "host_0";
string rack_0 = "rack_0";
FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb);
string user_0 = "user_0";
string user_1 = "user_1";
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
RMContext rmContext = TestUtils.GetMockRMContext();
RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource
(0, 0);
Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource
);
Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest
);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Priority priority_1 = TestUtils.CreateMockPriority(1);
// Submit first application with some resource-requests from user_0,
// and check headroom
ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0,
0);
FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_0, user_0);
IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>();
app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_0.UpdateResourceRequests(app_0_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource
(10 * 16 * Gb, 1);
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
// Submit second application from user_0, check headroom
ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1,
0);
FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_1, user_0);
IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>();
app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_1.UpdateResourceRequests(app_0_1_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
// no change
// Submit first application from user_1, check for new headroom
ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2,
0);
FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_1_0, user_1);
IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>();
app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_1_0.UpdateResourceRequests(app_1_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.CreateResource(90 * 16 * Gb);
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void TestContainerLogDirs()
{
FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
).GetAbsoluteFile();
string logdirwithFile = absLogDir.ToURI().ToString();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
, new ApplicationACLsManager(conf), new NMNullStateStoreService());
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
string user = "******";
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
nmContext.GetApplications()[appId] = app;
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> files = null;
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// After container is completed, it is removed from nmContext
Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
// Create a new context to check if correct container log dirs are fetched
// on full disk.
LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
);
// good log dirs are empty and nm log dir is in the full log dir list.
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
<string>());
Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
.AsList(new string[] { absLogDir.GetAbsolutePath() }));
nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
(conf), new NMNullStateStoreService());
nmContext.GetApplications()[appId] = app;
container.SetState(ContainerState.Running);
nmContext.GetContainers()[container1] = container;
IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
);
FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);
NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestContainerLogPageAccess()
{
// SecureIOUtils require Native IO to be enabled. This test will run
// only if it is enabled.
Assume.AssumeTrue(NativeIO.IsAvailable());
string user = "******" + Runtime.CurrentTimeMillis();
FilePath absLogDir = null;
FilePath appDir = null;
FilePath containerDir = null;
FilePath syslog = null;
try
{
// target log directory
absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
.GetAbsoluteFile();
absLogDir.Mkdir();
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
healthChecker.Init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf);
long clusterTimeStamp = 1234;
ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
, 1);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
// Making sure that application returns a random user. This is required
// for SecureIOUtils' file owner check.
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
, 0);
// Testing secure read access for log files
// Creating application and container directory and syslog file.
appDir = new FilePath(absLogDir, appId.ToString());
appDir.Mkdir();
containerDir = new FilePath(appDir, container1.ToString());
containerDir.Mkdir();
syslog = new FilePath(containerDir, "syslog");
syslog.CreateNewFile();
BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
);
@out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
@out.Close();
Context context = Org.Mockito.Mockito.Mock <Context>();
ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
> appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
>();
appMap[appId] = app;
Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
> containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
>();
Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
conf, user, appId, 1);
container.SetState(ContainerState.Running);
context.GetContainers()[container1] = container;
ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
(context);
IDictionary <string, string> @params = new Dictionary <string, string>();
@params[YarnWebParams.ContainerId] = container1.ToString();
@params[YarnWebParams.ContainerLogType] = "syslog";
Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
ContainerLogsPage), cLogsBlock, @params, (Module[])null);
PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
+ user + "' doesn't own requested log file : syslog");
}
finally
{
if (syslog != null)
{
syslog.Delete();
}
if (containerDir != null)
{
containerDir.Delete();
}
if (appDir != null)
{
appDir.Delete();
}
if (absLogDir != null)
{
absLogDir.Delete();
}
}
}
internal static RecordFactory CreatePBRecordFactory()
{
Configuration conf = new Configuration();
return(RecordFactoryProvider.GetRecordFactory(conf));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint
nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler
)
{
lock (this)
{
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
ContainerLocalizer localizer = new ContainerLocalizer(lfs, user, appId, locId, GetPaths
(localDirs), RecordFactoryProvider.GetRecordFactory(GetConf()));
CreateUserLocalDirs(localDirs, user);
CreateUserCacheDirs(localDirs, user);
CreateAppDirs(localDirs, user, appId);
CreateAppLogDirs(appId, logDirs, user);
// randomly choose the local directory
Path appStorageDir = GetWorkingDir(localDirs, user, appId);
string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
CopyFile(nmPrivateContainerTokensPath, tokenDst, user);
Log.Info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
lfs.SetWorkingDirectory(appStorageDir);
Log.Info("CWD set to " + appStorageDir + " = " + lfs.GetWorkingDirectory());
// TODO: DO it over RPC for maintaining similarity?
localizer.RunLocalization(nmAddr);
}
}
public virtual void TestNodeHealthScript()
{
RecordFactory factory = RecordFactoryProvider.GetRecordFactory(null);
NodeHealthStatus healthStatus = factory.NewRecordInstance <NodeHealthStatus>();
string errorScript = "echo ERROR\n echo \"Tracker not healthy\"";
string normalScript = "echo \"I am all fine\"";
string timeOutScript = Shell.Windows ? "@echo off\nping -n 4 127.0.0.1 >nul\necho \"I am fine\""
: "sleep 4\necho \"I am fine\"";
Configuration conf = GetConfForNodeHealthScript();
conf.WriteXml(new FileOutputStream(nodeHealthConfigFile));
conf.AddResource(nodeHealthConfigFile.GetName());
WriteNodeHealthScriptFile(normalScript, true);
NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.Init(conf);
NodeHealthScriptRunner nodeHealthScriptRunner = nodeHealthChecker.GetNodeHealthScriptRunner
();
TimerTask timerTask = nodeHealthScriptRunner.GetTimerTask();
timerTask.Run();
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking initial healthy condition");
// Check proper report conditions.
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetHealthReport().Equals(nodeHealthChecker.GetHealthReport()));
// write out error file.
// Healthy to unhealthy transition
WriteNodeHealthScriptFile(errorScript, true);
// Run timer
timerTask.Run();
// update health status
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking Healthy--->Unhealthy");
NUnit.Framework.Assert.IsFalse("Node health status reported healthy", healthStatus
.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node health status reported healthy", healthStatus
.GetHealthReport().Equals(nodeHealthChecker.GetHealthReport()));
// Check unhealthy to healthy transitions.
WriteNodeHealthScriptFile(normalScript, true);
timerTask.Run();
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking UnHealthy--->healthy");
// Check proper report conditions.
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetHealthReport().Equals(nodeHealthChecker.GetHealthReport()));
// Healthy to timeout transition.
WriteNodeHealthScriptFile(timeOutScript, true);
timerTask.Run();
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking Healthy--->timeout");
NUnit.Framework.Assert.IsFalse("Node health status reported healthy even after timeout"
, healthStatus.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node script time out message not propogated", healthStatus
.GetHealthReport().Equals(NodeHealthScriptRunner.NodeHealthScriptTimedOutMsg + NodeHealthCheckerService
.Separator + nodeHealthChecker.GetDiskHandler().GetDisksHealthReport(false)));
}
public virtual void TestUpdatedNodes()
{
int runCount = 0;
MRApp app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName
, true, ++runCount);
Configuration conf = new Configuration();
// after half of the map completion, reduce will start
conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f);
// uberization forces full slowstart (1.0), so disable that
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count);
IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator();
Task mapTask1 = it.Next();
Task mapTask2 = it.Next();
// all maps must be running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next();
TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
NodeId node1 = task1Attempt.GetNodeId();
NodeId node2 = task2Attempt.GetNodeId();
NUnit.Framework.Assert.AreEqual(node1, node2);
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
// all maps must be succeeded
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Succeeded);
TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events
.Length);
// send updated nodes info
AList <NodeReport> updatedNodes = new AList <NodeReport>();
NodeReport nr = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport
>();
nr.SetNodeId(node1);
nr.SetNodeState(NodeState.Unhealthy);
updatedNodes.AddItem(nr);
app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes
));
app.WaitForState(task1Attempt, TaskAttemptState.Killed);
app.WaitForState(task2Attempt, TaskAttemptState.Killed);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed",
4, events.Length);
// all maps must be back to running
app.WaitForState(mapTask1, TaskState.Running);
app.WaitForState(mapTask2, TaskState.Running);
IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator();
itr.Next();
task1Attempt = itr.Next();
// send the done signal to the task
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID
(), TaskAttemptEventType.TaDone));
// map1 must be succeeded. map2 must be running
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
5, events.Length);
// Crash the app again.
app.Stop();
// rerun
// in rerun the 1st map will be recovered from previous run
app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName,
false, ++runCount);
conf = new Configuration();
conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true);
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
job = app.Submit(conf);
app.WaitForState(job, JobState.Running);
NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count
);
it = job.GetTasks().Values.GetEnumerator();
mapTask1 = it.Next();
mapTask2 = it.Next();
Task reduceTask1 = it.Next();
Task reduceTask2 = it.Next();
// map 1 will be recovered, no need to send done
app.WaitForState(mapTask1, TaskState.Succeeded);
app.WaitForState(mapTask2, TaskState.Running);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1"
, 2, events.Length);
task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next();
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(mapTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success",
3, events.Length);
app.WaitForState(reduceTask1, TaskState.Running);
app.WaitForState(reduceTask2, TaskState.Running);
TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask1, TaskState.Succeeded);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID
(), TaskAttemptEventType.TaKill));
app.WaitForState(reduceTask1, TaskState.Succeeded);
TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next(
);
app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID
(), TaskAttemptEventType.TaDone));
app.WaitForState(reduceTask2, TaskState.Succeeded);
events = job.GetTaskAttemptCompletionEvents(0, 100);
NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success"
, 5, events.Length);
// job succeeds
app.WaitForState(job, JobState.Succeeded);
}
public ApplicationBlock(Context nmContext, Configuration conf)
{
this.conf = conf;
this.nmContext = nmContext;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(this.conf);
}