public virtual void TestLogsViewSingle() { Log.Info("HsLogsPage with params for single log and data limits"); MockAppContext ctx = new MockAppContext(0, 1, 1, 1); IDictionary <string, string> @params = new Dictionary <string, string>(); Configuration conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, true); @params["start"] = "-2048"; @params["end"] = "-1024"; @params[YarnWebParams.ContainerLogType] = "syslog"; @params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString( ); @params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs. NmPort).ToString(); @params[YarnWebParams.EntityString] = "container_10_0001_01_000001"; @params[YarnWebParams.AppOwner] = "owner"; Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage), ctx, @params, new _AbstractModule_201(conf)); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Logs not available for container_10_0001_01_000001." + " Aggregation may not be complete, " + "Check back later or try the nodemanager at " + MockJobs.NmHost + ":" + MockJobs.NmPort); }
/// <exception cref="System.IO.IOException"/> public virtual StartContainersResponse StartContainers(StartContainersRequest requests ) { StartContainerRequest request = requests.GetStartContainerRequests()[0]; ContainerTokenIdentifier containerTokenIdentifier = MRApp.NewContainerTokenIdentifier (request.GetContainerToken()); // Validate that the container is what RM is giving. NUnit.Framework.Assert.AreEqual(MRApp.NmHost + ":" + MRApp.NmPort, containerTokenIdentifier .GetNmHostAddress()); StartContainersResponse response = TestContainerLauncher.recordFactory.NewRecordInstance <StartContainersResponse>(); this.status = TestContainerLauncher.recordFactory.NewRecordInstance<ContainerStatus >(); try { // make the thread sleep to look like its not going to respond Sharpen.Thread.Sleep(15000); } catch (Exception e) { TestContainerLauncher.Log.Error(e); throw new UndeclaredThrowableException(e); } this.status.SetState(ContainerState.Running); this.status.SetContainerId(containerTokenIdentifier.GetContainerID()); this.status.SetExitStatus(0); return response; }
private Token CreateNewContainerToken(ContainerId contId, string containerManagerAddr ) { long currentTime = Runtime.CurrentTimeMillis(); return(MRApp.NewContainerToken(NodeId.NewInstance("127.0.0.1", 1234), Sharpen.Runtime.GetBytesForString ("password"), new ContainerTokenIdentifier(contId, containerManagerAddr, "user", Resource.NewInstance(1024, 1), currentTime + 10000L, 123, currentTime, Priority. NewInstance(0), 0))); }
private void VerifyAttempt(TaskAttempt attempt) { NUnit.Framework.Assert.AreEqual("TaskAttempt state not currect", TaskAttemptState .Succeeded, attempt.GetState()); NUnit.Framework.Assert.IsNotNull(attempt.GetAssignedContainerID()); //Verify the wrong ctor is not being used. Remove after mrv1 is removed. ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1); NUnit.Framework.Assert.IsFalse(attempt.GetAssignedContainerID().Equals(fakeCid)); //Verify complete contianerManagerAddress NUnit.Framework.Assert.AreEqual(MRApp.NmHost + ":" + MRApp.NmPort, attempt.GetAssignedContainerMgrAddress ()); }
public virtual void TestLogsView2() { Log.Info("HsLogsPage with data"); MockAppContext ctx = new MockAppContext(0, 1, 1, 1); IDictionary <string, string> @params = new Dictionary <string, string>(); @params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString( ); @params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs. NmPort).ToString(); @params[YarnWebParams.EntityString] = "container_10_0001_01_000001"; @params[YarnWebParams.AppOwner] = "owner"; Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage), ctx, @params); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Aggregation is not enabled. Try the nodemanager at " + MockJobs.NmHost + ":" + MockJobs.NmPort); }
public virtual void TestLogsViewBadStartEnd() { Log.Info("HsLogsPage with bad start/end params"); MockAppContext ctx = new MockAppContext(0, 1, 1, 1); IDictionary <string, string> @params = new Dictionary <string, string>(); @params["start"] = "foo"; @params["end"] = "bar"; @params[YarnWebParams.ContainerId] = MRApp.NewContainerId(1, 1, 333, 1).ToString( ); @params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs. NmPort).ToString(); @params[YarnWebParams.EntityString] = "container_10_0001_01_000001"; @params[YarnWebParams.AppOwner] = "owner"; Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage), ctx, @params); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Invalid log start value: foo"); Org.Mockito.Mockito.Verify(spyPw).Write("Invalid log end value: bar"); }
/// <exception cref="System.Exception"/> private static void VerifySpeculationMessage(MRApp app, TaskAttempt[] ta) { app.WaitForState(ta[0], TaskAttemptState.Succeeded); }
public virtual void TestSpeculateSuccessfulWithoutUpdateEvents() { Clock actualClock = new SystemClock(); ControlledClock clock = new ControlledClock(actualClock); clock.SetTime(Runtime.CurrentTimeMillis()); MRApp app = new MRApp(NumMappers, NumReducers, false, "test", true, clock); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration(), true, true); app.WaitForState(job, JobState.Running); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("Num tasks is not correct", NumMappers + NumReducers , tasks.Count); IEnumerator <Task> taskIter = tasks.Values.GetEnumerator(); while (taskIter.HasNext()) { app.WaitForState(taskIter.Next(), TaskState.Running); } // Process the update events clock.SetTime(Runtime.CurrentTimeMillis() + 2000); EventHandler appEventHandler = app.GetContext().GetEventHandler(); foreach (KeyValuePair <TaskId, Task> mapTask in tasks) { foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask.Value.GetAttempts ()) { TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt .Key, (float)0.8, TaskAttemptState.Running); TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt .Key, status); appEventHandler.Handle(@event); } } Random generator = new Random(); object[] taskValues = Sharpen.Collections.ToArray(tasks.Values); Task taskToBeSpeculated = (Task)taskValues[generator.Next(taskValues.Length)]; // Other than one random task, finish every other task. foreach (KeyValuePair <TaskId, Task> mapTask_1 in tasks) { foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask_1.Value. GetAttempts()) { if (mapTask_1.Key != taskToBeSpeculated.GetID()) { appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType .TaDone)); appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType .TaContainerCleaned)); app.WaitForState(taskAttempt.Value, TaskAttemptState.Succeeded); } } } GenericTestUtils.WaitFor(new _Supplier_111(taskToBeSpeculated, clock), 1000, 60000 ); // finish 1st TA, 2nd will be killed TaskAttempt[] ta = MakeFirstAttemptWin(appEventHandler, taskToBeSpeculated); VerifySpeculationMessage(app, ta); app.WaitForState(Service.STATE.Stopped); }
public virtual void TestSepculateSuccessfulWithUpdateEvents() { Clock actualClock = new SystemClock(); ControlledClock clock = new ControlledClock(actualClock); clock.SetTime(Runtime.CurrentTimeMillis()); MRApp app = new MRApp(NumMappers, NumReducers, false, "test", true, clock); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration(), true, true); app.WaitForState(job, JobState.Running); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("Num tasks is not correct", NumMappers + NumReducers , tasks.Count); IEnumerator <Task> taskIter = tasks.Values.GetEnumerator(); while (taskIter.HasNext()) { app.WaitForState(taskIter.Next(), TaskState.Running); } // Process the update events clock.SetTime(Runtime.CurrentTimeMillis() + 1000); EventHandler appEventHandler = app.GetContext().GetEventHandler(); foreach (KeyValuePair <TaskId, Task> mapTask in tasks) { foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in mapTask.Value.GetAttempts ()) { TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt .Key, (float)0.5, TaskAttemptState.Running); TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt .Key, status); appEventHandler.Handle(@event); } } Task speculatedTask = null; int numTasksToFinish = NumMappers + NumReducers - 1; clock.SetTime(Runtime.CurrentTimeMillis() + 1000); foreach (KeyValuePair <TaskId, Task> task in tasks) { foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in task.Value.GetAttempts ()) { if (numTasksToFinish > 0) { appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType .TaDone)); appEventHandler.Handle(new TaskAttemptEvent(taskAttempt.Key, TaskAttemptEventType .TaContainerCleaned)); numTasksToFinish--; app.WaitForState(taskAttempt.Value, TaskAttemptState.Succeeded); } else { // The last task is chosen for speculation TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt .Key, (float)0.75, TaskAttemptState.Running); speculatedTask = task.Value; TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt .Key, status); appEventHandler.Handle(@event); } } } clock.SetTime(Runtime.CurrentTimeMillis() + 15000); foreach (KeyValuePair <TaskId, Task> task_1 in tasks) { foreach (KeyValuePair <TaskAttemptId, TaskAttempt> taskAttempt in task_1.Value.GetAttempts ()) { if (taskAttempt.Value.GetState() != TaskAttemptState.Succeeded) { TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = CreateTaskAttemptStatus(taskAttempt .Key, (float)0.75, TaskAttemptState.Running); TaskAttemptStatusUpdateEvent @event = new TaskAttemptStatusUpdateEvent(taskAttempt .Key, status); appEventHandler.Handle(@event); } } } Task speculatedTaskConst = speculatedTask; GenericTestUtils.WaitFor(new _Supplier_205(speculatedTaskConst, clock), 1000, 60000 ); TaskAttempt[] ta = MakeFirstAttemptWin(appEventHandler, speculatedTask); VerifySpeculationMessage(app, ta); app.WaitForState(Service.STATE.Stopped); }
/// <exception cref="System.Exception"/> private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps ) { Configuration conf = new Configuration(); conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name")); long amStartTimeEst = Runtime.CurrentTimeMillis(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true, this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf); FileContext fc = null; try { fc = FileContext.GetFileContext(conf); } catch (IOException ioe) { Log.Info("Can not get FileContext", ioe); throw (new Exception("Can not get File Context")); } if (numMaps == numSuccessfulMaps) { string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId); Path summaryFile = new Path(jobhistoryDir, summaryFileName); string jobSummaryString = GetJobSummary(fc, summaryFile); NUnit.Framework.Assert.IsNotNull(jobSummaryString); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100")); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100") ); IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>(); StringTokenizer strToken = new StringTokenizer(jobSummaryString, ","); while (strToken.HasMoreTokens()) { string keypair = strToken.NextToken(); jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1]; } NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements ["jobId"]); NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements ["jobName"]); NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements ["submitTime"]) != 0); NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements ["launchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstMapTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstReduceTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements ["finishTime"]) != 0); NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32 (jobSummaryElements["numMaps"])); NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32 (jobSummaryElements["numReduces"])); NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name" ), jobSummaryElements["user"]); NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements ["queue"]); NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements ["status"]); } JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser.JobInfo jobInfo; long numFinishedMaps; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; Log.Info("JobHistoryFile is: " + historyFilePath); try { @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } JobHistoryParser parser = new JobHistoryParser(@in); EventReader realReader = new EventReader(@in); EventReader reader = Org.Mockito.Mockito.Mock <EventReader>(); if (numMaps == numSuccessfulMaps) { reader = realReader; } else { AtomicInteger numFinishedEvents = new AtomicInteger(0); // Hack! Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader , numFinishedEvents, numSuccessfulMaps)); } jobInfo = parser.Parse(reader); numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps); if (numFinishedMaps != numMaps) { Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException ); } } NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name" ), jobInfo.GetUsername()); NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname( )); NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName ()); NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath ()); NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps ); NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo .GetFinishedReduces()); NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized ()); IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks(); int totalTasks = allTasks.Count; NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps + numReduces), totalTasks); // Verify aminfo NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost ()); JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0]; NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort() ); NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId()); NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId() .GetApplicationAttemptId()); NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis( ) && amInfo.GetStartTime() >= amStartTimeEst); ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1); // Assert at taskAttempt level foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values) { int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count; NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount ); JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator ().Next(); NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId()); // Verify the wrong ctor is not being used. Remove after mrv1 is removed. NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid)); } // Deep compare Job and JobInfo foreach (Task task in job.GetTasks().Values) { JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID ())]; NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1); foreach (TaskAttempt taskAttempt in task.GetAttempts().Values) { JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts( )[TypeConverter.FromYarn((taskAttempt.GetID()))]; NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo); NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt .GetShufflePort(), taskAttemptInfo.GetShufflePort()); if (numMaps == numSuccessfulMaps) { NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname()); NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort()); // Verify rack-name NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname (), RackName); } } } // test output for HistoryViewer TextWriter stdps = System.Console.Out; try { Runtime.SetOut(new TextWriter(outContent)); HistoryViewer viewer; lock (fileInfo) { viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString() , conf, true); } viewer.Print(); foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values) { string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus ()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId( ).GetJobID(); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId( ).ToString()) > 0); } } finally { Runtime.SetOut(stdps); } }