Пример #1
0
				private CustomEventProcessor(CustomContainerLauncher _enclosing, ContainerLauncherEvent
					 @event)
					: base(_enclosing)
				{
					this._enclosing = _enclosing;
					this.@event = @event;
				}
Пример #2
0
 public virtual void Handle(ContainerLauncherEvent @event)
 {
     try
     {
         eventQueue.Put(@event);
     }
     catch (Exception e)
     {
         throw new YarnRuntimeException(e);
     }
 }
Пример #3
0
			protected internal override ContainerLauncherImpl.EventProcessor CreateEventProcessor
				(ContainerLauncherEvent @event)
			{
				// At this point of time, the EventProcessor is being created and so no
				// additional threads would have been created.
				// Core-pool-size should have increased by now.
				if (this.expectedCorePoolSize != this.launcherPool.GetCorePoolSize())
				{
					this.foundErrors = "Expected " + this.expectedCorePoolSize + " but found " + this
						.launcherPool.GetCorePoolSize();
				}
				return new TestContainerLauncher.CustomContainerLauncher.CustomEventProcessor(this
					, @event);
			}
Пример #4
0
        private ContainerLauncherImpl.Container GetContainer(ContainerLauncherEvent @event
                                                             )
        {
            ContainerId id = @event.GetContainerID();

            ContainerLauncherImpl.Container c = containers[id];
            if (c == null)
            {
                c = new ContainerLauncherImpl.Container(this, @event.GetTaskAttemptID(), @event.GetContainerID
                                                            (), @event.GetContainerMgrAddress());
                ContainerLauncherImpl.Container old = containers.PutIfAbsent(id, c);
                if (old != null)
                {
                    c = old;
                }
            }
            return(c);
        }
Пример #5
0
            public override void Run()
            {
                ContainerLauncherEvent @event   = null;
                ICollection <string>   allNodes = new HashSet <string>();

                while (!this._enclosing.stopped.Get() && !Sharpen.Thread.CurrentThread().IsInterrupted
                           ())
                {
                    try
                    {
                        @event = this._enclosing.eventQueue.Take();
                    }
                    catch (Exception e)
                    {
                        if (!this._enclosing.stopped.Get())
                        {
                            ContainerLauncherImpl.Log.Error("Returning, interrupted : " + e);
                        }
                        return;
                    }
                    allNodes.AddItem(@event.GetContainerMgrAddress());
                    int poolSize = this._enclosing.launcherPool.GetCorePoolSize();
                    if (poolSize != this._enclosing.limitOnPoolSize)
                    {
                        int numNodes      = allNodes.Count;
                        int idealPoolSize = Math.Min(this._enclosing.limitOnPoolSize, numNodes);
                        if (poolSize < idealPoolSize)
                        {
                            int newPoolSize = Math.Min(this._enclosing.limitOnPoolSize, idealPoolSize + this.
                                                       _enclosing.initialPoolSize);
                            ContainerLauncherImpl.Log.Info("Setting ContainerLauncher pool size to " + newPoolSize
                                                           + " as number-of-nodes to talk to is " + numNodes);
                            this._enclosing.launcherPool.SetCorePoolSize(newPoolSize);
                        }
                    }
                    this._enclosing.launcherPool.Execute(this._enclosing.CreateEventProcessor(@event)
                                                         );
                }
            }
        /// <exception cref="System.Exception"/>
        public virtual void TestContainerCleaned()
        {
            Log.Info("STARTING testContainerCleaned");
            CyclicBarrier startLaunchBarrier    = new CyclicBarrier(2);
            CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);
            AppContext    mockContext           = Org.Mockito.Mockito.Mock <AppContext>();
            EventHandler  mockEventHandler      = Org.Mockito.Mockito.Mock <EventHandler>();

            Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler
                                                                               );
            TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = new TestContainerLauncherImpl.ContainerManagerForTest
                                                                                     (startLaunchBarrier, completeLaunchBarrier);
            TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest
                                                                              (mockContext, mockCM);
            Configuration conf = new Configuration();

            ut.Init(conf);
            ut.Start();
            try
            {
                ContainerId             contId        = MakeContainerId(0l, 0, 0, 1);
                TaskAttemptId           taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0);
                string                  cmAddress     = "127.0.0.1:8000";
                StartContainersResponse startResp     = recordFactory.NewRecordInstance <StartContainersResponse
                                                                                         >();
                startResp.SetAllServicesMetaData(serviceResponse);
                Log.Info("inserting launch event");
                ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent
                                                                                       >();
                Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType
                                                                               .ContainerRemoteLaunch);
                Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId);
                Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
                                                                                        );
                Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
                                                                                              );
                Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken
                                                                                             (contId, cmAddress));
                ut.Handle(mockLaunchEvent);
                startLaunchBarrier.Await();
                Log.Info("inserting cleanup event");
                ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent
                                                                                    >();
                Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType
                                                                                .ContainerRemoteCleanup);
                Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId);
                Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
                                                                                         );
                Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
                                                                                               );
                ut.Handle(mockCleanupEvent);
                completeLaunchBarrier.Await();
                ut.WaitForPoolToIdle();
                ArgumentCaptor <Org.Apache.Hadoop.Yarn.Event.Event> arg = ArgumentCaptor.ForClass <
                    Org.Apache.Hadoop.Yarn.Event.Event>();
                Org.Mockito.Mockito.Verify(mockEventHandler, Org.Mockito.Mockito.AtLeast(2)).Handle
                    (arg.Capture());
                bool containerCleaned = false;
                for (int i = 0; i < arg.GetAllValues().Count; i++)
                {
                    Log.Info(arg.GetAllValues()[i].ToString());
                    Org.Apache.Hadoop.Yarn.Event.Event currentEvent = arg.GetAllValues()[i];
                    if (currentEvent.GetType() == TaskAttemptEventType.TaContainerCleaned)
                    {
                        containerCleaned = true;
                    }
                }
                System.Diagnostics.Debug.Assert((containerCleaned));
            }
            finally
            {
                ut.Stop();
            }
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestOutOfOrder()
        {
            Log.Info("STARTING testOutOfOrder");
            AppContext   mockContext      = Org.Mockito.Mockito.Mock <AppContext>();
            EventHandler mockEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();

            Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler
                                                                               );
            TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = Org.Mockito.Mockito.Mock
                                                                                 <TestContainerLauncherImpl.ContainerManagementProtocolClient>();
            TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest
                                                                              (mockContext, mockCM);
            Configuration conf = new Configuration();

            ut.Init(conf);
            ut.Start();
            try
            {
                ContainerId             contId        = MakeContainerId(0l, 0, 0, 1);
                TaskAttemptId           taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0);
                string                  cmAddress     = "127.0.0.1:8000";
                StartContainersResponse startResp     = recordFactory.NewRecordInstance <StartContainersResponse
                                                                                         >();
                startResp.SetAllServicesMetaData(serviceResponse);
                Log.Info("inserting cleanup event");
                ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent
                                                                                    >();
                Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType
                                                                                .ContainerRemoteCleanup);
                Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId);
                Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
                                                                                         );
                Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
                                                                                               );
                ut.Handle(mockCleanupEvent);
                ut.WaitForPoolToIdle();
                Org.Mockito.Mockito.Verify(mockCM, Org.Mockito.Mockito.Never()).StopContainers(Matchers.Any
                                                                                               <StopContainersRequest>());
                Log.Info("inserting launch event");
                ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent
                                                                                       >();
                Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType
                                                                               .ContainerRemoteLaunch);
                Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId);
                Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
                                                                                        );
                Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
                                                                                              );
                Org.Mockito.Mockito.When(mockCM.StartContainers(Matchers.Any <StartContainersRequest
                                                                              >())).ThenReturn(startResp);
                Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken
                                                                                             (contId, cmAddress));
                ut.Handle(mockLaunchEvent);
                ut.WaitForPoolToIdle();
                Org.Mockito.Mockito.Verify(mockCM, Org.Mockito.Mockito.Never()).StartContainers(Matchers.Any
                                                                                                <StartContainersRequest>());
            }
            finally
            {
                ut.Stop();
            }
        }
Пример #8
0
 internal EventProcessor(ContainerLauncherImpl _enclosing, ContainerLauncherEvent
                         @event)
 {
     this._enclosing = _enclosing;
     this.@event     = @event;
 }
Пример #9
0
 protected internal virtual ContainerLauncherImpl.EventProcessor CreateEventProcessor
     (ContainerLauncherEvent @event)
 {
     return(new ContainerLauncherImpl.EventProcessor(this, @event));
 }