示例#1
0
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual void LocalizeFiles(LocalizationProtocol nodemanager, CompletionService
                                                      <Path> cs, UserGroupInformation ugi)
        {
            while (true)
            {
                try
                {
                    LocalizerStatus            status   = CreateStatus();
                    LocalizerHeartbeatResponse response = nodemanager.Heartbeat(status);
                    switch (response.GetLocalizerAction())
                    {
                    case LocalizerAction.Live:
                    {
                        IList <ResourceLocalizationSpec> newRsrcs = response.GetResourceSpecs();
                        foreach (ResourceLocalizationSpec newRsrc in newRsrcs)
                        {
                            if (!pendingResources.Contains(newRsrc.GetResource()))
                            {
                                pendingResources[newRsrc.GetResource()] = cs.Submit(Download(new Path(newRsrc.GetDestinationDirectory
                                                                                                          ().GetFile()), newRsrc.GetResource(), ugi));
                            }
                        }
                        break;
                    }

                    case LocalizerAction.Die:
                    {
                        // killall running localizations
                        foreach (Future <Path> pending in pendingResources.Values)
                        {
                            pending.Cancel(true);
                        }
                        status = CreateStatus();
                        // ignore response
                        try
                        {
                            nodemanager.Heartbeat(status);
                        }
                        catch (YarnException)
                        {
                        }
                        return;
                    }
                    }
                    cs.Poll(1000, TimeUnit.Milliseconds);
                }
                catch (Exception)
                {
                    return;
                }
                catch (YarnException)
                {
                    // TODO cleanup
                    return;
                }
            }
        }
示例#2
0
        /// <exception cref="System.Exception"/>
        private ContainerLocalizer SetupContainerLocalizerForTest()
        {
            // mocked generics
            // don't actually create dirs
            Org.Mockito.Mockito.DoNothing().When(spylfs).Mkdir(Matchers.IsA <Path>(), Matchers.IsA
                                                               <FsPermission>(), Matchers.AnyBoolean());
            Configuration conf = new Configuration();
            FileContext   lfs  = FileContext.GetFileContext(spylfs, conf);

            localDirs = new AList <Path>();
            for (int i = 0; i < 4; ++i)
            {
                localDirs.AddItem(lfs.MakeQualified(new Path(basedir, i + string.Empty)));
            }
            RecordFactory      mockRF      = GetMockLocalizerRecordFactory();
            ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, appId, containerId
                                                                    , localDirs, mockRF);
            ContainerLocalizer localizer = Org.Mockito.Mockito.Spy(concreteLoc);

            // return credential stream instead of opening local file
            random = new Random();
            long seed = random.NextLong();

            System.Console.Out.WriteLine("SEED: " + seed);
            random.SetSeed(seed);
            DataInputBuffer appTokens = CreateFakeCredentials(random, 10);

            tokenPath = lfs.MakeQualified(new Path(string.Format(ContainerLocalizer.TokenFileNameFmt
                                                                 , containerId)));
            Org.Mockito.Mockito.DoReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens
                                                                                         ))).When(spylfs).Open(tokenPath);
            nmProxy = Org.Mockito.Mockito.Mock <LocalizationProtocol>();
            Org.Mockito.Mockito.DoReturn(nmProxy).When(localizer).GetProxy(nmAddr);
            Org.Mockito.Mockito.DoNothing().When(localizer).Sleep(Matchers.AnyInt());
            // return result instantly for deterministic test
            ExecutorService          syncExec = Org.Mockito.Mockito.Mock <ExecutorService>();
            CompletionService <Path> cs       = Org.Mockito.Mockito.Mock <CompletionService>();

            Org.Mockito.Mockito.When(cs.Submit(Matchers.IsA <Callable>())).ThenAnswer(new _Answer_279
                                                                                          ());
            Org.Mockito.Mockito.DoReturn(syncExec).When(localizer).CreateDownloadThreadPool();
            Org.Mockito.Mockito.DoReturn(cs).When(localizer).CreateCompletionService(syncExec
                                                                                     );
            return(localizer);
        }
示例#3
0
        private void TestPbClientFactory()
        {
            IPEndPoint addr = new IPEndPoint(0);

            System.Console.Error.WriteLine(addr.GetHostName() + addr.Port);
            Configuration        conf     = new Configuration();
            LocalizationProtocol instance = new TestRPCFactories.LocalizationProtocolTestImpl
                                                (this);

            Org.Apache.Hadoop.Ipc.Server server = null;
            try
            {
                server = RpcServerFactoryPBImpl.Get().GetServer(typeof(LocalizationProtocol), instance
                                                                , addr, conf, null, 1);
                server.Start();
                System.Console.Error.WriteLine(server.GetListenerAddress());
                System.Console.Error.WriteLine(NetUtils.GetConnectAddress(server));
                try
                {
                    LocalizationProtocol client = (LocalizationProtocol)RpcClientFactoryPBImpl.Get().
                                                  GetClient(typeof(LocalizationProtocol), 1, NetUtils.GetConnectAddress(server), conf
                                                            );
                    NUnit.Framework.Assert.IsNotNull(client);
                }
                catch (YarnRuntimeException e)
                {
                    Sharpen.Runtime.PrintStackTrace(e);
                    NUnit.Framework.Assert.Fail("Failed to create client");
                }
            }
            catch (YarnRuntimeException e)
            {
                Sharpen.Runtime.PrintStackTrace(e);
                NUnit.Framework.Assert.Fail("Failed to create server");
            }
            finally
            {
                server.Stop();
            }
        }
示例#4
0
        public virtual void TestLocalizerRPC()
        {
            IPEndPoint locAddr = new IPEndPoint("0.0.0.0", 8040);

            TestPBLocalizerRPC.LocalizerService server = new TestPBLocalizerRPC.LocalizerService
                                                             (locAddr);
            try
            {
                server.Start();
                Configuration        conf   = new Configuration();
                YarnRPC              rpc    = YarnRPC.Create(conf);
                LocalizationProtocol client = (LocalizationProtocol)rpc.GetProxy(typeof(LocalizationProtocol
                                                                                        ), locAddr, conf);
                LocalizerStatus status = recordFactory.NewRecordInstance <LocalizerStatus>();
                status.SetLocalizerId("localizer0");
                LocalizerHeartbeatResponse response = client.Heartbeat(status);
                NUnit.Framework.Assert.AreEqual(DieHBResponse(), response);
            }
            finally
            {
                server.Stop();
            }
            NUnit.Framework.Assert.IsTrue(true);
        }
 public LocalizationProtocolPBServiceImpl(LocalizationProtocol impl)
 {
     this.real = impl;
 }
示例#6
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual int RunLocalization(IPEndPoint nmAddr)
        {
            // load credentials
            InitDirs(conf, user, appId, lfs, localDirs);
            Credentials     creds    = new Credentials();
            DataInputStream credFile = null;

            try
            {
                // assume credentials in cwd
                // TODO: Fix
                Path tokenPath = new Path(string.Format(TokenFileNameFmt, localizerId));
                credFile = lfs.Open(tokenPath);
                creds.ReadTokenStorageStream(credFile);
                // Explicitly deleting token file.
                lfs.Delete(tokenPath, false);
            }
            finally
            {
                if (credFile != null)
                {
                    credFile.Close();
                }
            }
            // create localizer context
            UserGroupInformation remoteUser = UserGroupInformation.CreateRemoteUser(user);

            remoteUser.AddToken(creds.GetToken(LocalizerTokenIdentifier.Kind));
            LocalizationProtocol nodeManager = remoteUser.DoAs(new _PrivilegedAction_151(this
                                                                                         , nmAddr));
            // create user context
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);

            foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in creds.GetAllTokens
                         ())
            {
                ugi.AddToken(token);
            }
            ExecutorService exec = null;

            try
            {
                exec = CreateDownloadThreadPool();
                CompletionService <Path> ecs = CreateCompletionService(exec);
                LocalizeFiles(nodeManager, ecs, ugi);
                return(0);
            }
            catch (Exception e)
            {
                // Print traces to stdout so that they can be logged by the NM address
                // space.
                Sharpen.Runtime.PrintStackTrace(e, System.Console.Out);
                return(-1);
            }
            finally
            {
                try
                {
                    if (exec != null)
                    {
                        exec.ShutdownNow();
                    }
                    LocalDirAllocator.RemoveContext(appCacheDirContextName);
                }
                finally
                {
                    CloseFileSystems(ugi);
                }
            }
        }
 public ContainerLocalizationImpl(Dispatcher dispatcher, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                  app, LocalizationProtocol localization)
 {
 }