Exemple #1
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
                                          , FilePath scriptFileDir, FilePath processStartFile)
        {
            FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
                                                            );
            ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                             >();
            NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
                                                                                       ).ToString(), 12345);
            URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
                                                                                           Path(scriptFile.GetAbsolutePath())));
            LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();

            localResource.SetResource(localResourceUri);
            localResource.SetSize(-1);
            localResource.SetVisibility(LocalResourceVisibility.Application);
            localResource.SetType(LocalResourceType.File);
            localResource.SetTimestamp(scriptFile.LastModified());
            string destinationFile = "dest_file";
            IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
                                                                                 >();

            localResources[destinationFile] = localResource;
            containerLaunchContext.SetLocalResources(localResources);
            IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));

            containerLaunchContext.SetCommands(commands);
            IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
                                                                                      , 12345);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
                                                                                         ());

            Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
                                                                                 .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
                                                                                                                                                                (), nodeId, user), containerManagerBindAddress);
            currentUser.AddToken(nmToken);
            ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
                                                                                ());
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
                                                                                                                                ().GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            IList <ContainerId> containerIds = new AList <ContainerId>();

            containerIds.AddItem(cId);
            GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
                                                                                          );
            ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
                                              GetContainerStatuses()[0];

            NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
                                                ));
        }
Exemple #2
0
        public virtual void TestOperationsWhileInSafeMode()
        {
            Path file1 = new Path("/file1");

            NUnit.Framework.Assert.IsFalse(dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeGet
                                                           ));
            DFSTestUtil.CreateFile(fs, file1, 1024, (short)1, 0);
            NUnit.Framework.Assert.IsTrue("Could not enter SM", dfs.SetSafeMode(HdfsConstants.SafeModeAction
                                                                                .SafemodeEnter));
            RunFsFun("Set quota while in SM", new _FSRun_319(file1));
            RunFsFun("Set perm while in SM", new _FSRun_325(file1));
            RunFsFun("Set owner while in SM", new _FSRun_331(file1));
            RunFsFun("Set repl while in SM", new _FSRun_337(file1));
            RunFsFun("Append file while in SM", new _FSRun_343(file1));
            RunFsFun("Truncate file while in SM", new _FSRun_349(file1));
            RunFsFun("Delete file while in SM", new _FSRun_355(file1));
            RunFsFun("Rename file while in SM", new _FSRun_361(file1));
            RunFsFun("Set time while in SM", new _FSRun_367(file1));
            RunFsFun("modifyAclEntries while in SM", new _FSRun_373(file1));
            RunFsFun("removeAclEntries while in SM", new _FSRun_379(file1));
            RunFsFun("removeDefaultAcl while in SM", new _FSRun_385(file1));
            RunFsFun("removeAcl while in SM", new _FSRun_391(file1));
            RunFsFun("setAcl while in SM", new _FSRun_397(file1));
            RunFsFun("setXAttr while in SM", new _FSRun_403(file1));
            RunFsFun("removeXAttr while in SM", new _FSRun_409(file1));
            try
            {
                DFSTestUtil.ReadFile(fs, file1);
            }
            catch (IOException)
            {
                NUnit.Framework.Assert.Fail("Set times failed while in SM");
            }
            try
            {
                fs.GetAclStatus(file1);
            }
            catch (IOException)
            {
                NUnit.Framework.Assert.Fail("getAclStatus failed while in SM");
            }
            // Test access
            UserGroupInformation ugiX = UserGroupInformation.CreateRemoteUser("userX");
            FileSystem           myfs = ugiX.DoAs(new _PrivilegedExceptionAction_429(this));

            myfs.Access(file1, FsAction.Read);
            try
            {
                myfs.Access(file1, FsAction.Write);
                NUnit.Framework.Assert.Fail("The access call should have failed.");
            }
            catch (AccessControlException)
            {
            }
            // expected
            NUnit.Framework.Assert.IsFalse("Could not leave SM", dfs.SetSafeMode(HdfsConstants.SafeModeAction
                                                                                 .SafemodeLeave));
        }
Exemple #3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestUGIAuthMethod()
        {
            UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();

            UserGroupInformation.AuthenticationMethod am = UserGroupInformation.AuthenticationMethod
                                                           .Kerberos;
            ugi.SetAuthenticationMethod(am);
            Assert.Equal(am, ugi.GetAuthenticationMethod());
            ugi.DoAs(new _PrivilegedExceptionAction_668(am));
        }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private Token GetDelegationToken(UserGroupInformation loggedInUser, MRClientProtocol
                                         hsService, string renewerString)
        {
            // Get the delegation token directly as it is a little difficult to setup
            // the kerberos based rpc.
            Token token = loggedInUser.DoAs(new _PrivilegedExceptionAction_236(renewerString,
                                                                               hsService));

            return(token);
        }
        public virtual void TestDelegationTokenWithDoAs()
        {
            DistributedFileSystem dfs   = cluster.GetFileSystem();
            Credentials           creds = new Credentials();

            Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = dfs.AddDelegationTokens
                                                                           ("JobTracker", creds);
            NUnit.Framework.Assert.AreEqual(1, tokens.Length);
            Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = (Org.Apache.Hadoop.Security.Token.Token
                                                                                        <DelegationTokenIdentifier>)tokens[0];
            UserGroupInformation longUgi = UserGroupInformation.CreateRemoteUser("JobTracker/[email protected]"
                                                                                 );
            UserGroupInformation shortUgi = UserGroupInformation.CreateRemoteUser("JobTracker"
                                                                                  );

            longUgi.DoAs(new _PrivilegedExceptionAction_212(this, token, longUgi));
            shortUgi.DoAs(new _PrivilegedExceptionAction_223(this, token));
            longUgi.DoAs(new _PrivilegedExceptionAction_230(this, token, longUgi));
        }
Exemple #6
0
        public virtual SaslServer Create(Server.Connection connection, IDictionary <string
                                                                                    , object> saslProperties, SecretManager <TokenIdentifier> secretManager)
        {
            UserGroupInformation ugi = null;
            CallbackHandler      callback;

            switch (authMethod)
            {
            case SaslRpcServer.AuthMethod.Token:
            {
                callback = new SaslRpcServer.SaslDigestCallbackHandler(secretManager, connection);
                break;
            }

            case SaslRpcServer.AuthMethod.Kerberos:
            {
                ugi = UserGroupInformation.GetCurrentUser();
                if (serverId.IsEmpty())
                {
                    throw new AccessControlException("Kerberos principal name does NOT have the expected "
                                                     + "hostname part: " + ugi.GetUserName());
                }
                callback = new SaslRpcServer.SaslGssCallbackHandler();
                break;
            }

            default:
            {
                // we should never be able to get here
                throw new AccessControlException("Server does not support SASL " + authMethod);
            }
            }
            SaslServer saslServer;

            if (ugi != null)
            {
                saslServer = ugi.DoAs(new _PrivilegedExceptionAction_159(this, saslProperties, callback
                                                                         ));
            }
            else
            {
                saslServer = saslFactory.CreateSaslServer(mechanism, protocol, serverId, saslProperties
                                                          , callback);
            }
            if (saslServer == null)
            {
                throw new AccessControlException("Unable to find SASL server implementation for "
                                                 + mechanism);
            }
            if (Log.IsDebugEnabled())
            {
                Log.Debug("Created SASL server with mechanism = " + mechanism);
            }
            return(saslServer);
        }
Exemple #7
0
        public virtual Response GetKeyNames()
        {
            KMSWebApp.GetAdminCallsMeter().Mark();
            UserGroupInformation user = HttpUserGroupInformation.Get();

            AssertAccess(KMSACLs.Type.GetKeys, user, KMS.KMSOp.GetKeys);
            IList <string> json = user.DoAs(new _PrivilegedExceptionAction_256(this));

            kmsAudit.Ok(user, KMS.KMSOp.GetKeys, string.Empty);
            return(Response.Ok().Type(MediaType.ApplicationJson).Entity(json).Build());
        }
Exemple #8
0
        public virtual Response DeleteKey(string name)
        {
            KMSWebApp.GetAdminCallsMeter().Mark();
            UserGroupInformation user = HttpUserGroupInformation.Get();

            AssertAccess(KMSACLs.Type.Delete, user, KMS.KMSOp.DeleteKey, name);
            KMSClientProvider.CheckNotEmpty(name, "name");
            user.DoAs(new _PrivilegedExceptionAction_168(this, name));
            kmsAudit.Ok(user, KMS.KMSOp.DeleteKey, name, string.Empty);
            return(Response.Ok().Build());
        }
        public virtual void TestValidateResourceBlacklistRequest()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
                                                                                         YarnConfiguration(), containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
                                                                                          , rmBindAddress, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();

            client.RegisterApplicationMaster(request);
            ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
                Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
            AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
                                                                          , blacklistRequest);
            bool error = false;

            try
            {
                client.Allocate(allocateRequest);
            }
            catch (InvalidResourceBlacklistRequestException)
            {
                error = true;
            }
            rm.Stop();
            NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
                                          , error);
        }
Exemple #10
0
        /// <summary>given user name - get all the groups.</summary>
        /// <remarks>
        /// given user name - get all the groups.
        /// Needs to happen before creating the test users
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual void TestGetServerSideGroups()
        {
            // get the user name
            SystemProcess  pp = Runtime.GetRuntime().Exec("whoami");
            BufferedReader br = new BufferedReader(new InputStreamReader(pp.GetInputStream())
                                                   );
            string userName = br.ReadLine().Trim();

            // If on windows domain, token format is DOMAIN\\user and we want to
            // extract only the user name
            if (Shell.Windows)
            {
                int sp = userName.LastIndexOf('\\');
                if (sp != -1)
                {
                    userName = Runtime.Substring(userName, sp + 1);
                }
                // user names are case insensitive on Windows. Make consistent
                userName = StringUtils.ToLowerCase(userName);
            }
            // get the groups
            pp = Runtime.GetRuntime().Exec(Shell.Windows ? Shell.Winutils + " groups -F" : "id -Gn"
                                           );
            br = new BufferedReader(new InputStreamReader(pp.GetInputStream()));
            string line = br.ReadLine();

            System.Console.Out.WriteLine(userName + ":" + line);
            ICollection <string> groups = new LinkedHashSet <string>();

            string[] tokens = line.Split(Shell.TokenSeparatorRegex);
            foreach (string s in tokens)
            {
                groups.AddItem(s);
            }
            UserGroupInformation login = UserGroupInformation.GetCurrentUser();
            string loginUserName       = login.GetShortUserName();

            if (Shell.Windows)
            {
                // user names are case insensitive on Windows. Make consistent
                loginUserName = StringUtils.ToLowerCase(loginUserName);
            }
            Assert.Equal(userName, loginUserName);
            string[] gi = login.GetGroupNames();
            Assert.Equal(groups.Count, gi.Length);
            for (int i = 0; i < gi.Length; i++)
            {
                Assert.True(groups.Contains(gi[i]));
            }
            UserGroupInformation fakeUser = UserGroupInformation.CreateRemoteUser("foo.bar");

            fakeUser.DoAs(new _PrivilegedExceptionAction_248(login, fakeUser));
        }
Exemple #11
0
        public virtual void TestDelegationToProvider()
        {
            NUnit.Framework.Assert.IsTrue(Called.Contains("start"));
            FileSystem fs = FileSystem.Get(miniDFS.GetConfiguration(0));

            fs.Mkdirs(new Path("/tmp"));
            fs.SetPermission(new Path("/tmp"), new FsPermission((short)0x1ff));
            UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("u1", new string
                                                                                 [] { "g1" });

            ugi.DoAs(new _PrivilegedExceptionAction_201(this));
        }
Exemple #12
0
 /// <exception cref="System.IO.IOException"/>
 private static T DoAsUser <T>(UserGroupInformation ugi, PrivilegedExceptionAction <
                                   T> action)
 {
     try
     {
         return(ugi.DoAs(action));
     }
     catch (Exception ie)
     {
         throw new IOException(ie);
     }
 }
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual MRClientProtocol InstantiateHistoryProxy(Configuration
                                                                            conf, IPEndPoint hsAddress)
        {
            if (Log.IsDebugEnabled())
            {
                Log.Debug("Connecting to MRHistoryServer at: " + hsAddress);
            }
            YarnRPC rpc = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.GetCurrentUser();

            return(currentUser.DoAs(new _PrivilegedAction_113(rpc, hsAddress, conf)));
        }
        /// <exception cref="System.Exception"/>
        private AllocateResponse Allocate(ApplicationAttemptId attemptId, AllocateRequest
                                          req)
        {
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(attemptId.ToString
                                                                                 ());

            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm.GetRMContext
                                                                                     ().GetRMApps()[attemptId.GetApplicationId()].GetRMAppAttempt(attemptId).GetAMRMToken
                                                                                     ();
            ugi.AddTokenIdentifier(token.DecodeIdentifier());
            return(ugi.DoAs(new _PrivilegedExceptionAction_67(this, req)));
        }
        private MRClientProtocol GetMRClientProtocol(Token token, IPEndPoint hsAddress, string
                                                     user, Configuration conf)
        {
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);

            ugi.AddToken(ConverterUtils.ConvertFromYarn(token, hsAddress));
            YarnRPC          rpc      = YarnRPC.Create(conf);
            MRClientProtocol hsWithDT = ugi.DoAs(new _PrivilegedAction_288(rpc, hsAddress, conf
                                                                           ));

            return(hsWithDT);
        }
Exemple #16
0
        public virtual void TestMultipleKMSInstancesWithZKSigner()
        {
            FilePath      testDir  = TestKMS.GetTestDir();
            Configuration conf     = CreateBaseKMSConf(testDir);
            TestingServer zkServer = new TestingServer();

            zkServer.Start();
            MiniKMS kms1 = null;
            MiniKMS kms2 = null;

            conf.Set(KMSAuthenticationFilter.ConfigPrefix + AuthenticationFilter.SignerSecretProvider
                     , "zookeeper");
            conf.Set(KMSAuthenticationFilter.ConfigPrefix + ZKSignerSecretProvider.ZookeeperConnectionString
                     , zkServer.GetConnectString());
            conf.Set(KMSAuthenticationFilter.ConfigPrefix + ZKSignerSecretProvider.ZookeeperPath
                     , "/secret");
            TestKMS.WriteConf(testDir, conf);
            try
            {
                kms1 = new MiniKMS.Builder().SetKmsConfDir(testDir).SetLog4jConfFile("log4j.properties"
                                                                                     ).Build();
                kms1.Start();
                kms2 = new MiniKMS.Builder().SetKmsConfDir(testDir).SetLog4jConfFile("log4j.properties"
                                                                                     ).Build();
                kms2.Start();
                Uri url1 = new Uri(kms1.GetKMSUrl().ToExternalForm() + KMSRESTConstants.ServiceVersion
                                   + "/" + KMSRESTConstants.KeysNamesResource);
                Uri url2 = new Uri(kms2.GetKMSUrl().ToExternalForm() + KMSRESTConstants.ServiceVersion
                                   + "/" + KMSRESTConstants.KeysNamesResource);
                DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token
                                                                  ();
                DelegationTokenAuthenticatedURL aUrl   = new DelegationTokenAuthenticatedURL();
                UserGroupInformation            ugiFoo = UserGroupInformation.CreateUserForTesting("foo", new
                                                                                                   string[] { "gfoo" });
                UserGroupInformation ugiBar = UserGroupInformation.CreateUserForTesting("bar", new
                                                                                        string[] { "gBar" });
                ugiFoo.DoAs(new _PrivilegedExceptionAction_135(aUrl, url1, token));
                ugiBar.DoAs(new _PrivilegedExceptionAction_145(aUrl, url2, token));
                ugiBar.DoAs(new _PrivilegedExceptionAction_155(aUrl, url2));
            }
            finally
            {
                if (kms2 != null)
                {
                    kms2.Stop();
                }
                if (kms1 != null)
                {
                    kms1.Stop();
                }
                zkServer.Stop();
            }
        }
Exemple #17
0
        public virtual void TestAuthorizedAccess()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            int waitCount = 0;

            while (containerManager.containerTokens == null && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Launch to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
                                                                                          , rpc, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();
            RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
                                                                                          );

            NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
            if (UserGroupInformation.IsSecurityEnabled())
            {
                NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
                                                   ()).Length > 0);
            }
            NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
                                                ()[ApplicationAccessType.ViewApp]);
        }
        public virtual void Pipeline_02_03()
        {
            Configuration conf = new HdfsConfiguration();

            conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
            // create cluster
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();

            try
            {
                //change the lease limits.
                cluster.SetLeasePeriod(SoftLeaseLimit, HardLeaseLimit);
                //wait for the cluster
                cluster.WaitActive();
                FileSystem fs   = cluster.GetFileSystem();
                Path       p    = new Path(Dir, "file1");
                int        half = BlockSize / 2;
                {
                    //a. On Machine M1, Create file. Write half block of data.
                    //   Invoke DFSOutputStream.hflush() on the dfs file handle.
                    //   Do not close file yet.
                    FSDataOutputStream @out = fs.Create(p, true, fs.GetConf().GetInt(CommonConfigurationKeys
                                                                                     .IoFileBufferSizeKey, 4096), (short)3, BlockSize);
                    Write(@out, 0, half);
                    //hflush
                    ((DFSOutputStream)@out.GetWrappedStream()).Hflush();
                }
                //b. On another machine M2, open file and verify that the half-block
                //   of data can be read successfully.
                CheckFile(p, half, conf);
                AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()");
                ((DistributedFileSystem)fs).dfs.GetLeaseRenewer().InterruptAndJoin();
                {
                    //c. On M1, append another half block of data.  Close file on M1.
                    //sleep to let the lease is expired.
                    Sharpen.Thread.Sleep(2 * SoftLeaseLimit);
                    UserGroupInformation current = UserGroupInformation.GetCurrentUser();
                    UserGroupInformation ugi     = UserGroupInformation.CreateUserForTesting(current.GetShortUserName
                                                                                                 () + "x", new string[] { "supergroup" });
                    DistributedFileSystem dfs  = ugi.DoAs(new _PrivilegedExceptionAction_102(conf));
                    FSDataOutputStream    @out = Append(dfs, p);
                    Write(@out, 0, half);
                    @out.Close();
                }
                //d. On M2, open file and read 1 block of data from it. Close file.
                CheckFile(p, 2 * half, conf);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Exemple #19
0
        /// <exception cref="System.Exception"/>
        protected override void ChannelRead0(ChannelHandlerContext ctx, HttpRequest req)
        {
            Preconditions.CheckArgument(req.GetUri().StartsWith(WebhdfsPrefix));
            QueryStringDecoder queryString = new QueryStringDecoder(req.GetUri());

            @params = new ParameterParser(queryString, conf);
            DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(@params);

            ugi  = ugiProvider.Ugi();
            path = @params.Path();
            InjectToken();
            ugi.DoAs(new _PrivilegedExceptionAction_110(this, ctx, req));
        }
Exemple #20
0
        public virtual void TestCreateProxyUser()
        {
            // ensure that doAs works correctly
            UserGroupInformation realUserUgi = UserGroupInformation.CreateRemoteUser(RealUserName
                                                                                     );
            UserGroupInformation proxyUserUgi = UserGroupInformation.CreateProxyUser(ProxyUserName
                                                                                     , realUserUgi);
            UserGroupInformation curUGI = proxyUserUgi.DoAs(new _PrivilegedExceptionAction_122
                                                                ());

            Assert.Equal(ProxyUserName + " (auth:PROXY) via " + RealUserName
                         + " (auth:SIMPLE)", curUGI.ToString());
        }
Exemple #21
0
 /// <exception cref="System.Exception"/>
 public virtual AllocateResponse DoAllocateAs(UserGroupInformation ugi, AllocateRequest
                                              req)
 {
     req.SetResponseId(++responseId);
     try
     {
         return(ugi.DoAs(new _PrivilegedExceptionAction_234(this, req)));
     }
     catch (UndeclaredThrowableException e)
     {
         throw (Exception)e.InnerException;
     }
 }
        /// <exception cref="System.Exception"/>
        public virtual void TestDelegationTokenWithDoAs()
        {
            Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken
                                                                                           (fs, "JobTracker");
            UserGroupInformation longUgi = UserGroupInformation.CreateRemoteUser("JobTracker/[email protected]"
                                                                                 );
            UserGroupInformation shortUgi = UserGroupInformation.CreateRemoteUser("JobTracker"
                                                                                  );

            longUgi.DoAs(new _PrivilegedExceptionAction_260(token));
            // try renew with long name
            shortUgi.DoAs(new _PrivilegedExceptionAction_268(token));
            longUgi.DoAs(new _PrivilegedExceptionAction_275(token));
        }
Exemple #23
0
        public virtual void TestAMRMTokenUpdate()
        {
            Configuration        conf      = new Configuration();
            ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(ApplicationId.NewInstance
                                                                                  (1, 1), 1);
            AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
            AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);

            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = new Org.Apache.Hadoop.Security.Token.Token
                                                                                    <AMRMTokenIdentifier>(oldTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("oldpassword"
                                                                                                                                                                   ), oldTokenId.GetKind(), new Text());
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token
                                                                                    <AMRMTokenIdentifier>(newTokenId.GetBytes(), Sharpen.Runtime.GetBytesForString("newpassword"
                                                                                                                                                                   ), newTokenId.GetKind(), new Text());
            TestLocalContainerAllocator.MockScheduler scheduler = new TestLocalContainerAllocator.MockScheduler
                                                                      ();
            scheduler.amToken = newToken;
            LocalContainerAllocator lca = new TestLocalContainerAllocator.StubbedLocalContainerAllocator
                                              (scheduler);

            lca.Init(conf);
            lca.Start();
            UserGroupInformation testUgi = UserGroupInformation.CreateUserForTesting("someuser"
                                                                                     , new string[0]);

            testUgi.AddToken(oldToken);
            testUgi.DoAs(new _PrivilegedExceptionAction_144(lca));
            lca.Close();
            // verify there is only one AMRM token in the UGI and it matches the
            // updated token from the RM
            int tokenCount = 0;

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> ugiToken = null;
            foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in testUgi
                     .GetTokens())
            {
                if (AMRMTokenIdentifier.KindName.Equals(token.GetKind()))
                {
                    ugiToken = token;
                    ++tokenCount;
                }
            }
            NUnit.Framework.Assert.AreEqual("too many AMRM tokens", 1, tokenCount);
            Assert.AssertArrayEquals("token identifier not updated", newToken.GetIdentifier()
                                     , ugiToken.GetIdentifier());
            Assert.AssertArrayEquals("token password not updated", newToken.GetPassword(), ugiToken
                                     .GetPassword());
            NUnit.Framework.Assert.AreEqual("AMRM token service not updated", new Text(ClientRMProxy
                                                                                       .GetAMRMTokenService(conf)), ugiToken.GetService());
        }
Exemple #24
0
        public virtual Response GetKeyVersions(string name)
        {
            UserGroupInformation user = HttpUserGroupInformation.Get();

            KMSClientProvider.CheckNotEmpty(name, "name");
            KMSWebApp.GetKeyCallsMeter().Mark();
            AssertAccess(KMSACLs.Type.Get, user, KMS.KMSOp.GetKeyVersions, name);
            IList <KeyProvider.KeyVersion> ret = user.DoAs(new _PrivilegedExceptionAction_469(
                                                               this, name));
            object json = KMSServerJSONUtils.ToJSON(ret);

            kmsAudit.Ok(user, KMS.KMSOp.GetKeyVersions, name, string.Empty);
            return(Response.Ok().Type(MediaType.ApplicationJson).Entity(json).Build());
        }
Exemple #25
0
        public virtual Response CreateKey(IDictionary jsonKey)
        {
            KMSWebApp.GetAdminCallsMeter().Mark();
            UserGroupInformation user = HttpUserGroupInformation.Get();
            string name = (string)jsonKey[KMSRESTConstants.NameField];

            KMSClientProvider.CheckNotEmpty(name, KMSRESTConstants.NameField);
            AssertAccess(KMSACLs.Type.Create, user, KMS.KMSOp.CreateKey, name);
            string cipher   = (string)jsonKey[KMSRESTConstants.CipherField];
            string material = (string)jsonKey[KMSRESTConstants.MaterialField];
            int    length   = (jsonKey.Contains(KMSRESTConstants.LengthField)) ? (int)jsonKey[KMSRESTConstants
                                                                                              .LengthField] : 0;
            string description = (string)jsonKey[KMSRESTConstants.DescriptionField];
            IDictionary <string, string> attributes = (IDictionary <string, string>)jsonKey[KMSRESTConstants
                                                                                            .AttributesField];

            if (material != null)
            {
                AssertAccess(KMSACLs.Type.SetKeyMaterial, user, KMS.KMSOp.CreateKey, name);
            }
            KeyProvider.Options options = new KeyProvider.Options(KMSWebApp.GetConfiguration(
                                                                      ));
            if (cipher != null)
            {
                options.SetCipher(cipher);
            }
            if (length != 0)
            {
                options.SetBitLength(length);
            }
            options.SetDescription(description);
            options.SetAttributes(attributes);
            KeyProvider.KeyVersion keyVersion = user.DoAs(new _PrivilegedExceptionAction_132(
                                                              this, material, name, options));
            kmsAudit.Ok(user, KMS.KMSOp.CreateKey, name, "UserProvidedMaterial:" + (material
                                                                                    != null) + " Description:" + description);
            if (!KMSWebApp.GetACLs().HasAccess(KMSACLs.Type.Get, user))
            {
                keyVersion = RemoveKeyMaterial(keyVersion);
            }
            IDictionary json       = KMSServerJSONUtils.ToJSON(keyVersion);
            string      requestURL = KMSMDCFilter.GetURL();
            int         idx        = requestURL.LastIndexOf(KMSRESTConstants.KeysResource);

            requestURL = Runtime.Substring(requestURL, 0, idx);
            string keyURL = requestURL + KMSRESTConstants.KeyResource + "/" + name;

            return(Response.Created(GetKeyURI(name)).Type(MediaType.ApplicationJson).Header("Location"
                                                                                            , keyURL).Entity(json).Build());
        }
Exemple #26
0
        private ApplicationClientProtocol GetClientRMProtocolWithDT(Org.Apache.Hadoop.Yarn.Api.Records.Token
                                                                    token, IPEndPoint rmAddress, string user, Configuration conf)
        {
            // Maybe consider converting to Hadoop token, serialize de-serialize etc
            // before trying to renew the token.
            UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser(user);

            ugi.AddToken(ConverterUtils.ConvertFromYarn(token, rmAddress));
            YarnRPC rpc = YarnRPC.Create(conf);
            ApplicationClientProtocol clientRMWithDT = ugi.DoAs(new _PrivilegedAction_464(rpc
                                                                                          , rmAddress, conf));

            return(clientRMWithDT);
        }
Exemple #27
0
        public virtual Response GetKeysMetadata(IList <string> keyNamesList)
        {
            KMSWebApp.GetAdminCallsMeter().Mark();
            UserGroupInformation user = HttpUserGroupInformation.Get();

            string[] keyNames = Collections.ToArray(keyNamesList, new string[keyNamesList
                                                                             .Count]);
            AssertAccess(KMSACLs.Type.GetMetadata, user, KMS.KMSOp.GetKeysMetadata);
            KeyProvider.Metadata[] keysMeta = user.DoAs(new _PrivilegedExceptionAction_234(this
                                                                                           , keyNames));
            object json = KMSServerJSONUtils.ToJSON(keyNames, keysMeta);

            kmsAudit.Ok(user, KMS.KMSOp.GetKeysMetadata, string.Empty);
            return(Response.Ok().Type(MediaType.ApplicationJson).Entity(json).Build());
        }
Exemple #28
0
 protected internal virtual void CreateAppDir(string user, ApplicationId appId, UserGroupInformation
                                              userUgi)
 {
     try
     {
         userUgi.DoAs(new _PrivilegedExceptionAction_261(this, appId, user));
     }
     catch (Exception e)
     {
         // TODO: Reuse FS for user?
         // Only creating directories if they are missing to avoid
         // unnecessary load on the filesystem from all of the nodes
         throw new YarnRuntimeException(e);
     }
 }
Exemple #29
0
        /// <exception cref="System.IO.IOException"/>
        protected internal virtual MRClientProtocol InstantiateHistoryProxy()
        {
            string serviceAddr = conf.Get(JHAdminConfig.MrHistoryAddress);

            if (StringUtils.IsEmpty(serviceAddr))
            {
                return(null);
            }
            Log.Debug("Connecting to HistoryServer at: " + serviceAddr);
            YarnRPC rpc = YarnRPC.Create(conf);

            Log.Debug("Connected to HistoryServer at: " + serviceAddr);
            UserGroupInformation currentUser = UserGroupInformation.GetCurrentUser();

            return(currentUser.DoAs(new _PrivilegedAction_92(this, rpc, serviceAddr)));
        }
 /// <exception cref="System.IO.IOException"/>
 public virtual void TestDelegationTokenWithRealUser()
 {
     try
     {
         Org.Apache.Hadoop.Security.Token.Token <object>[] tokens = proxyUgi.DoAs(new _PrivilegedExceptionAction_131
                                                                                      ());
         DelegationTokenIdentifier identifier = new DelegationTokenIdentifier();
         byte[] tokenId = tokens[0].GetIdentifier();
         identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
         NUnit.Framework.Assert.AreEqual(identifier.GetUser().GetUserName(), ProxyUser);
         NUnit.Framework.Assert.AreEqual(identifier.GetUser().GetRealUser().GetUserName(),
                                         RealUser);
     }
     catch (Exception)
     {
     }
 }