/// <exception cref="System.Exception"/>
        public virtual void TestClientAccessPrivilegeForRemove()
        {
            // Configure ro access for nfs1 service
            config.Set("dfs.nfs.exports.allowed.hosts", "* ro");
            // Start nfs
            Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3
                                                           (config);
            nfs.StartServiceInternal(false);
            RpcProgramNfs3 nfsd = (RpcProgramNfs3)nfs.GetRpcProgram();
            // Create a remove request
            HdfsFileStatus status  = nn.GetRpcServer().GetFileInfo(testdir);
            long           dirId   = status.GetFileId();
            XDR            xdr_req = new XDR();
            FileHandle     handle  = new FileHandle(dirId);

            handle.Serialize(xdr_req);
            xdr_req.WriteString("f1");
            // Remove operation
            REMOVE3Response response = nfsd.Remove(xdr_req.AsReadOnlyWrap(), securityHandler,
                                                   new IPEndPoint("localhost", 1234));

            // Assert on return code
            NUnit.Framework.Assert.AreEqual("Incorrect return code", Nfs3Status.Nfs3errAcces,
                                            response.GetStatus());
        }
Ejemplo n.º 2
0
        public virtual void TestCreate()
        {
            string src = "/testNamenodeRetryCache/testCreate/file";

            // Two retried calls succeed
            NewCall();
            HdfsFileStatus status = nnRpc.Create(src, perm, "holder", new EnumSetWritable <CreateFlag
                                                                                           >(EnumSet.Of(CreateFlag.Create)), true, (short)1, BlockSize, null);

            NUnit.Framework.Assert.AreEqual(status, nnRpc.Create(src, perm, "holder", new EnumSetWritable
                                                                 <CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)1, BlockSize, null));
            NUnit.Framework.Assert.AreEqual(status, nnRpc.Create(src, perm, "holder", new EnumSetWritable
                                                                 <CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)1, BlockSize, null));
            // A non-retried call fails
            NewCall();
            try
            {
                nnRpc.Create(src, perm, "holder", new EnumSetWritable <CreateFlag>(EnumSet.Of(CreateFlag
                                                                                              .Create)), true, (short)1, BlockSize, null);
                NUnit.Framework.Assert.Fail("testCreate - expected exception is not thrown");
            }
            catch (IOException)
            {
            }
        }
Ejemplo n.º 3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestMkdir()
        {
            //FixME
            HdfsFileStatus status  = nn.GetRpcServer().GetFileInfo(testdir);
            long           dirId   = status.GetFileId();
            XDR            xdr_req = new XDR();
            FileHandle     handle  = new FileHandle(dirId);
            MKDIR3Request  req     = new MKDIR3Request(handle, "fubar1", new SetAttr3());

            req.Serialize(xdr_req);
            // Attempt to mkdir by an unprivileged user should fail.
            MKDIR3Response response1 = nfsd.Mkdir(xdr_req.AsReadOnlyWrap(), securityHandlerUnpriviledged
                                                  , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces
                                            , response1.GetStatus());
            XDR           xdr_req2 = new XDR();
            MKDIR3Request req2     = new MKDIR3Request(handle, "fubar2", new SetAttr3());

            req2.Serialize(xdr_req2);
            // Attempt to mkdir by a privileged user should pass.
            MKDIR3Response response2 = nfsd.Mkdir(xdr_req2.AsReadOnlyWrap(), securityHandler,
                                                  new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3Ok, response2
                                            .GetStatus());
        }
Ejemplo n.º 4
0
            /// <exception cref="System.IO.IOException"/>
            public Void Run()
            {
                ClientProtocol nn              = this._enclosing.CreateNameNodeProxy();
                string         path            = ServletUtil.GetDecodedPath(request, "/data");
                string         encodedPath     = ServletUtil.GetRawPath(request, "/data");
                string         delegationToken = request.GetParameter(JspHelper.DelegationParameterName);
                HdfsFileStatus info            = nn.GetFileInfo(path);

                if (info != null && !info.IsDir())
                {
                    response.SendRedirect(this._enclosing.CreateRedirectURL(path, encodedPath, info,
                                                                            ugi, nn, request, delegationToken).ToString());
                }
                else
                {
                    if (info == null)
                    {
                        response.SendError(400, "File not found " + path);
                    }
                    else
                    {
                        response.SendError(400, path + ": is a directory");
                    }
                }
                return(null);
            }
Ejemplo n.º 5
0
        internal static FSDirRenameOp.RenameOldResult RenameToInt(FSDirectory fsd, string
                                                                  srcArg, string dstArg, bool logRetryCache)
        {
            string src = srcArg;
            string dst = dstArg;

            if (NameNode.stateChangeLog.IsDebugEnabled())
            {
                NameNode.stateChangeLog.Debug("DIR* NameSystem.renameTo: " + src + " to " + dst);
            }
            if (!DFSUtil.IsValidName(dst))
            {
                throw new IOException("Invalid name: " + dst);
            }
            FSPermissionChecker pc = fsd.GetPermissionChecker();

            byte[][]       srcComponents = FSDirectory.GetPathComponentsForReservedPath(src);
            byte[][]       dstComponents = FSDirectory.GetPathComponentsForReservedPath(dst);
            HdfsFileStatus resultingStat = null;

            src = fsd.ResolvePath(pc, src, srcComponents);
            dst = fsd.ResolvePath(pc, dst, dstComponents);
            bool status = RenameTo(fsd, pc, src, dst, logRetryCache);

            if (status)
            {
                INodesInPath dstIIP = fsd.GetINodesInPath(dst, false);
                resultingStat = fsd.GetAuditFileInfo(dstIIP);
            }
            return(new FSDirRenameOp.RenameOldResult(status, resultingStat));
        }
Ejemplo n.º 6
0
        /* fsid */
        /// <exception cref="System.IO.IOException"/>
        public static Nfs3FileAttributes GetFileAttr(DFSClient client, string fileIdPath,
                                                     IdMappingServiceProvider iug)
        {
            HdfsFileStatus fs = GetFileStatus(client, fileIdPath);

            return(fs == null ? null : GetNfs3FileAttrFromFileStatus(fs, iug));
        }
Ejemplo n.º 7
0
 internal static FileStatus ToFileStatus(HdfsFileStatus f, string parent)
 {
     return(new FileStatus(f.GetLen(), f.IsDir(), f.GetReplication(), f.GetBlockSize()
                           , f.GetModificationTime(), f.GetAccessTime(), f.GetPermission(), f.GetOwner(), f
                           .GetGroup(), f.IsSymlink() ? new Path(f.GetSymlink()) : null, new Path(f.GetFullName
                                                                                                      (parent))));
 }
Ejemplo n.º 8
0
        /// <summary>Create a redirection URL</summary>
        /// <exception cref="System.IO.IOException"/>
        private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status
                                      , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request,
                                      string dt)
        {
            string        scheme = request.GetScheme();
            LocatedBlocks blks   = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path))
                                                             .ToUri().GetPath(), 0, 1);
            Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext());
            DatanodeID    host = PickSrcDatanode(blks, status, conf);
            string        hostname;

            if (host is DatanodeInfo)
            {
                hostname = host.GetHostName();
            }
            else
            {
                hostname = host.GetIpAddr();
            }
            int    port    = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort();
            string dtParam = string.Empty;

            if (dt != null)
            {
                dtParam = JspHelper.GetDelegationTokenUrlParam(dt);
            }
            // Add namenode address to the url params
            NameNode nn        = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
            string   addr      = nn.GetNameNodeAddressHostPortString();
            string   addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr);

            return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi="
                           + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam));
        }
Ejemplo n.º 9
0
        /// <summary>Get a listing of all the snapshots of a snapshottable directory</summary>
        /// <exception cref="System.IO.IOException"/>
        private static DirectoryListing GetSnapshotsListing(FSDirectory fsd, string src,
                                                            byte[] startAfter)
        {
            Preconditions.CheckState(fsd.HasReadLock());
            Preconditions.CheckArgument(src.EndsWith(HdfsConstants.SeparatorDotSnapshotDir),
                                        "%s does not end with %s", src, HdfsConstants.SeparatorDotSnapshotDir);
            string dirPath = FSDirectory.NormalizePath(Sharpen.Runtime.Substring(src, 0, src.
                                                                                 Length - HdfsConstants.DotSnapshotDir.Length));
            INode          node              = fsd.GetINode(dirPath);
            INodeDirectory dirNode           = INodeDirectory.ValueOf(node, dirPath);
            DirectorySnapshottableFeature sf = dirNode.GetDirectorySnapshottableFeature();

            if (sf == null)
            {
                throw new SnapshotException("Directory is not a snapshottable directory: " + dirPath
                                            );
            }
            ReadOnlyList <Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot> snapshots =
                sf.GetSnapshotList();
            int skipSize = ReadOnlyList.Util.BinarySearch(snapshots, startAfter);

            skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1;
            int numOfListing = Math.Min(snapshots.Size() - skipSize, fsd.GetLsLimit());

            HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing];
            for (int i = 0; i < numOfListing; i++)
            {
                Snapshot.Root sRoot = snapshots.Get(i + skipSize).GetRoot();
                listing[i] = CreateFileStatus(fsd, src, sRoot.GetLocalNameBytes(), sRoot, BlockStoragePolicySuite
                                              .IdUnspecified, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId
                                              , false, INodesInPath.FromINode(sRoot));
            }
            return(new DirectoryListing(listing, snapshots.Size() - skipSize - numOfListing));
        }
Ejemplo n.º 10
0
        /// <summary>The new rename which has the POSIX semantic.</summary>
        /// <exception cref="System.IO.IOException"/>
        internal static KeyValuePair <INode.BlocksMapUpdateInfo, HdfsFileStatus> RenameToInt
            (FSDirectory fsd, string srcArg, string dstArg, bool logRetryCache, params Options.Rename
            [] options)
        {
            string src = srcArg;
            string dst = dstArg;

            if (NameNode.stateChangeLog.IsDebugEnabled())
            {
                NameNode.stateChangeLog.Debug("DIR* NameSystem.renameTo: with options -" + " " +
                                              src + " to " + dst);
            }
            if (!DFSUtil.IsValidName(dst))
            {
                throw new InvalidPathException("Invalid name: " + dst);
            }
            FSPermissionChecker pc = fsd.GetPermissionChecker();

            byte[][] srcComponents = FSDirectory.GetPathComponentsForReservedPath(src);
            byte[][] dstComponents = FSDirectory.GetPathComponentsForReservedPath(dst);
            INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
            src = fsd.ResolvePath(pc, src, srcComponents);
            dst = fsd.ResolvePath(pc, dst, dstComponents);
            RenameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
            INodesInPath   dstIIP        = fsd.GetINodesInPath(dst, false);
            HdfsFileStatus resultingStat = fsd.GetAuditFileInfo(dstIIP);

            return(new AbstractMap.SimpleImmutableEntry <INode.BlocksMapUpdateInfo, HdfsFileStatus
                                                         >(collectedBlocks, resultingStat));
        }
Ejemplo n.º 11
0
        /// <exception cref="System.Exception"/>
        public virtual void TestWrite()
        {
            HdfsFileStatus status = nn.GetRpcServer().GetFileInfo("/tmp/bar");
            long           dirId  = status.GetFileId();
            FileHandle     handle = new FileHandle(dirId);

            byte[] buffer = new byte[10];
            for (int i = 0; i < 10; i++)
            {
                buffer[i] = unchecked ((byte)i);
            }
            WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, Nfs3Constant.WriteStableHow
                                                       .DataSync, ByteBuffer.Wrap(buffer));
            XDR xdr_req = new XDR();

            writeReq.Serialize(xdr_req);
            // Attempt by an unpriviledged user should fail.
            WRITE3Response response1 = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandlerUnpriviledged
                                                  , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces
                                            , response1.GetStatus());
            // Attempt by a priviledged user should pass.
            WRITE3Response response2 = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandler
                                                  , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect response:", null, response2);
        }
Ejemplo n.º 12
0
        /// <exception cref="System.Exception"/>
        public virtual void TestReadlink()
        {
            // Create a symlink first.
            HdfsFileStatus  status  = nn.GetRpcServer().GetFileInfo(testdir);
            long            dirId   = status.GetFileId();
            XDR             xdr_req = new XDR();
            FileHandle      handle  = new FileHandle(dirId);
            SYMLINK3Request req     = new SYMLINK3Request(handle, "fubar", new SetAttr3(), "bar");

            req.Serialize(xdr_req);
            SYMLINK3Response response = nfsd.Symlink(xdr_req.AsReadOnlyWrap(), securityHandler
                                                     , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3Ok, response
                                            .GetStatus());
            // Now perform readlink operations.
            FileHandle       handle2  = response.GetObjFileHandle();
            XDR              xdr_req2 = new XDR();
            READLINK3Request req2     = new READLINK3Request(handle2);

            req2.Serialize(xdr_req2);
            // Attempt by an unpriviledged user should fail.
            READLINK3Response response1 = nfsd.Readlink(xdr_req2.AsReadOnlyWrap(), securityHandlerUnpriviledged
                                                        , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces
                                            , response1.GetStatus());
            // Attempt by a priviledged user should pass.
            READLINK3Response response2 = nfsd.Readlink(xdr_req2.AsReadOnlyWrap(), securityHandler
                                                        , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3Ok, response2
                                            .GetStatus());
        }
Ejemplo n.º 13
0
        public virtual void TestReaddirPlus()
        {
            // Test readdirplus
            // Get inodeId of /tmp
            HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(testdir);
            long           dirId  = status.GetFileId();
            // Create related part of the XDR request
            XDR        xdr_req = new XDR();
            FileHandle handle  = new FileHandle(dirId);

            handle.Serialize(xdr_req);
            xdr_req.WriteLongAsHyper(0);
            // cookie
            xdr_req.WriteLongAsHyper(0);
            // verifier
            xdr_req.WriteInt(100);
            // dirCount
            xdr_req.WriteInt(1000);
            // maxCount
            READDIRPLUS3Response responsePlus = nfsd.Readdirplus(xdr_req.AsReadOnlyWrap(), securityHandler
                                                                 , new IPEndPoint("localhost", 1234));
            IList <READDIRPLUS3Response.EntryPlus3> direntPlus = responsePlus.GetDirListPlus()
                                                                 .GetEntries();

            NUnit.Framework.Assert.IsTrue(direntPlus.Count == 5);
            // including dot, dotdot
            // Test start listing from f2
            status = nn.GetRpcServer().GetFileInfo(testdir + "/f2");
            long f2Id = status.GetFileId();

            // Create related part of the XDR request
            xdr_req = new XDR();
            handle  = new FileHandle(dirId);
            handle.Serialize(xdr_req);
            xdr_req.WriteLongAsHyper(f2Id);
            // cookie
            xdr_req.WriteLongAsHyper(0);
            // verifier
            xdr_req.WriteInt(100);
            // dirCount
            xdr_req.WriteInt(1000);
            // maxCount
            responsePlus = nfsd.Readdirplus(xdr_req.AsReadOnlyWrap(), securityHandler, new IPEndPoint
                                                ("localhost", 1234));
            direntPlus = responsePlus.GetDirListPlus().GetEntries();
            NUnit.Framework.Assert.IsTrue(direntPlus.Count == 1);
            READDIRPLUS3Response.EntryPlus3 entryPlus = direntPlus[0];
            NUnit.Framework.Assert.IsTrue(entryPlus.GetName().Equals("f3"));
            // When the cookie is deleted, list starts over no including dot, dotdot
            hdfs.Delete(new Path(testdir + "/f2"), false);
            responsePlus = nfsd.Readdirplus(xdr_req.AsReadOnlyWrap(), securityHandler, new IPEndPoint
                                                ("localhost", 1234));
            direntPlus = responsePlus.GetDirListPlus().GetEntries();
            NUnit.Framework.Assert.IsTrue(direntPlus.Count == 2);
        }
Ejemplo n.º 14
0
 /// <summary>Select a datanode to service this request.</summary>
 /// <remarks>
 /// Select a datanode to service this request.
 /// Currently, this looks at no more than the first five blocks of a file,
 /// selecting a datanode randomly from the most represented.
 /// </remarks>
 /// <param name="conf"></param>
 /// <exception cref="System.IO.IOException"/>
 private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration
                                    conf)
 {
     if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0)
     {
         // pick a random datanode
         NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
         return(NamenodeJspHelper.GetRandomDatanode(nn));
     }
     return(JspHelper.BestNode(blks, conf));
 }
Ejemplo n.º 15
0
        public HdfsFileStatus HdfsFileStatus()
        {
            HdfsFileStatus status = new HdfsFileStatus
                                    (
                Name: pathSuffix,
                Owner: owner,
                FileType: type.HdfsFileType(),
                Permission: permission
                                    );

            return(status);
        }
Ejemplo n.º 16
0
        public virtual void TestPolicyPropagation()
        {
            StartUpCluster(false, -1);
            string MethodName = GenericTestUtils.GetMethodName();
            Path   path       = new Path("/" + MethodName + ".dat");

            MakeTestFile(path, 0, true);
            // Stat the file and check that the lazyPersist flag is returned back.
            HdfsFileStatus status = client.GetFileInfo(path.ToString());

            Assert.AssertThat(status.GetStoragePolicy(), IS.Is(LazyPersistPolicyId));
        }
Ejemplo n.º 17
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
        public override FileStatus GetFileStatus(Path f)
        {
            HdfsFileStatus fi = dfs.GetFileInfo(GetUriPath(f));

            if (fi != null)
            {
                return(fi.MakeQualified(GetUri(), f));
            }
            else
            {
                throw new FileNotFoundException("File does not exist: " + f.ToString());
            }
        }
Ejemplo n.º 18
0
        public virtual void TestPolicyNotSetByDefault()
        {
            StartUpCluster(false, -1);
            string MethodName = GenericTestUtils.GetMethodName();
            Path   path       = new Path("/" + MethodName + ".dat");

            MakeTestFile(path, 0, false);
            // Stat the file and check that the LAZY_PERSIST policy is not
            // returned back.
            HdfsFileStatus status = client.GetFileInfo(path.ToString());

            Assert.AssertThat(status.GetStoragePolicy(), IsNot.Not(LazyPersistPolicyId));
        }
Ejemplo n.º 19
0
        /// <exception cref="System.IO.IOException"/>
        public static WccAttr GetWccAttr(DFSClient client, string fileIdPath)
        {
            HdfsFileStatus fstat = GetFileStatus(client, fileIdPath);

            if (fstat == null)
            {
                return(null);
            }
            long size = fstat.IsDir() ? GetDirSize(fstat.GetChildrenNum()) : fstat.GetLen();

            return(new WccAttr(size, new NfsTime(fstat.GetModificationTime()), new NfsTime(fstat
                                                                                           .GetModificationTime())));
        }
Ejemplo n.º 20
0
        public override XDR Mnt(XDR xdr, XDR @out, int xid, IPAddress client)
        {
            if (hostsMatcher == null)
            {
                return(MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errAcces, @out, xid, null));
            }
            AccessPrivilege accessPrivilege = hostsMatcher.GetAccessPrivilege(client);

            if (accessPrivilege == AccessPrivilege.None)
            {
                return(MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errAcces, @out, xid, null));
            }
            string path = xdr.ReadString();

            if (Log.IsDebugEnabled())
            {
                Log.Debug("MOUNT MNT path: " + path + " client: " + client);
            }
            string host = client.GetHostName();

            if (Log.IsDebugEnabled())
            {
                Log.Debug("Got host: " + host + " path: " + path);
            }
            if (!exports.Contains(path))
            {
                Log.Info("Path " + path + " is not shared.");
                MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errNoent, @out, xid, null);
                return(@out);
            }
            FileHandle handle = null;

            try
            {
                HdfsFileStatus exFileStatus = dfsClient.GetFileInfo(path);
                handle = new FileHandle(exFileStatus.GetFileId());
            }
            catch (IOException e)
            {
                Log.Error("Can't get handle for export:" + path, e);
                MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errNoent, @out, xid, null);
                return(@out);
            }
            System.Diagnostics.Debug.Assert((handle != null));
            Log.Info("Giving handle (fileId:" + handle.GetFileId() + ") to client for export "
                     + path);
            mounts.AddItem(new MountEntry(host, path));
            MountResponse.WriteMNTResponse(Nfs3Status.Nfs3Ok, @out, xid, handle.GetContent());
            return(@out);
        }
Ejemplo n.º 21
0
        public static Nfs3FileAttributes GetNfs3FileAttrFromFileStatus(HdfsFileStatus fs,
                                                                       IdMappingServiceProvider iug)
        {
            NfsFileType fileType = fs.IsDir() ? NfsFileType.Nfsdir : NfsFileType.Nfsreg;

            fileType = fs.IsSymlink() ? NfsFileType.Nfslnk : fileType;
            int  nlink = (fileType == NfsFileType.Nfsdir) ? fs.GetChildrenNum() + 2 : 1;
            long size  = (fileType == NfsFileType.Nfsdir) ? GetDirSize(fs.GetChildrenNum()) :
                         fs.GetLen();

            return(new Nfs3FileAttributes(fileType, nlink, fs.GetPermission().ToShort(), iug.
                                          GetUidAllowingUnknown(fs.GetOwner()), iug.GetGidAllowingUnknown(fs.GetGroup()),
                                          size, 0, fs.GetFileId(), fs.GetModificationTime(), fs.GetAccessTime(), new Nfs3FileAttributes.Specdata3
                                              ()));
        }
Ejemplo n.º 22
0
        /// <exception cref="System.Exception"/>
        private void CreateFileUsingNfs(string fileName, byte[] buffer)
        {
            DFSTestUtil.CreateFile(hdfs, new Path(fileName), 0, (short)1, 0);
            HdfsFileStatus status   = nn.GetRpcServer().GetFileInfo(fileName);
            long           dirId    = status.GetFileId();
            FileHandle     handle   = new FileHandle(dirId);
            WRITE3Request  writeReq = new WRITE3Request(handle, 0, buffer.Length, Nfs3Constant.WriteStableHow
                                                        .DataSync, ByteBuffer.Wrap(buffer));
            XDR xdr_req = new XDR();

            writeReq.Serialize(xdr_req);
            WRITE3Response response = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandler
                                                 , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect response: ", null, response);
        }
Ejemplo n.º 23
0
        /// <exception cref="System.Exception"/>
        private void Commit(string fileName, int len)
        {
            HdfsFileStatus status  = nn.GetRpcServer().GetFileInfo(fileName);
            long           dirId   = status.GetFileId();
            FileHandle     handle  = new FileHandle(dirId);
            XDR            xdr_req = new XDR();
            COMMIT3Request req     = new COMMIT3Request(handle, 0, len);

            req.Serialize(xdr_req);
            Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel
                                                                           >();
            COMMIT3Response response2 = nfsd.Commit(xdr_req.AsReadOnlyWrap(), ch, 1, securityHandler
                                                    , new IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect COMMIT3Response:", null, response2);
        }
Ejemplo n.º 24
0
        /// <summary>Convert a HdfsFileStatus object to a Json string.</summary>
        public static string ToJsonString(HdfsFileStatus status, bool includeType)
        {
            if (status == null)
            {
                return(null);
            }
            IDictionary <string, object> m = new SortedDictionary <string, object>();

            m["pathSuffix"] = status.GetLocalName();
            m["type"]       = JsonUtil.PathType.ValueOf(status);
            if (status.IsSymlink())
            {
                m["symlink"] = status.GetSymlink();
            }
            m["length"] = status.GetLen();
            m["owner"]  = status.GetOwner();
            m["group"]  = status.GetGroup();
            FsPermission perm = status.GetPermission();

            m["permission"] = ToString(perm);
            if (perm.GetAclBit())
            {
                m["aclBit"] = true;
            }
            if (perm.GetEncryptedBit())
            {
                m["encBit"] = true;
            }
            m["accessTime"]       = status.GetAccessTime();
            m["modificationTime"] = status.GetModificationTime();
            m["blockSize"]        = status.GetBlockSize();
            m["replication"]      = status.GetReplication();
            m["fileId"]           = status.GetFileId();
            m["childrenNum"]      = status.GetChildrenNum();
            m["storagePolicy"]    = status.GetStoragePolicy();
            ObjectMapper mapper = new ObjectMapper();

            try
            {
                return(includeType ? ToJsonString(typeof(FileStatus), m) : mapper.WriteValueAsString
                           (m));
            }
            catch (IOException)
            {
            }
            return(null);
        }
Ejemplo n.º 25
0
        public virtual void TestPolicyPersistenceInFsImage()
        {
            StartUpCluster(false, -1);
            string MethodName = GenericTestUtils.GetMethodName();
            Path   path       = new Path("/" + MethodName + ".dat");

            MakeTestFile(path, 0, true);
            // checkpoint
            fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
            fs.SaveNamespace();
            fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
            cluster.RestartNameNode(true);
            // Stat the file and check that the lazyPersist flag is returned back.
            HdfsFileStatus status = client.GetFileInfo(path.ToString());

            Assert.AssertThat(status.GetStoragePolicy(), IS.Is(LazyPersistPolicyId));
        }
Ejemplo n.º 26
0
        public void GetFileStatus_AddRandomFileWithRandomContent_ResultFileStatus()
        {
            //Assign
            WebHdfsClient client       = GetWebHdfsClient();
            string        fileName     = Guid.NewGuid().ToString();
            String        fileNamePath = $"/dewey/tests/{fileName}";
            string        text         = $"{Guid.NewGuid()}";

            Stream textStream  = GenerateStreamFromString(text);
            bool   resultWrite = client.WriteStream(textStream, fileNamePath).Result;
            //Act
            HdfsFileStatus fileStatus = client.GetFileStatus(fileNamePath).Result;

            //Assert
            Assert.IsTrue(resultWrite);
            Assert.AreEqual(HdfsFileType.FILE, fileStatus.FileType);
        }
Ejemplo n.º 27
0
        /// <exception cref="System.Exception"/>
        private byte[] GetFileContentsUsingNfs(string fileName, int len)
        {
            HdfsFileStatus status  = nn.GetRpcServer().GetFileInfo(fileName);
            long           dirId   = status.GetFileId();
            FileHandle     handle  = new FileHandle(dirId);
            READ3Request   readReq = new READ3Request(handle, 0, len);
            XDR            xdr_req = new XDR();

            readReq.Serialize(xdr_req);
            READ3Response response = nfsd.Read(xdr_req.AsReadOnlyWrap(), securityHandler, new
                                               IPEndPoint("localhost", 1234));

            NUnit.Framework.Assert.AreEqual("Incorrect return code: ", Nfs3Status.Nfs3Ok, response
                                            .GetStatus());
            NUnit.Framework.Assert.IsTrue("expected full read", response.IsEof());
            return((byte[])response.GetData().Array());
        }
Ejemplo n.º 28
0
            /// <exception cref="System.IO.IOException"/>
            public virtual int Run(Configuration conf, IList <string> args)
            {
                string path = StringUtils.PopOptionWithArgument("-path", args);

                if (path == null)
                {
                    System.Console.Error.WriteLine("Please specify the path with -path.\nUsage:" + GetLongUsage
                                                       ());
                    return(1);
                }
                DistributedFileSystem dfs = AdminHelper.GetDFS(conf);

                try
                {
                    HdfsFileStatus status = dfs.GetClient().GetFileInfo(path);
                    if (status == null)
                    {
                        System.Console.Error.WriteLine("File/Directory does not exist: " + path);
                        return(2);
                    }
                    byte storagePolicyId = status.GetStoragePolicy();
                    if (storagePolicyId == BlockStoragePolicySuite.IdUnspecified)
                    {
                        System.Console.Out.WriteLine("The storage policy of " + path + " is unspecified");
                        return(0);
                    }
                    BlockStoragePolicy[] policies = dfs.GetStoragePolicies();
                    foreach (BlockStoragePolicy p in policies)
                    {
                        if (p.GetId() == storagePolicyId)
                        {
                            System.Console.Out.WriteLine("The storage policy of " + path + ":\n" + p);
                            return(0);
                        }
                    }
                }
                catch (Exception e)
                {
                    System.Console.Error.WriteLine(AdminHelper.PrettifyException(e));
                    return(2);
                }
                System.Console.Error.WriteLine("Cannot identify the storage policy for " + path);
                return(2);
            }
Ejemplo n.º 29
0
            /// <returns>whether the migration requires next round</returns>
            private bool ProcessRecursively(string parent, HdfsFileStatus status)
            {
                string fullPath     = status.GetFullName(parent);
                bool   hasRemaining = false;

                if (status.IsDir())
                {
                    if (!fullPath.EndsWith(Path.Separator))
                    {
                        fullPath = fullPath + Path.Separator;
                    }
                    hasRemaining = this.ProcessPath(fullPath);
                    // process snapshots if this is a snapshottable directory
                    if (this.snapshottableDirs.Contains(fullPath))
                    {
                        string dirSnapshot = fullPath + HdfsConstants.DotSnapshotDir;
                        hasRemaining |= this.ProcessPath(dirSnapshot);
                    }
                }
                else
                {
                    if (!status.IsSymlink())
                    {
                        // file
                        try
                        {
                            if (!this.IsSnapshotPathInCurrent(fullPath))
                            {
                                // the full path is a snapshot path but it is also included in the
                                // current directory tree, thus ignore it.
                                hasRemaining = this.ProcessFile(fullPath, (HdfsLocatedFileStatus)status);
                            }
                        }
                        catch (IOException e)
                        {
                            Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to check the status of "
                                                                               + parent + ". Ignore it and continue.", e);
                            return(false);
                        }
                    }
                }
                return(hasRemaining);
            }
Ejemplo n.º 30
0
        public void MakeDirectory_MakerRandomDirectory_ResultListStatusDirectory()
        {
            //Assign
            WebHdfsClient client            = GetWebHdfsClient();
            string        directoryName     = Guid.NewGuid().ToString();
            string        rootPath          = "/dewey/tests";
            String        directoryNamePath = $"{rootPath}/{directoryName}";


            //Act
            bool resultMakeDirectory = client.MakeDirectory(directoryNamePath).Result;

            //Assert
            Assert.IsTrue(resultMakeDirectory);
            IEnumerable <HdfsFileStatus> fileStatus = client.ListStatus(rootPath).Result;

            HdfsFileStatus findStatus =
                fileStatus.SingleOrDefault(a => a.FileType == HdfsFileType.DIRECTORY && a.Name == directoryName);

            Assert.IsNotNull(findStatus);
        }