Пример #1
0
 internal static FileStatus ToFileStatus(HdfsFileStatus f, string parent)
 {
     return(new FileStatus(f.GetLen(), f.IsDir(), f.GetReplication(), f.GetBlockSize()
                           , f.GetModificationTime(), f.GetAccessTime(), f.GetPermission(), f.GetOwner(), f
                           .GetGroup(), f.IsSymlink() ? new Path(f.GetSymlink()) : null, new Path(f.GetFullName
                                                                                                      (parent))));
 }
Пример #2
0
            /// <exception cref="System.IO.IOException"/>
            public Void Run()
            {
                ClientProtocol nn              = this._enclosing.CreateNameNodeProxy();
                string         path            = ServletUtil.GetDecodedPath(request, "/data");
                string         encodedPath     = ServletUtil.GetRawPath(request, "/data");
                string         delegationToken = request.GetParameter(JspHelper.DelegationParameterName);
                HdfsFileStatus info            = nn.GetFileInfo(path);

                if (info != null && !info.IsDir())
                {
                    response.SendRedirect(this._enclosing.CreateRedirectURL(path, encodedPath, info,
                                                                            ugi, nn, request, delegationToken).ToString());
                }
                else
                {
                    if (info == null)
                    {
                        response.SendError(400, "File not found " + path);
                    }
                    else
                    {
                        response.SendError(400, path + ": is a directory");
                    }
                }
                return(null);
            }
Пример #3
0
        /// <summary>Write a node to output.</summary>
        /// <remarks>
        /// Write a node to output.
        /// Node information includes path, modification, permission, owner and group.
        /// For files, it also includes size, replication and block-size.
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        internal static void WriteInfo(Path fullpath, HdfsFileStatus i, XMLOutputter doc)
        {
            SimpleDateFormat ldf = df.Get();

            doc.StartTag(i.IsDir() ? "directory" : "file");
            doc.Attribute("path", fullpath.ToUri().GetPath());
            doc.Attribute("modified", ldf.Format(Sharpen.Extensions.CreateDate(i.GetModificationTime
                                                                                   ())));
            doc.Attribute("accesstime", ldf.Format(Sharpen.Extensions.CreateDate(i.GetAccessTime
                                                                                     ())));
            if (!i.IsDir())
            {
                doc.Attribute("size", i.GetLen().ToString());
                doc.Attribute("replication", i.GetReplication().ToString());
                doc.Attribute("blocksize", i.GetBlockSize().ToString());
            }
            doc.Attribute("permission", (i.IsDir() ? "d" : "-") + i.GetPermission());
            doc.Attribute("owner", i.GetOwner());
            doc.Attribute("group", i.GetGroup());
            doc.EndTag();
        }
Пример #4
0
        /// <exception cref="System.IO.IOException"/>
        public static WccAttr GetWccAttr(DFSClient client, string fileIdPath)
        {
            HdfsFileStatus fstat = GetFileStatus(client, fileIdPath);

            if (fstat == null)
            {
                return(null);
            }
            long size = fstat.IsDir() ? GetDirSize(fstat.GetChildrenNum()) : fstat.GetLen();

            return(new WccAttr(size, new NfsTime(fstat.GetModificationTime()), new NfsTime(fstat
                                                                                           .GetModificationTime())));
        }
Пример #5
0
        public static Nfs3FileAttributes GetNfs3FileAttrFromFileStatus(HdfsFileStatus fs,
                                                                       IdMappingServiceProvider iug)
        {
            NfsFileType fileType = fs.IsDir() ? NfsFileType.Nfsdir : NfsFileType.Nfsreg;

            fileType = fs.IsSymlink() ? NfsFileType.Nfslnk : fileType;
            int  nlink = (fileType == NfsFileType.Nfsdir) ? fs.GetChildrenNum() + 2 : 1;
            long size  = (fileType == NfsFileType.Nfsdir) ? GetDirSize(fs.GetChildrenNum()) :
                         fs.GetLen();

            return(new Nfs3FileAttributes(fileType, nlink, fs.GetPermission().ToShort(), iug.
                                          GetUidAllowingUnknown(fs.GetOwner()), iug.GetGidAllowingUnknown(fs.GetGroup()),
                                          size, 0, fs.GetFileId(), fs.GetModificationTime(), fs.GetAccessTime(), new Nfs3FileAttributes.Specdata3
                                              ()));
        }
Пример #6
0
            /// <returns>whether the migration requires next round</returns>
            private bool ProcessRecursively(string parent, HdfsFileStatus status)
            {
                string fullPath     = status.GetFullName(parent);
                bool   hasRemaining = false;

                if (status.IsDir())
                {
                    if (!fullPath.EndsWith(Path.Separator))
                    {
                        fullPath = fullPath + Path.Separator;
                    }
                    hasRemaining = this.ProcessPath(fullPath);
                    // process snapshots if this is a snapshottable directory
                    if (this.snapshottableDirs.Contains(fullPath))
                    {
                        string dirSnapshot = fullPath + HdfsConstants.DotSnapshotDir;
                        hasRemaining |= this.ProcessPath(dirSnapshot);
                    }
                }
                else
                {
                    if (!status.IsSymlink())
                    {
                        // file
                        try
                        {
                            if (!this.IsSnapshotPathInCurrent(fullPath))
                            {
                                // the full path is a snapshot path but it is also included in the
                                // current directory tree, thus ignore it.
                                hasRemaining = this.ProcessFile(fullPath, (HdfsLocatedFileStatus)status);
                            }
                        }
                        catch (IOException e)
                        {
                            Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to check the status of "
                                                                               + parent + ". Ignore it and continue.", e);
                            return(false);
                        }
                    }
                }
                return(hasRemaining);
            }
Пример #7
0
 /// <exception cref="System.Exception"/>
 private void VerifyRecursively(Path parent, HdfsFileStatus status)
 {
     if (status.IsDir())
     {
         Path             fullPath = parent == null ? new Path("/") : status.GetFullPath(parent);
         DirectoryListing children = this.dfs.GetClient().ListPaths(fullPath.ToString(), HdfsFileStatus
                                                                    .EmptyName, true);
         foreach (HdfsFileStatus child in children.GetPartialListing())
         {
             this.VerifyRecursively(fullPath, child);
         }
     }
     else
     {
         if (!status.IsSymlink())
         {
             // is file
             this.VerifyFile(parent, status, null);
         }
     }
 }
        /// <exception cref="System.IO.IOException"/>
        internal static void RecoverAllLeases(DFSClient dfs, Path path)
        {
            string         pathStr = path.ToString();
            HdfsFileStatus status  = dfs.GetFileInfo(pathStr);

            if (!status.IsDir())
            {
                dfs.RecoverLease(pathStr);
                return;
            }
            byte[]           prev = HdfsFileStatus.EmptyName;
            DirectoryListing dirList;

            do
            {
                dirList = dfs.ListPaths(pathStr, prev);
                HdfsFileStatus[] files = dirList.GetPartialListing();
                foreach (HdfsFileStatus f in files)
                {
                    RecoverAllLeases(dfs, f.GetFullPath(path));
                }
                prev = dirList.GetLastName();
            }while (dirList.HasMore());
        }
Пример #9
0
            /// <exception cref="System.IO.IOException"/>
            public Void Run()
            {
                ClientProtocol nn = this._enclosing.CreateNameNodeProxy();

                doc.Declaration();
                doc.StartTag("listing");
                foreach (KeyValuePair <string, string> m in root)
                {
                    doc.Attribute(m.Key, m.Value);
                }
                HdfsFileStatus @base = nn.GetFileInfo(filePath);

                if ((@base != null) && @base.IsDir())
                {
                    ListPathsServlet.WriteInfo(@base.GetFullPath(new Path(path)), @base, doc);
                }
                Stack <string> pathstack = new Stack <string>();

                pathstack.Push(path);
                while (!pathstack.Empty())
                {
                    string p = pathstack.Pop();
                    try
                    {
                        byte[]           lastReturnedName = HdfsFileStatus.EmptyName;
                        DirectoryListing thisListing;
                        do
                        {
                            System.Diagnostics.Debug.Assert(lastReturnedName != null);
                            thisListing = nn.GetListing(p, lastReturnedName, false);
                            if (thisListing == null)
                            {
                                if (lastReturnedName.Length == 0)
                                {
                                    DfsServlet.Log.Warn("ListPathsServlet - Path " + p + " does not exist");
                                }
                                break;
                            }
                            HdfsFileStatus[] listing = thisListing.GetPartialListing();
                            foreach (HdfsFileStatus i in listing)
                            {
                                Path   fullpath  = i.GetFullPath(new Path(p));
                                string localName = fullpath.GetName();
                                if (exclude.Matcher(localName).Matches() || !filter.Matcher(localName).Matches())
                                {
                                    continue;
                                }
                                if (recur && i.IsDir())
                                {
                                    pathstack.Push(new Path(p, localName).ToUri().GetPath());
                                }
                                ListPathsServlet.WriteInfo(fullpath, i, doc);
                            }
                            lastReturnedName = thisListing.GetLastName();
                        }while (thisListing.HasMore());
                    }
                    catch (IOException re)
                    {
                        this._enclosing.WriteXml(re, p, doc);
                    }
                }
                return(null);
            }
Пример #10
0
 internal static JsonUtil.PathType ValueOf(HdfsFileStatus status)
 {
     return(status.IsDir() ? JsonUtil.PathType.Directory : status.IsSymlink() ? JsonUtil.PathType
            .Symlink : JsonUtil.PathType.File);
 }