/// <exception cref="System.IO.IOException"/> private void DumpINodeSection(InputStream @in) { FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in); @out.Write("<INodeSection>"); O("lastInodeId", s.GetLastInodeId()); for (int i = 0; i < s.GetNumInodes(); ++i) { FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom (@in); @out.Write("<inode>"); O("id", p.GetId()).O("type", p.GetType()).O("name", p.GetName().ToStringUtf8()); if (p.HasFile()) { DumpINodeFile(p.GetFile()); } else { if (p.HasDirectory()) { DumpINodeDirectory(p.GetDirectory()); } else { if (p.HasSymlink()) { DumpINodeSymlink(p.GetSymlink()); } } } @out.Write("</inode>\n"); } @out.Write("</INodeSection>\n"); }
/// <exception cref="System.IO.IOException"/> private void OutputINodes(InputStream @in) { FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in); Log.Info("Found {} INodes in the INode section", s.GetNumInodes()); for (int i = 0; i < s.GetNumInodes(); ++i) { FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom (@in); string parentPath = metadataMap.GetParentPath(p.GetId()); @out.WriteLine(GetEntry(parentPath, p)); if (Log.IsDebugEnabled() && i % 100000 == 0) { Log.Debug("Outputted {} INodes.", i); } } Log.Info("Outputted {} INodes.", s.GetNumInodes()); }
/// <exception cref="System.IO.IOException"/> private static byte[][] LoadINodeSection(InputStream @in) { FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in); Log.Info("Loading " + s.GetNumInodes() + " inodes."); byte[][] inodes = new byte[(int)s.GetNumInodes()][]; for (int i = 0; i < s.GetNumInodes(); ++i) { int size = CodedInputStream.ReadRawVarint32(@in.Read(), @in); byte[] bytes = new byte[size]; IOUtils.ReadFully(@in, bytes, 0, size); inodes[i] = bytes; } Log.Debug("Sorting inodes"); Arrays.Sort(inodes, InodeBytesComparator); Log.Debug("Finished sorting inodes"); return(inodes); }
/// <exception cref="System.IO.IOException"/> internal void LoadINodeSection(InputStream @in) { FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in); fsn.dir.ResetLastInodeId(s.GetLastInodeId()); Log.Info("Loading " + s.GetNumInodes() + " INodes."); for (int i = 0; i < s.GetNumInodes(); ++i) { FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom (@in); if (p.GetId() == INodeId.RootInodeId) { LoadRootINode(p); } else { INode n = LoadINode(p); dir.AddToInodeMap(n); } } }
/// <summary>Load the filenames of the directories from the INode section.</summary> /// <exception cref="System.IO.IOException"/> private void LoadDirectoriesInINodeSection(InputStream @in) { FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in); Log.Info("Loading directories in INode section."); int numDirs = 0; for (int i = 0; i < s.GetNumInodes(); ++i) { FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom (@in); if (Log.IsDebugEnabled() && i % 10000 == 0) { Log.Debug("Scanned {} inodes.", i); } if (p.HasDirectory()) { metadataMap.PutDir(p); numDirs++; } } Log.Info("Found {} directories in INode section.", numDirs); }
/// <exception cref="System.IO.IOException"/> internal void SerializeINodeSection(OutputStream @out) { INodeMap inodesMap = fsn.dir.GetINodeMap(); FsImageProto.INodeSection.Builder b = FsImageProto.INodeSection.NewBuilder().SetLastInodeId (fsn.dir.GetLastInodeId()).SetNumInodes(inodesMap.Size()); FsImageProto.INodeSection s = ((FsImageProto.INodeSection)b.Build()); s.WriteDelimitedTo(@out); int i = 0; IEnumerator <INodeWithAdditionalFields> iter = inodesMap.GetMapIterator(); while (iter.HasNext()) { INodeWithAdditionalFields n = iter.Next(); Save(@out, n); ++i; if (i % FSImageFormatProtobuf.Saver.CheckCancelInterval == 0) { context.CheckCancelled(); } } parent.CommitSection(summary, FSImageFormatProtobuf.SectionName.Inode); }
/// <exception cref="System.IO.IOException"/> private void Run(InputStream @in) { FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in); for (int i = 0; i < s.GetNumInodes(); ++i) { FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom (@in); if (p.GetType() == FsImageProto.INodeSection.INode.Type.File) { ++totalFiles; FsImageProto.INodeSection.INodeFile f = p.GetFile(); totalBlocks += f.GetBlocksCount(); long fileSize = 0; foreach (HdfsProtos.BlockProto b in f.GetBlocksList()) { fileSize += b.GetNumBytes(); } maxFileSize = Math.Max(fileSize, maxFileSize); totalSpace += fileSize * f.GetReplication(); int bucket = fileSize > maxSize ? distribution.Length - 1 : (int)Math.Ceil((double )fileSize / steps); ++distribution[bucket]; } else { if (p.GetType() == FsImageProto.INodeSection.INode.Type.Directory) { ++totalDirectories; } } if (i % (1 << 20) == 0) { @out.WriteLine("Processed " + i + " inodes."); } } }