private void DumpINodeFile(FsImageProto.INodeSection.INodeFile f)
{
O("replication", f.GetReplication()).O("mtime", f.GetModificationTime()).O("atime"
, f.GetAccessTime()).O("perferredBlockSize", f.GetPreferredBlockSize()).O("permission"
, DumpPermission(f.GetPermission()));
if (f.GetBlocksCount() > 0)
{
@out.Write("<blocks>");
foreach (HdfsProtos.BlockProto b in f.GetBlocksList())
{
@out.Write("<block>");
O("id", b.GetBlockId()).O("genstamp", b.GetGenStamp()).O("numBytes", b.GetNumBytes
());
@out.Write("</block>\n");
}
@out.Write("</blocks>\n");
}
if (f.HasFileUC())
{
FsImageProto.INodeSection.FileUnderConstructionFeature u = f.GetFileUC();
@out.Write("<file-under-construction>");
O("clientName", u.GetClientName()).O("clientMachine", u.GetClientMachine());
@out.Write("</file-under-construction>\n");
}
}
/// <exception cref="System.IO.IOException"/>
private PermissionStatus GetPermissionStatus(string path)
{
long id = Lookup(path);
FsImageProto.INodeSection.INode inode = FromINodeId(id);
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile f = inode.GetFile();
return(FSImageFormatPBINode.Loader.LoadPermission(f.GetPermission(), stringTable));
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory d = inode.GetDirectory();
return(FSImageFormatPBINode.Loader.LoadPermission(d.GetPermission(), stringTable));
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
FsImageProto.INodeSection.INodeSymlink s = inode.GetSymlink();
return(FSImageFormatPBINode.Loader.LoadPermission(s.GetPermission(), stringTable));
}
default:
{
return(null);
}
}
}
/// <exception cref="System.IO.IOException"/>
private IList <AclEntry> GetAclEntryList(string path)
{
long id = Lookup(path);
FsImageProto.INodeSection.INode inode = FromINodeId(id);
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile f = inode.GetFile();
return(FSImageFormatPBINode.Loader.LoadAclEntries(f.GetAcl(), stringTable));
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory d = inode.GetDirectory();
return(FSImageFormatPBINode.Loader.LoadAclEntries(d.GetAcl(), stringTable));
}
default:
{
return(new AList <AclEntry>());
}
}
}
internal static long GetFileSize(FsImageProto.INodeSection.INodeFile f)
{
long size = 0;
foreach (HdfsProtos.BlockProto p in f.GetBlocksList())
{
size += p.GetNumBytes();
}
return(size);
}
/// <summary>Load FileDiff list for a file with snapshot feature</summary>
/// <exception cref="System.IO.IOException"/>
private void LoadFileDiffList(InputStream @in, INodeFile file, int size)
{
FileDiffList diffs = new FileDiffList();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
for (int i = 0; i < size; i++)
{
FsImageProto.SnapshotDiffSection.FileDiff pbf = FsImageProto.SnapshotDiffSection.FileDiff
.ParseDelimitedFrom(@in);
INodeFileAttributes copy = null;
if (pbf.HasSnapshotCopy())
{
FsImageProto.INodeSection.INodeFile fileInPb = pbf.GetSnapshotCopy();
PermissionStatus permission = FSImageFormatPBINode.Loader.LoadPermission(fileInPb
.GetPermission(), state.GetStringTable());
AclFeature acl = null;
if (fileInPb.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(FSImageFormatPBINode.Loader.LoadAclEntries
(fileInPb.GetAcl(), state.GetStringTable()));
acl = new AclFeature(entries);
}
XAttrFeature xAttrs = null;
if (fileInPb.HasXAttrs())
{
xAttrs = new XAttrFeature(FSImageFormatPBINode.Loader.LoadXAttrs(fileInPb.GetXAttrs
(), state.GetStringTable()));
}
copy = new INodeFileAttributes.SnapshotCopy(pbf.GetName().ToByteArray(), permission
, acl, fileInPb.GetModificationTime(), fileInPb.GetAccessTime(), (short)fileInPb
.GetReplication(), fileInPb.GetPreferredBlockSize(), unchecked ((byte)fileInPb.GetStoragePolicyID
()), xAttrs);
}
FileDiff diff = new FileDiff(pbf.GetSnapshotId(), copy, null, pbf.GetFileSize());
IList <HdfsProtos.BlockProto> bpl = pbf.GetBlocksList();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.Count];
for (int j = 0; j < e; ++j)
{
Block blk = PBHelper.Convert(bpl[j]);
BlockInfoContiguous storedBlock = fsn.GetBlockManager().GetStoredBlock(blk);
if (storedBlock == null)
{
storedBlock = fsn.GetBlockManager().AddBlockCollection(new BlockInfoContiguous(blk
, copy.GetFileReplication()), file);
}
blocks[j] = storedBlock;
}
if (blocks.Length > 0)
{
diff.SetBlocks(blocks);
}
diffs.AddFirst(diff);
}
file.AddSnapshotFeature(diffs);
}
private INodeFile LoadINodeFile(FsImageProto.INodeSection.INode n)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.File);
FsImageProto.INodeSection.INodeFile f = n.GetFile();
IList <HdfsProtos.BlockProto> bp = f.GetBlocksList();
short replication = (short)f.GetReplication();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.Count];
for (int i = 0; i < e; ++i)
{
blocks[i] = new BlockInfoContiguous(PBHelper.Convert(bp[i]), replication);
}
PermissionStatus permissions = LoadPermission(f.GetPermission(), parent.GetLoaderContext
().GetStringTable());
INodeFile file = new INodeFile(n.GetId(), n.GetName().ToByteArray(), permissions,
f.GetModificationTime(), f.GetAccessTime(), blocks, replication, f.GetPreferredBlockSize
(), unchecked ((byte)f.GetStoragePolicyID()));
if (f.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(f.GetAcl(), state.GetStringTable
()));
file.AddAclFeature(new AclFeature(entries));
}
if (f.HasXAttrs())
{
file.AddXAttrFeature(new XAttrFeature(LoadXAttrs(f.GetXAttrs(), state.GetStringTable
())));
}
// under-construction information
if (f.HasFileUC())
{
FsImageProto.INodeSection.FileUnderConstructionFeature uc = f.GetFileUC();
file.ToUnderConstruction(uc.GetClientName(), uc.GetClientMachine());
if (blocks.Length > 0)
{
BlockInfoContiguous lastBlk = file.GetLastBlock();
// replace the last block of file
file.SetBlock(file.NumBlocks() - 1, new BlockInfoContiguousUnderConstruction(lastBlk
, replication));
}
}
return(file);
}
/// <exception cref="System.IO.IOException"/>
private void Run(InputStream @in)
{
FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in);
for (int i = 0; i < s.GetNumInodes(); ++i)
{
FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom
(@in);
if (p.GetType() == FsImageProto.INodeSection.INode.Type.File)
{
++totalFiles;
FsImageProto.INodeSection.INodeFile f = p.GetFile();
totalBlocks += f.GetBlocksCount();
long fileSize = 0;
foreach (HdfsProtos.BlockProto b in f.GetBlocksList())
{
fileSize += b.GetNumBytes();
}
maxFileSize = Math.Max(fileSize, maxFileSize);
totalSpace += fileSize * f.GetReplication();
int bucket = fileSize > maxSize ? distribution.Length - 1 : (int)Math.Ceil((double
)fileSize / steps);
++distribution[bucket];
}
else
{
if (p.GetType() == FsImageProto.INodeSection.INode.Type.Directory)
{
++totalDirectories;
}
}
if (i % (1 << 20) == 0)
{
@out.WriteLine("Processed " + i + " inodes.");
}
}
}
private IDictionary <string, object> GetFileStatus(FsImageProto.INodeSection.INode
inode, bool printSuffix)
{
IDictionary <string, object> map = Maps.NewHashMap();
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile f = inode.GetFile();
PermissionStatus p = FSImageFormatPBINode.Loader.LoadPermission(f.GetPermission()
, stringTable);
map["accessTime"] = f.GetAccessTime();
map["blockSize"] = f.GetPreferredBlockSize();
map["group"] = p.GetGroupName();
map["length"] = GetFileSize(f);
map["modificationTime"] = f.GetModificationTime();
map["owner"] = p.GetUserName();
map["pathSuffix"] = printSuffix ? inode.GetName().ToStringUtf8() : string.Empty;
map["permission"] = ToString(p.GetPermission());
map["replication"] = f.GetReplication();
map["type"] = inode.GetType();
map["fileId"] = inode.GetId();
map["childrenNum"] = 0;
return(map);
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory d = inode.GetDirectory();
PermissionStatus p = FSImageFormatPBINode.Loader.LoadPermission(d.GetPermission()
, stringTable);
map["accessTime"] = 0;
map["blockSize"] = 0;
map["group"] = p.GetGroupName();
map["length"] = 0;
map["modificationTime"] = d.GetModificationTime();
map["owner"] = p.GetUserName();
map["pathSuffix"] = printSuffix ? inode.GetName().ToStringUtf8() : string.Empty;
map["permission"] = ToString(p.GetPermission());
map["replication"] = 0;
map["type"] = inode.GetType();
map["fileId"] = inode.GetId();
map["childrenNum"] = dirmap.Contains(inode.GetId()) ? dirmap[inode.GetId()].Length
: 0;
return(map);
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
FsImageProto.INodeSection.INodeSymlink d = inode.GetSymlink();
PermissionStatus p = FSImageFormatPBINode.Loader.LoadPermission(d.GetPermission()
, stringTable);
map["accessTime"] = d.GetAccessTime();
map["blockSize"] = 0;
map["group"] = p.GetGroupName();
map["length"] = 0;
map["modificationTime"] = d.GetModificationTime();
map["owner"] = p.GetUserName();
map["pathSuffix"] = printSuffix ? inode.GetName().ToStringUtf8() : string.Empty;
map["permission"] = ToString(p.GetPermission());
map["replication"] = 0;
map["type"] = inode.GetType();
map["symlink"] = d.GetTarget().ToStringUtf8();
map["fileId"] = inode.GetId();
map["childrenNum"] = 0;
return(map);
}
default:
{
return(null);
}
}
}
protected internal override string GetEntry(string parent, FsImageProto.INodeSection.INode
inode)
{
StringBuilder buffer = new StringBuilder();
string inodeName = inode.GetName().ToStringUtf8();
Path path = new Path(parent.IsEmpty() ? "/" : parent, inodeName.IsEmpty() ? "/" :
inodeName);
buffer.Append(path.ToString());
PermissionStatus p = null;
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile file = inode.GetFile();
p = GetPermission(file.GetPermission());
Append(buffer, file.GetReplication());
Append(buffer, FormatDate(file.GetModificationTime()));
Append(buffer, FormatDate(file.GetAccessTime()));
Append(buffer, file.GetPreferredBlockSize());
Append(buffer, file.GetBlocksCount());
Append(buffer, FSImageLoader.GetFileSize(file));
Append(buffer, 0);
// NS_QUOTA
Append(buffer, 0);
// DS_QUOTA
break;
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory dir = inode.GetDirectory();
p = GetPermission(dir.GetPermission());
Append(buffer, 0);
// Replication
Append(buffer, FormatDate(dir.GetModificationTime()));
Append(buffer, FormatDate(0));
// Access time.
Append(buffer, 0);
// Block size.
Append(buffer, 0);
// Num blocks.
Append(buffer, 0);
// Num bytes.
Append(buffer, dir.GetNsQuota());
Append(buffer, dir.GetDsQuota());
break;
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
FsImageProto.INodeSection.INodeSymlink s = inode.GetSymlink();
p = GetPermission(s.GetPermission());
Append(buffer, 0);
// Replication
Append(buffer, FormatDate(s.GetModificationTime()));
Append(buffer, FormatDate(s.GetAccessTime()));
Append(buffer, 0);
// Block size.
Append(buffer, 0);
// Num blocks.
Append(buffer, 0);
// Num bytes.
Append(buffer, 0);
// NS_QUOTA
Append(buffer, 0);
// DS_QUOTA
break;
}
default:
{
break;
}
}
System.Diagnostics.Debug.Assert(p != null);
Append(buffer, p.GetPermission().ToString());
Append(buffer, p.GetUserName());
Append(buffer, p.GetGroupName());
return(buffer.ToString());
}
