private INode LoadINode(FsImageProto.INodeSection.INode n)
{
switch (n.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
return(LoadINodeFile(n));
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
return(LoadINodeDirectory(n, parent.GetLoaderContext()));
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
return(LoadINodeSymlink(n));
}
default:
{
break;
}
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
private void DumpINodeSection(InputStream @in)
{
FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in);
@out.Write("<INodeSection>");
O("lastInodeId", s.GetLastInodeId());
for (int i = 0; i < s.GetNumInodes(); ++i)
{
FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom
(@in);
@out.Write("<inode>");
O("id", p.GetId()).O("type", p.GetType()).O("name", p.GetName().ToStringUtf8());
if (p.HasFile())
{
DumpINodeFile(p.GetFile());
}
else
{
if (p.HasDirectory())
{
DumpINodeDirectory(p.GetDirectory());
}
else
{
if (p.HasSymlink())
{
DumpINodeSymlink(p.GetSymlink());
}
}
}
@out.Write("</inode>\n");
}
@out.Write("</INodeSection>\n");
}
/// <exception cref="System.IO.IOException"/>
private PermissionStatus GetPermissionStatus(string path)
{
long id = Lookup(path);
FsImageProto.INodeSection.INode inode = FromINodeId(id);
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile f = inode.GetFile();
return(FSImageFormatPBINode.Loader.LoadPermission(f.GetPermission(), stringTable));
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory d = inode.GetDirectory();
return(FSImageFormatPBINode.Loader.LoadPermission(d.GetPermission(), stringTable));
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
FsImageProto.INodeSection.INodeSymlink s = inode.GetSymlink();
return(FSImageFormatPBINode.Loader.LoadPermission(s.GetPermission(), stringTable));
}
default:
{
return(null);
}
}
}
/// <exception cref="System.IO.IOException"/>
private IList <AclEntry> GetAclEntryList(string path)
{
long id = Lookup(path);
FsImageProto.INodeSection.INode inode = FromINodeId(id);
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile f = inode.GetFile();
return(FSImageFormatPBINode.Loader.LoadAclEntries(f.GetAcl(), stringTable));
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory d = inode.GetDirectory();
return(FSImageFormatPBINode.Loader.LoadAclEntries(d.GetAcl(), stringTable));
}
default:
{
return(new AList <AclEntry>());
}
}
}
/// <exception cref="System.IO.IOException"/>
private IList <IDictionary <string, object> > GetFileStatusList(string path)
{
IList <IDictionary <string, object> > list = new AList <IDictionary <string, object> >(
);
long id = Lookup(path);
FsImageProto.INodeSection.INode inode = FromINodeId(id);
if (inode.GetType() == FsImageProto.INodeSection.INode.Type.Directory)
{
if (!dirmap.Contains(id))
{
// if the directory is empty, return empty list
return(list);
}
long[] children = dirmap[id];
foreach (long cid in children)
{
list.AddItem(GetFileStatus(FromINodeId(cid), true));
}
}
else
{
list.AddItem(GetFileStatus(inode, false));
}
return(list);
}
private INodeSymlink LoadINodeSymlink(FsImageProto.INodeSection.INode n)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.Symlink);
FsImageProto.INodeSection.INodeSymlink s = n.GetSymlink();
PermissionStatus permissions = LoadPermission(s.GetPermission(), parent.GetLoaderContext
().GetStringTable());
INodeSymlink sym = new INodeSymlink(n.GetId(), n.GetName().ToByteArray(), permissions
, s.GetModificationTime(), s.GetAccessTime(), s.GetTarget().ToStringUtf8());
return(sym);
}
/// <exception cref="System.IO.IOException"/>
private void Run(InputStream @in)
{
FsImageProto.INodeSection s = FsImageProto.INodeSection.ParseDelimitedFrom(@in);
for (int i = 0; i < s.GetNumInodes(); ++i)
{
FsImageProto.INodeSection.INode p = FsImageProto.INodeSection.INode.ParseDelimitedFrom
(@in);
if (p.GetType() == FsImageProto.INodeSection.INode.Type.File)
{
++totalFiles;
FsImageProto.INodeSection.INodeFile f = p.GetFile();
totalBlocks += f.GetBlocksCount();
long fileSize = 0;
foreach (HdfsProtos.BlockProto b in f.GetBlocksList())
{
fileSize += b.GetNumBytes();
}
maxFileSize = Math.Max(fileSize, maxFileSize);
totalSpace += fileSize * f.GetReplication();
int bucket = fileSize > maxSize ? distribution.Length - 1 : (int)Math.Ceil((double
)fileSize / steps);
++distribution[bucket];
}
else
{
if (p.GetType() == FsImageProto.INodeSection.INode.Type.Directory)
{
++totalDirectories;
}
}
if (i % (1 << 20) == 0)
{
@out.WriteLine("Processed " + i + " inodes.");
}
}
}
private INodeFile LoadINodeFile(FsImageProto.INodeSection.INode n)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.File);
FsImageProto.INodeSection.INodeFile f = n.GetFile();
IList <HdfsProtos.BlockProto> bp = f.GetBlocksList();
short replication = (short)f.GetReplication();
FSImageFormatProtobuf.LoaderContext state = parent.GetLoaderContext();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.Count];
for (int i = 0; i < e; ++i)
{
blocks[i] = new BlockInfoContiguous(PBHelper.Convert(bp[i]), replication);
}
PermissionStatus permissions = LoadPermission(f.GetPermission(), parent.GetLoaderContext
().GetStringTable());
INodeFile file = new INodeFile(n.GetId(), n.GetName().ToByteArray(), permissions,
f.GetModificationTime(), f.GetAccessTime(), blocks, replication, f.GetPreferredBlockSize
(), unchecked ((byte)f.GetStoragePolicyID()));
if (f.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(f.GetAcl(), state.GetStringTable
()));
file.AddAclFeature(new AclFeature(entries));
}
if (f.HasXAttrs())
{
file.AddXAttrFeature(new XAttrFeature(LoadXAttrs(f.GetXAttrs(), state.GetStringTable
())));
}
// under-construction information
if (f.HasFileUC())
{
FsImageProto.INodeSection.FileUnderConstructionFeature uc = f.GetFileUC();
file.ToUnderConstruction(uc.GetClientName(), uc.GetClientMachine());
if (blocks.Length > 0)
{
BlockInfoContiguous lastBlk = file.GetLastBlock();
// replace the last block of file
file.SetBlock(file.NumBlocks() - 1, new BlockInfoContiguousUnderConstruction(lastBlk
, replication));
}
}
return(file);
}
public static INodeDirectory LoadINodeDirectory(FsImageProto.INodeSection.INode n
, FSImageFormatProtobuf.LoaderContext state)
{
System.Diagnostics.Debug.Assert(n.GetType() == FsImageProto.INodeSection.INode.Type
.Directory);
FsImageProto.INodeSection.INodeDirectory d = n.GetDirectory();
PermissionStatus permissions = LoadPermission(d.GetPermission(), state.GetStringTable
());
INodeDirectory dir = new INodeDirectory(n.GetId(), n.GetName().ToByteArray(), permissions
, d.GetModificationTime());
long nsQuota = d.GetNsQuota();
long dsQuota = d.GetDsQuota();
if (nsQuota >= 0 || dsQuota >= 0)
{
dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().NameSpaceQuota
(nsQuota).StorageSpaceQuota(dsQuota).Build());
}
EnumCounters <StorageType> typeQuotas = null;
if (d.HasTypeQuotas())
{
ImmutableList <QuotaByStorageTypeEntry> qes = LoadQuotaByStorageTypeEntries(d.GetTypeQuotas
());
typeQuotas = new EnumCounters <StorageType>(typeof(StorageType), HdfsConstants.QuotaReset
);
foreach (QuotaByStorageTypeEntry qe in qes)
{
if (qe.GetQuota() >= 0 && qe.GetStorageType() != null && qe.GetStorageType().SupportTypeQuota
())
{
typeQuotas.Set(qe.GetStorageType(), qe.GetQuota());
}
}
if (typeQuotas.AnyGreaterOrEqual(0))
{
DirectoryWithQuotaFeature q = dir.GetDirectoryWithQuotaFeature();
if (q == null)
{
dir.AddDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().TypeQuotas
(typeQuotas).Build());
}
else
{
q.SetQuota(typeQuotas);
}
}
}
if (d.HasAcl())
{
int[] entries = AclEntryStatusFormat.ToInt(LoadAclEntries(d.GetAcl(), state.GetStringTable
()));
dir.AddAclFeature(new AclFeature(entries));
}
if (d.HasXAttrs())
{
dir.AddXAttrFeature(new XAttrFeature(LoadXAttrs(d.GetXAttrs(), state.GetStringTable
())));
}
return(dir);
}
private IDictionary <string, object> GetFileStatus(FsImageProto.INodeSection.INode
inode, bool printSuffix)
{
IDictionary <string, object> map = Maps.NewHashMap();
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile f = inode.GetFile();
PermissionStatus p = FSImageFormatPBINode.Loader.LoadPermission(f.GetPermission()
, stringTable);
map["accessTime"] = f.GetAccessTime();
map["blockSize"] = f.GetPreferredBlockSize();
map["group"] = p.GetGroupName();
map["length"] = GetFileSize(f);
map["modificationTime"] = f.GetModificationTime();
map["owner"] = p.GetUserName();
map["pathSuffix"] = printSuffix ? inode.GetName().ToStringUtf8() : string.Empty;
map["permission"] = ToString(p.GetPermission());
map["replication"] = f.GetReplication();
map["type"] = inode.GetType();
map["fileId"] = inode.GetId();
map["childrenNum"] = 0;
return(map);
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory d = inode.GetDirectory();
PermissionStatus p = FSImageFormatPBINode.Loader.LoadPermission(d.GetPermission()
, stringTable);
map["accessTime"] = 0;
map["blockSize"] = 0;
map["group"] = p.GetGroupName();
map["length"] = 0;
map["modificationTime"] = d.GetModificationTime();
map["owner"] = p.GetUserName();
map["pathSuffix"] = printSuffix ? inode.GetName().ToStringUtf8() : string.Empty;
map["permission"] = ToString(p.GetPermission());
map["replication"] = 0;
map["type"] = inode.GetType();
map["fileId"] = inode.GetId();
map["childrenNum"] = dirmap.Contains(inode.GetId()) ? dirmap[inode.GetId()].Length
: 0;
return(map);
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
FsImageProto.INodeSection.INodeSymlink d = inode.GetSymlink();
PermissionStatus p = FSImageFormatPBINode.Loader.LoadPermission(d.GetPermission()
, stringTable);
map["accessTime"] = d.GetAccessTime();
map["blockSize"] = 0;
map["group"] = p.GetGroupName();
map["length"] = 0;
map["modificationTime"] = d.GetModificationTime();
map["owner"] = p.GetUserName();
map["pathSuffix"] = printSuffix ? inode.GetName().ToStringUtf8() : string.Empty;
map["permission"] = ToString(p.GetPermission());
map["replication"] = 0;
map["type"] = inode.GetType();
map["symlink"] = d.GetTarget().ToStringUtf8();
map["fileId"] = inode.GetId();
map["childrenNum"] = 0;
return(map);
}
default:
{
return(null);
}
}
}
protected internal override string GetEntry(string parent, FsImageProto.INodeSection.INode
inode)
{
StringBuilder buffer = new StringBuilder();
string inodeName = inode.GetName().ToStringUtf8();
Path path = new Path(parent.IsEmpty() ? "/" : parent, inodeName.IsEmpty() ? "/" :
inodeName);
buffer.Append(path.ToString());
PermissionStatus p = null;
switch (inode.GetType())
{
case FsImageProto.INodeSection.INode.Type.File:
{
FsImageProto.INodeSection.INodeFile file = inode.GetFile();
p = GetPermission(file.GetPermission());
Append(buffer, file.GetReplication());
Append(buffer, FormatDate(file.GetModificationTime()));
Append(buffer, FormatDate(file.GetAccessTime()));
Append(buffer, file.GetPreferredBlockSize());
Append(buffer, file.GetBlocksCount());
Append(buffer, FSImageLoader.GetFileSize(file));
Append(buffer, 0);
// NS_QUOTA
Append(buffer, 0);
// DS_QUOTA
break;
}
case FsImageProto.INodeSection.INode.Type.Directory:
{
FsImageProto.INodeSection.INodeDirectory dir = inode.GetDirectory();
p = GetPermission(dir.GetPermission());
Append(buffer, 0);
// Replication
Append(buffer, FormatDate(dir.GetModificationTime()));
Append(buffer, FormatDate(0));
// Access time.
Append(buffer, 0);
// Block size.
Append(buffer, 0);
// Num blocks.
Append(buffer, 0);
// Num bytes.
Append(buffer, dir.GetNsQuota());
Append(buffer, dir.GetDsQuota());
break;
}
case FsImageProto.INodeSection.INode.Type.Symlink:
{
FsImageProto.INodeSection.INodeSymlink s = inode.GetSymlink();
p = GetPermission(s.GetPermission());
Append(buffer, 0);
// Replication
Append(buffer, FormatDate(s.GetModificationTime()));
Append(buffer, FormatDate(s.GetAccessTime()));
Append(buffer, 0);
// Block size.
Append(buffer, 0);
// Num blocks.
Append(buffer, 0);
// Num bytes.
Append(buffer, 0);
// NS_QUOTA
Append(buffer, 0);
// DS_QUOTA
break;
}
default:
{
break;
}
}
System.Diagnostics.Debug.Assert(p != null);
Append(buffer, p.GetPermission().ToString());
Append(buffer, p.GetUserName());
Append(buffer, p.GetGroupName());
return(buffer.ToString());
}