/// <exception cref="System.IO.IOException"/>
public static FsImageProto.FileSummary LoadSummary(RandomAccessFile file)
{
int FileLengthFieldSize = 4;
long fileLength = file.Length();
file.Seek(fileLength - FileLengthFieldSize);
int summaryLength = file.ReadInt();
if (summaryLength <= 0)
{
throw new IOException("Negative length of the file");
}
file.Seek(fileLength - FileLengthFieldSize - summaryLength);
byte[] summaryBytes = new byte[summaryLength];
file.ReadFully(summaryBytes);
FsImageProto.FileSummary summary = FsImageProto.FileSummary.ParseDelimitedFrom(new
ByteArrayInputStream(summaryBytes));
if (summary.GetOndiskVersion() != FileVersion)
{
throw new IOException("Unsupported file version " + summary.GetOndiskVersion());
}
if (!NameNodeLayoutVersion.Supports(LayoutVersion.Feature.ProtobufFormat, summary
.GetLayoutVersion()))
{
throw new IOException("Unsupported layout version " + summary.GetLayoutVersion());
}
return(summary);
}
/// <exception cref="System.IO.IOException"/>
private static void SaveFileSummary(OutputStream @out, FsImageProto.FileSummary summary
)
{
summary.WriteDelimitedTo(@out);
int length = GetOndiskTrunkSize(summary);
byte[] lengthBytes = new byte[4];
ByteBuffer.Wrap(lengthBytes).AsIntBuffer().Put(length);
@out.Write(lengthBytes);
}
/// <exception cref="System.IO.IOException"/>
private void SaveInternal(FileOutputStream fout, FSImageCompression compression,
string filePath)
{
StartupProgress prog = NameNode.GetStartupProgress();
MessageDigest digester = MD5Hash.GetDigester();
underlyingOutputStream = new DigestOutputStream(new BufferedOutputStream(fout), digester
);
underlyingOutputStream.Write(FSImageUtil.MagicHeader);
fileChannel = fout.GetChannel();
FsImageProto.FileSummary.Builder b = FsImageProto.FileSummary.NewBuilder().SetOndiskVersion
(FSImageUtil.FileVersion).SetLayoutVersion(NameNodeLayoutVersion.CurrentLayoutVersion
);
codec = compression.GetImageCodec();
if (codec != null)
{
b.SetCodec(codec.GetType().GetCanonicalName());
sectionOutputStream = codec.CreateOutputStream(underlyingOutputStream);
}
else
{
sectionOutputStream = underlyingOutputStream;
}
SaveNameSystemSection(b);
// Check for cancellation right after serializing the name system section.
// Some unit tests, such as TestSaveNamespace#testCancelSaveNameSpace
// depends on this behavior.
context.CheckCancelled();
Step step = new Step(StepType.Inodes, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveInodes(b);
SaveSnapshots(b);
prog.EndStep(Phase.SavingCheckpoint, step);
step = new Step(StepType.DelegationTokens, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveSecretManagerSection(b);
prog.EndStep(Phase.SavingCheckpoint, step);
step = new Step(StepType.CachePools, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveCacheManagerSection(b);
prog.EndStep(Phase.SavingCheckpoint, step);
SaveStringTableSection(b);
// We use the underlyingOutputStream to write the header. Therefore flush
// the buffered stream (which is potentially compressed) first.
FlushSectionOutputStream();
FsImageProto.FileSummary summary = ((FsImageProto.FileSummary)b.Build());
SaveFileSummary(underlyingOutputStream, summary);
underlyingOutputStream.Close();
savedDigest = new MD5Hash(digester.Digest());
}
/// <exception cref="System.IO.IOException"/>
private void LoadINodeDirSection(FileInputStream fin, IList <FsImageProto.FileSummary.Section
> sections, FsImageProto.FileSummary summary, Configuration conf)
{
Log.Info("Loading INode directory section.");
long startTime = Time.MonotonicNow();
foreach (FsImageProto.FileSummary.Section section in sections)
{
if (FSImageFormatProtobuf.SectionName.FromString(section.GetName()) == FSImageFormatProtobuf.SectionName
.InodeDir)
{
fin.GetChannel().Position(section.GetOffset());
InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
(), new BufferedInputStream(new LimitInputStream(fin, section.GetLength())));
BuildNamespace(@is);
}
}
long timeTaken = Time.MonotonicNow() - startTime;
Log.Info("Finished loading INode directory section in {}ms", timeTaken);
}
/// <exception cref="System.IO.IOException"/>
private void Output(Configuration conf, FsImageProto.FileSummary summary, FileInputStream
fin, AList <FsImageProto.FileSummary.Section> sections)
{
InputStream @is;
long startTime = Time.MonotonicNow();
foreach (FsImageProto.FileSummary.Section section in sections)
{
if (FSImageFormatProtobuf.SectionName.FromString(section.GetName()) == FSImageFormatProtobuf.SectionName
.Inode)
{
fin.GetChannel().Position(section.GetOffset());
@is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), new BufferedInputStream
(new LimitInputStream(fin, section.GetLength())));
OutputINodes(@is);
}
}
long timeTaken = Time.MonotonicNow() - startTime;
Log.Debug("Time to output inodes: {}ms", timeTaken);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Visit(RandomAccessFile file)
{
Configuration conf = new Configuration();
if (!FSImageUtil.CheckFileFormat(file))
{
throw new IOException("Unrecognized FSImage");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
using (FileInputStream fin = new FileInputStream(file.GetFD()))
{
InputStream @is;
AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
());
sections.Sort(new _IComparer_427());
foreach (FsImageProto.FileSummary.Section section in sections)
{
fin.GetChannel().Position(section.GetOffset());
@is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), new BufferedInputStream
(new LimitInputStream(fin, section.GetLength())));
switch (FSImageFormatProtobuf.SectionName.FromString(section.GetName()))
{
case FSImageFormatProtobuf.SectionName.StringTable:
{
stringTable = FSImageLoader.LoadStringTable(@is);
break;
}
default:
{
break;
}
}
}
LoadDirectories(fin, sections, summary, conf);
LoadINodeDirSection(fin, sections, summary, conf);
metadataMap.Sync();
Output(conf, summary, fin, sections);
}
}
/// <exception cref="System.IO.IOException"/>
internal void Visit(RandomAccessFile file)
{
if (!FSImageUtil.CheckFileFormat(file))
{
throw new IOException("Unrecognized FSImage");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
using (FileInputStream @in = new FileInputStream(file.GetFD()))
{
foreach (FsImageProto.FileSummary.Section s in summary.GetSectionsList())
{
if (FSImageFormatProtobuf.SectionName.FromString(s.GetName()) != FSImageFormatProtobuf.SectionName
.Inode)
{
continue;
}
@in.GetChannel().Position(s.GetOffset());
InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
(), new BufferedInputStream(new LimitInputStream(@in, s.GetLength())));
Run(@is);
Output();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadInternal(RandomAccessFile raFile, FileInputStream fin)
{
if (!FSImageUtil.CheckFileFormat(raFile))
{
throw new IOException("Unrecognized file format");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(raFile);
if (requireSameLayoutVersion && summary.GetLayoutVersion() != HdfsConstants.NamenodeLayoutVersion)
{
throw new IOException("Image version " + summary.GetLayoutVersion() + " is not equal to the software version "
+ HdfsConstants.NamenodeLayoutVersion);
}
FileChannel channel = fin.GetChannel();
FSImageFormatPBINode.Loader inodeLoader = new FSImageFormatPBINode.Loader(fsn, this
);
FSImageFormatPBSnapshot.Loader snapshotLoader = new FSImageFormatPBSnapshot.Loader
(fsn, this);
AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
());
sections.Sort(new _IComparer_210());
StartupProgress prog = NameNode.GetStartupProgress();
Step currentStep = null;
foreach (FsImageProto.FileSummary.Section s in sections)
{
channel.Position(s.GetOffset());
InputStream @in = new BufferedInputStream(new LimitInputStream(fin, s.GetLength()
));
@in = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), @in);
string n = s.GetName();
switch (FSImageFormatProtobuf.SectionName.FromString(n))
{
case FSImageFormatProtobuf.SectionName.NsInfo:
{
LoadNameSystemSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.StringTable:
{
LoadStringTableSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.Inode:
{
currentStep = new Step(StepType.Inodes);
prog.BeginStep(Phase.LoadingFsimage, currentStep);
inodeLoader.LoadINodeSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.InodeReference:
{
snapshotLoader.LoadINodeReferenceSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.InodeDir:
{
inodeLoader.LoadINodeDirectorySection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.FilesUnderconstruction:
{
inodeLoader.LoadFilesUnderConstructionSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.Snapshot:
{
snapshotLoader.LoadSnapshotSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.SnapshotDiff:
{
snapshotLoader.LoadSnapshotDiffSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.SecretManager:
{
prog.EndStep(Phase.LoadingFsimage, currentStep);
Step step = new Step(StepType.DelegationTokens);
prog.BeginStep(Phase.LoadingFsimage, step);
LoadSecretManagerSection(@in);
prog.EndStep(Phase.LoadingFsimage, step);
break;
}
case FSImageFormatProtobuf.SectionName.CacheManager:
{
Step step = new Step(StepType.CachePools);
prog.BeginStep(Phase.LoadingFsimage, step);
LoadCacheManagerSection(@in);
prog.EndStep(Phase.LoadingFsimage, step);
break;
}
default:
{
Log.Warn("Unrecognized section " + n);
break;
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public void Visit(RandomAccessFile file)
{
if (!FSImageUtil.CheckFileFormat(file))
{
throw new IOException("Unrecognized FSImage");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
using (FileInputStream fin = new FileInputStream(file.GetFD()))
{
@out.Write("<?xml version=\"1.0\"?>\n<fsimage>");
AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
());
sections.Sort(new _IComparer_83());
foreach (FsImageProto.FileSummary.Section s in sections)
{
fin.GetChannel().Position(s.GetOffset());
InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
(), new BufferedInputStream(new LimitInputStream(fin, s.GetLength())));
switch (FSImageFormatProtobuf.SectionName.FromString(s.GetName()))
{
case FSImageFormatProtobuf.SectionName.NsInfo:
{
DumpNameSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.StringTable:
{
LoadStringTable(@is);
break;
}
case FSImageFormatProtobuf.SectionName.Inode:
{
DumpINodeSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.InodeReference:
{
DumpINodeReferenceSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.InodeDir:
{
DumpINodeDirectorySection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.FilesUnderconstruction:
{
DumpFileUnderConstructionSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.Snapshot:
{
DumpSnapshotSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.SnapshotDiff:
{
DumpSnapshotDiffSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.SecretManager:
{
DumpSecretManagerSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.CacheManager:
{
DumpCacheManagerSection(@is);
break;
}
default:
{
break;
}
}
}
@out.Write("</fsimage>\n");
}
}
/// <summary>Load fsimage into the memory.</summary>
/// <param name="inputFile">the filepath of the fsimage to load.</param>
/// <returns>FSImageLoader</returns>
/// <exception cref="System.IO.IOException">if failed to load fsimage.</exception>
internal static Org.Apache.Hadoop.Hdfs.Tools.OfflineImageViewer.FSImageLoader Load
(string inputFile)
{
Configuration conf = new Configuration();
RandomAccessFile file = new RandomAccessFile(inputFile, "r");
if (!FSImageUtil.CheckFileFormat(file))
{
throw new IOException("Unrecognized FSImage");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
using (FileInputStream fin = new FileInputStream(file.GetFD()))
{
// Map to record INodeReference to the referred id
ImmutableList <long> refIdList = null;
string[] stringTable = null;
byte[][] inodes = null;
IDictionary <long, long[]> dirmap = null;
AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
());
sections.Sort(new _IComparer_126());
foreach (FsImageProto.FileSummary.Section s in sections)
{
fin.GetChannel().Position(s.GetOffset());
InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
(), new BufferedInputStream(new LimitInputStream(fin, s.GetLength())));
Log.Debug("Loading section " + s.GetName() + " length: " + s.GetLength());
switch (FSImageFormatProtobuf.SectionName.FromString(s.GetName()))
{
case FSImageFormatProtobuf.SectionName.StringTable:
{
stringTable = LoadStringTable(@is);
break;
}
case FSImageFormatProtobuf.SectionName.Inode:
{
inodes = LoadINodeSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.InodeReference:
{
refIdList = LoadINodeReferenceSection(@is);
break;
}
case FSImageFormatProtobuf.SectionName.InodeDir:
{
dirmap = LoadINodeDirectorySection(@is, refIdList);
break;
}
default:
{
break;
}
}
}
return(new Org.Apache.Hadoop.Hdfs.Tools.OfflineImageViewer.FSImageLoader(stringTable
, inodes, dirmap));
}
}
