/// <exception cref="System.IO.IOException"/>
private void SaveInternal(FileOutputStream fout, FSImageCompression compression,
string filePath)
{
StartupProgress prog = NameNode.GetStartupProgress();
MessageDigest digester = MD5Hash.GetDigester();
underlyingOutputStream = new DigestOutputStream(new BufferedOutputStream(fout), digester
);
underlyingOutputStream.Write(FSImageUtil.MagicHeader);
fileChannel = fout.GetChannel();
FsImageProto.FileSummary.Builder b = FsImageProto.FileSummary.NewBuilder().SetOndiskVersion
(FSImageUtil.FileVersion).SetLayoutVersion(NameNodeLayoutVersion.CurrentLayoutVersion
);
codec = compression.GetImageCodec();
if (codec != null)
{
b.SetCodec(codec.GetType().GetCanonicalName());
sectionOutputStream = codec.CreateOutputStream(underlyingOutputStream);
}
else
{
sectionOutputStream = underlyingOutputStream;
}
SaveNameSystemSection(b);
// Check for cancellation right after serializing the name system section.
// Some unit tests, such as TestSaveNamespace#testCancelSaveNameSpace
// depends on this behavior.
context.CheckCancelled();
Step step = new Step(StepType.Inodes, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveInodes(b);
SaveSnapshots(b);
prog.EndStep(Phase.SavingCheckpoint, step);
step = new Step(StepType.DelegationTokens, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveSecretManagerSection(b);
prog.EndStep(Phase.SavingCheckpoint, step);
step = new Step(StepType.CachePools, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveCacheManagerSection(b);
prog.EndStep(Phase.SavingCheckpoint, step);
SaveStringTableSection(b);
// We use the underlyingOutputStream to write the header. Therefore flush
// the buffered stream (which is potentially compressed) first.
FlushSectionOutputStream();
FsImageProto.FileSummary summary = ((FsImageProto.FileSummary)b.Build());
SaveFileSummary(underlyingOutputStream, summary);
underlyingOutputStream.Close();
savedDigest = new MD5Hash(digester.Digest());
}
public virtual void SetUp()
{
startupProgress = new StartupProgress();
ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();
Org.Mockito.Mockito.When(context.GetAttribute(NameNodeHttpServer.StartupProgressAttributeKey
)).ThenReturn(startupProgress);
servlet = Org.Mockito.Mockito.Mock <StartupProgressServlet>();
Org.Mockito.Mockito.When(servlet.GetServletContext()).ThenReturn(context);
Org.Mockito.Mockito.DoCallRealMethod().When(servlet).DoGet(Any <HttpServletRequest
>(), Any <HttpServletResponse>());
req = Org.Mockito.Mockito.Mock <HttpServletRequest>();
respOut = new ByteArrayOutputStream();
PrintWriter writer = new PrintWriter(respOut);
resp = Org.Mockito.Mockito.Mock <HttpServletResponse>();
Org.Mockito.Mockito.When(resp.GetWriter()).ThenReturn(writer);
}
/// <summary>Private helper method to load delegation keys from fsimage.</summary>
/// <exception cref="System.IO.IOException">on error</exception>
private void LoadAllKeys(DataInput @in)
{
lock (this)
{
StartupProgress prog = NameNode.GetStartupProgress();
Step step = new Step(StepType.DelegationKeys);
prog.BeginStep(Phase.LoadingFsimage, step);
int numberOfKeys = @in.ReadInt();
prog.SetTotal(Phase.LoadingFsimage, step, numberOfKeys);
StartupProgress.Counter counter = prog.GetCounter(Phase.LoadingFsimage, step);
for (int i = 0; i < numberOfKeys; i++)
{
DelegationKey value = new DelegationKey();
value.ReadFields(@in);
this._enclosing.AddKey(value);
counter.Increment();
}
prog.EndStep(Phase.LoadingFsimage, step);
}
}
/*
* Save the current state of allKeys
*/
/// <exception cref="System.IO.IOException"/>
private void SaveAllKeys(DataOutputStream @out, string sdPath)
{
lock (this)
{
StartupProgress prog = NameNode.GetStartupProgress();
Step step = new Step(StepType.DelegationKeys, sdPath);
prog.BeginStep(Phase.SavingCheckpoint, step);
prog.SetTotal(Phase.SavingCheckpoint, step, this._enclosing.currentTokens.Count);
StartupProgress.Counter counter = prog.GetCounter(Phase.SavingCheckpoint, step);
@out.WriteInt(this._enclosing.allKeys.Count);
IEnumerator <int> iter = this._enclosing.allKeys.Keys.GetEnumerator();
while (iter.HasNext())
{
int key = iter.Next();
this._enclosing.allKeys[key].Write(@out);
counter.Increment();
}
prog.EndStep(Phase.SavingCheckpoint, step);
}
}
/// <summary>Private helper methods to load Delegation tokens from fsimage</summary>
/// <exception cref="System.IO.IOException"/>
private void LoadCurrentTokens(DataInput @in)
{
lock (this)
{
StartupProgress prog = NameNode.GetStartupProgress();
Step step = new Step(StepType.DelegationTokens);
prog.BeginStep(Phase.LoadingFsimage, step);
int numberOfTokens = @in.ReadInt();
prog.SetTotal(Phase.LoadingFsimage, step, numberOfTokens);
StartupProgress.Counter counter = prog.GetCounter(Phase.LoadingFsimage, step);
for (int i = 0; i < numberOfTokens; i++)
{
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.ReadFields(@in);
long expiryTime = @in.ReadLong();
this._enclosing.AddPersistedDelegationToken(id, expiryTime);
counter.Increment();
}
prog.EndStep(Phase.LoadingFsimage, step);
}
}
/// <summary>Private helper methods to save delegation keys and tokens in fsimage</summary>
/// <exception cref="System.IO.IOException"/>
private void SaveCurrentTokens(DataOutputStream @out, string sdPath)
{
lock (this)
{
StartupProgress prog = NameNode.GetStartupProgress();
Step step = new Step(StepType.DelegationTokens, sdPath);
prog.BeginStep(Phase.SavingCheckpoint, step);
prog.SetTotal(Phase.SavingCheckpoint, step, this._enclosing.currentTokens.Count);
StartupProgress.Counter counter = prog.GetCounter(Phase.SavingCheckpoint, step);
@out.WriteInt(this._enclosing.currentTokens.Count);
IEnumerator <DelegationTokenIdentifier> iter = this._enclosing.currentTokens.Keys.
GetEnumerator();
while (iter.HasNext())
{
DelegationTokenIdentifier id = iter.Next();
id.Write(@out);
AbstractDelegationTokenSecretManager.DelegationTokenInformation info = this._enclosing
.currentTokens[id];
@out.WriteLong(info.GetRenewDate());
counter.Increment();
}
prog.EndStep(Phase.SavingCheckpoint, step);
}
}
/// <exception cref="System.IO.IOException"/>
private void LoadInternal(RandomAccessFile raFile, FileInputStream fin)
{
if (!FSImageUtil.CheckFileFormat(raFile))
{
throw new IOException("Unrecognized file format");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(raFile);
if (requireSameLayoutVersion && summary.GetLayoutVersion() != HdfsConstants.NamenodeLayoutVersion)
{
throw new IOException("Image version " + summary.GetLayoutVersion() + " is not equal to the software version "
+ HdfsConstants.NamenodeLayoutVersion);
}
FileChannel channel = fin.GetChannel();
FSImageFormatPBINode.Loader inodeLoader = new FSImageFormatPBINode.Loader(fsn, this
);
FSImageFormatPBSnapshot.Loader snapshotLoader = new FSImageFormatPBSnapshot.Loader
(fsn, this);
AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
());
sections.Sort(new _IComparer_210());
StartupProgress prog = NameNode.GetStartupProgress();
Step currentStep = null;
foreach (FsImageProto.FileSummary.Section s in sections)
{
channel.Position(s.GetOffset());
InputStream @in = new BufferedInputStream(new LimitInputStream(fin, s.GetLength()
));
@in = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), @in);
string n = s.GetName();
switch (FSImageFormatProtobuf.SectionName.FromString(n))
{
case FSImageFormatProtobuf.SectionName.NsInfo:
{
LoadNameSystemSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.StringTable:
{
LoadStringTableSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.Inode:
{
currentStep = new Step(StepType.Inodes);
prog.BeginStep(Phase.LoadingFsimage, currentStep);
inodeLoader.LoadINodeSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.InodeReference:
{
snapshotLoader.LoadINodeReferenceSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.InodeDir:
{
inodeLoader.LoadINodeDirectorySection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.FilesUnderconstruction:
{
inodeLoader.LoadFilesUnderConstructionSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.Snapshot:
{
snapshotLoader.LoadSnapshotSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.SnapshotDiff:
{
snapshotLoader.LoadSnapshotDiffSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.SecretManager:
{
prog.EndStep(Phase.LoadingFsimage, currentStep);
Step step = new Step(StepType.DelegationTokens);
prog.BeginStep(Phase.LoadingFsimage, step);
LoadSecretManagerSection(@in);
prog.EndStep(Phase.LoadingFsimage, step);
break;
}
case FSImageFormatProtobuf.SectionName.CacheManager:
{
Step step = new Step(StepType.CachePools);
prog.BeginStep(Phase.LoadingFsimage, step);
LoadCacheManagerSection(@in);
prog.EndStep(Phase.LoadingFsimage, step);
break;
}
default:
{
Log.Warn("Unrecognized section " + n);
break;
}
}
}
}
/// <summary>Sets startup progress of namenode for use by servlets.</summary>
/// <param name="prog">StartupProgress to set</param>
internal virtual void SetStartupProgress(StartupProgress prog)
{
httpServer.SetAttribute(StartupProgressAttributeKey, prog);
}
