public virtual void TestNameNodeFeature()
{
LayoutVersion.LayoutFeature first = NameNodeLayoutVersion.Feature.RollingUpgrade;
NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LastNonReservedCommonFeature
, first.GetInfo().GetLayoutVersion()));
NUnit.Framework.Assert.AreEqual(LastCommonFeature.GetInfo().GetLayoutVersion() -
1, first.GetInfo().GetLayoutVersion());
}
/// <exception cref="System.IO.IOException"/>
private void ProcessDirectoryDiff(DataInputStream @in, ImageVisitor v, string currentINodeName
)
{
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiff);
int snapshotId = @in.ReadInt();
v.Visit(ImageVisitor.ImageElement.SnapshotDiffSnapshotid, snapshotId);
v.Visit(ImageVisitor.ImageElement.SnapshotDirDiffChildrenSize, @in.ReadInt());
// process snapshotINode
bool useRoot = @in.ReadBoolean();
if (!useRoot)
{
if (@in.ReadBoolean())
{
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotInodeDirectoryAttributes
);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.OptimizeSnapshotInodes,
imageVersion))
{
ProcessINodeDirectoryAttributes(@in, v, currentINodeName);
}
else
{
ProcessINode(@in, v, true, currentINodeName, true);
}
v.LeaveEnclosingElement();
}
}
// process createdList
int createdSize = @in.ReadInt();
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiffCreatedlist, ImageVisitor.ImageElement
.SnapshotDirDiffCreatedlistSize, createdSize);
for (int i = 0; i < createdSize; i++)
{
string createdNode = FSImageSerialization.ReadString(@in);
v.Visit(ImageVisitor.ImageElement.SnapshotDirDiffCreatedInode, createdNode);
}
v.LeaveEnclosingElement();
// process deletedList
int deletedSize = @in.ReadInt();
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiffDeletedlist, ImageVisitor.ImageElement
.SnapshotDirDiffDeletedlistSize, deletedSize);
for (int i_1 = 0; i_1 < deletedSize; i_1++)
{
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiffDeletedInode);
ProcessINode(@in, v, false, currentINodeName, true);
v.LeaveEnclosingElement();
}
v.LeaveEnclosingElement();
v.LeaveEnclosingElement();
}
public virtual void TestSNAPSHOT()
{
foreach (LayoutVersion.Feature f in LayoutVersion.Feature.Values())
{
int version = f.GetInfo().GetLayoutVersion();
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot, version))
{
NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LayoutVersion.Feature
.FsimageNameOptimization, version));
}
}
}
/// <summary>
/// Given feature
/// <paramref name="f"/>
/// , ensures the layout version of that feature
/// supports all the features supported by it's ancestor.
/// </summary>
private void ValidateFeatureList(LayoutVersion.LayoutFeature f)
{
LayoutVersion.FeatureInfo info = f.GetInfo();
int lv = info.GetLayoutVersion();
int ancestorLV = info.GetAncestorLayoutVersion();
ICollection <LayoutVersion.LayoutFeature> ancestorSet = NameNodeLayoutVersion.GetFeatures
(ancestorLV);
NUnit.Framework.Assert.IsNotNull(ancestorSet);
foreach (LayoutVersion.LayoutFeature feature in ancestorSet)
{
NUnit.Framework.Assert.IsTrue("LV " + lv + " does nto support " + feature + " supported by the ancestor LV "
+ info.GetAncestorLayoutVersion(), NameNodeLayoutVersion.Supports(feature, lv));
}
}
// DELEGATION_TOKENS
/// <summary>Process the INodes under construction section of the fsimage.</summary>
/// <param name="in">DataInputStream to process</param>
/// <param name="v">Visitor to walk over inodes</param>
/// <param name="skipBlocks">Walk over each block?</param>
/// <exception cref="System.IO.IOException"/>
private void ProcessINodesUC(DataInputStream @in, ImageVisitor v, bool skipBlocks
)
{
int numINUC = @in.ReadInt();
v.VisitEnclosingElement(ImageVisitor.ImageElement.InodesUnderConstruction, ImageVisitor.ImageElement
.NumInodesUnderConstruction, numINUC);
for (int i = 0; i < numINUC; i++)
{
v.VisitEnclosingElement(ImageVisitor.ImageElement.InodeUnderConstruction);
byte[] name = FSImageSerialization.ReadBytes(@in);
string n = Sharpen.Runtime.GetStringForBytes(name, "UTF8");
v.Visit(ImageVisitor.ImageElement.InodePath, n);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imageVersion
))
{
long inodeId = @in.ReadLong();
v.Visit(ImageVisitor.ImageElement.InodeId, inodeId);
}
v.Visit(ImageVisitor.ImageElement.Replication, @in.ReadShort());
v.Visit(ImageVisitor.ImageElement.ModificationTime, FormatDate(@in.ReadLong()));
v.Visit(ImageVisitor.ImageElement.PreferredBlockSize, @in.ReadLong());
int numBlocks = @in.ReadInt();
ProcessBlocks(@in, v, numBlocks, skipBlocks);
ProcessPermission(@in, v);
v.Visit(ImageVisitor.ImageElement.ClientName, FSImageSerialization.ReadString(@in
));
v.Visit(ImageVisitor.ImageElement.ClientMachine, FSImageSerialization.ReadString(
@in));
// Skip over the datanode descriptors, which are still stored in the
// file but are not used by the datanode or loaded into memory
int numLocs = @in.ReadInt();
for (int j = 0; j < numLocs; j++)
{
@in.ReadShort();
@in.ReadLong();
@in.ReadLong();
@in.ReadLong();
@in.ReadInt();
FSImageSerialization.ReadString(@in);
FSImageSerialization.ReadString(@in);
WritableUtils.ReadEnum <DatanodeInfo.AdminStates>(@in);
}
v.LeaveEnclosingElement();
}
// INodeUnderConstruction
v.LeaveEnclosingElement();
}
// INode
/// <exception cref="System.IO.IOException"/>
private void ProcessINodeFileAttributes(DataInputStream @in, ImageVisitor v, string
parentName)
{
string pathName = ReadINodePath(@in, parentName);
v.Visit(ImageVisitor.ImageElement.InodePath, pathName);
ProcessPermission(@in, v);
v.Visit(ImageVisitor.ImageElement.ModificationTime, FormatDate(@in.ReadLong()));
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FileAccessTime, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.AccessTime, FormatDate(@in.ReadLong()));
}
v.Visit(ImageVisitor.ImageElement.Replication, @in.ReadShort());
v.Visit(ImageVisitor.ImageElement.BlockSize, @in.ReadLong());
}
/// <exception cref="System.IO.IOException"/>
private void ProcessFileDiff(DataInputStream @in, ImageVisitor v, string currentINodeName
)
{
int snapshotId = @in.ReadInt();
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotFileDiff, ImageVisitor.ImageElement
.SnapshotDiffSnapshotid, snapshotId);
v.Visit(ImageVisitor.ImageElement.SnapshotFileSize, @in.ReadLong());
if (@in.ReadBoolean())
{
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotInodeFileAttributes);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.OptimizeSnapshotInodes,
imageVersion))
{
ProcessINodeFileAttributes(@in, v, currentINodeName);
}
else
{
ProcessINode(@in, v, true, currentINodeName, true);
}
v.LeaveEnclosingElement();
}
v.LeaveEnclosingElement();
}
// Permissions
/// <summary>Process the INode records stored in the fsimage.</summary>
/// <param name="in">Datastream to process</param>
/// <param name="v">Visitor to walk over INodes</param>
/// <param name="numInodes">Number of INodes stored in file</param>
/// <param name="skipBlocks">Process all the blocks within the INode?</param>
/// <param name="supportSnapshot">Whether or not the imageVersion supports snapshot</param>
/// <exception cref="VisitException"/>
/// <exception cref="System.IO.IOException"/>
private void ProcessINodes(DataInputStream @in, ImageVisitor v, long numInodes, bool
skipBlocks, bool supportSnapshot)
{
v.VisitEnclosingElement(ImageVisitor.ImageElement.Inodes, ImageVisitor.ImageElement
.NumInodes, numInodes);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FsimageNameOptimization,
imageVersion))
{
if (!supportSnapshot)
{
ProcessLocalNameINodes(@in, v, numInodes, skipBlocks);
}
else
{
ProcessLocalNameINodesWithSnapshot(@in, v, skipBlocks);
}
}
else
{
// full path name
ProcessFullNameINodes(@in, v, numInodes, skipBlocks);
}
v.LeaveEnclosingElement();
}
/// <summary>Process an INode</summary>
/// <param name="in">image stream</param>
/// <param name="v">visitor</param>
/// <param name="skipBlocks">skip blocks or not</param>
/// <param name="parentName">the name of its parent node</param>
/// <param name="isSnapshotCopy">whether or not the inode is a snapshot copy</param>
/// <exception cref="System.IO.IOException"/>
private void ProcessINode(DataInputStream @in, ImageVisitor v, bool skipBlocks, string
parentName, bool isSnapshotCopy)
{
bool supportSnapshot = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot
, imageVersion);
bool supportInodeId = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId
, imageVersion);
v.VisitEnclosingElement(ImageVisitor.ImageElement.Inode);
string pathName = ReadINodePath(@in, parentName);
v.Visit(ImageVisitor.ImageElement.InodePath, pathName);
long inodeId = INodeId.GrandfatherInodeId;
if (supportInodeId)
{
inodeId = @in.ReadLong();
v.Visit(ImageVisitor.ImageElement.InodeId, inodeId);
}
v.Visit(ImageVisitor.ImageElement.Replication, @in.ReadShort());
v.Visit(ImageVisitor.ImageElement.ModificationTime, FormatDate(@in.ReadLong()));
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FileAccessTime, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.AccessTime, FormatDate(@in.ReadLong()));
}
v.Visit(ImageVisitor.ImageElement.BlockSize, @in.ReadLong());
int numBlocks = @in.ReadInt();
ProcessBlocks(@in, v, numBlocks, skipBlocks);
if (numBlocks >= 0)
{
// File
if (supportSnapshot)
{
// make sure subtreeMap only contains entry for directory
Sharpen.Collections.Remove(subtreeMap, inodeId);
// process file diffs
ProcessFileDiffList(@in, v, parentName);
if (isSnapshotCopy)
{
bool underConstruction = @in.ReadBoolean();
if (underConstruction)
{
v.Visit(ImageVisitor.ImageElement.ClientName, FSImageSerialization.ReadString(@in
));
v.Visit(ImageVisitor.ImageElement.ClientMachine, FSImageSerialization.ReadString(
@in));
}
}
}
ProcessPermission(@in, v);
}
else
{
if (numBlocks == -1)
{
// Directory
if (supportSnapshot && supportInodeId)
{
dirNodeMap[inodeId] = pathName;
}
v.Visit(ImageVisitor.ImageElement.NsQuota, numBlocks == -1 ? @in.ReadLong() : -1);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.DiskspaceQuota, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.DsQuota, numBlocks == -1 ? @in.ReadLong() : -1);
}
if (supportSnapshot)
{
bool snapshottable = @in.ReadBoolean();
if (!snapshottable)
{
bool withSnapshot = @in.ReadBoolean();
v.Visit(ImageVisitor.ImageElement.IsWithsnapshotDir, bool.ToString(withSnapshot));
}
else
{
v.Visit(ImageVisitor.ImageElement.IsSnapshottableDir, bool.ToString(snapshottable
));
}
}
ProcessPermission(@in, v);
}
else
{
if (numBlocks == -2)
{
v.Visit(ImageVisitor.ImageElement.Symlink, Text.ReadString(@in));
ProcessPermission(@in, v);
}
else
{
if (numBlocks == -3)
{
// reference node
bool isWithName = @in.ReadBoolean();
int snapshotId = @in.ReadInt();
if (isWithName)
{
v.Visit(ImageVisitor.ImageElement.SnapshotLastSnapshotId, snapshotId);
}
else
{
v.Visit(ImageVisitor.ImageElement.SnapshotDstSnapshotId, snapshotId);
}
bool firstReferred = @in.ReadBoolean();
if (firstReferred)
{
// if a subtree is linked by multiple "parents", the corresponding dir
// must be referred by a reference node. we put the reference node into
// the subtreeMap here and let its value be false. when we later visit
// the subtree for the first time, we change the value to true.
subtreeMap[inodeId] = false;
v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotRefInode);
ProcessINode(@in, v, skipBlocks, parentName, isSnapshotCopy);
v.LeaveEnclosingElement();
}
else
{
// referred inode
v.Visit(ImageVisitor.ImageElement.SnapshotRefInodeId, @in.ReadLong());
}
}
}
}
}
v.LeaveEnclosingElement();
}
/* (non-Javadoc)
* @see ImageLoader#processImage(java.io.DataInputStream, ImageVisitor, boolean)
*/
/// <exception cref="System.IO.IOException"/>
public override void LoadImage(DataInputStream @in, ImageVisitor v, bool skipBlocks
)
{
bool done = false;
try
{
v.Start();
v.VisitEnclosingElement(ImageVisitor.ImageElement.FsImage);
imageVersion = @in.ReadInt();
if (!CanLoadVersion(imageVersion))
{
throw new IOException("Cannot process fslayout version " + imageVersion);
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddLayoutFlags, imageVersion
))
{
LayoutFlags.Read(@in);
}
v.Visit(ImageVisitor.ImageElement.ImageVersion, imageVersion);
v.Visit(ImageVisitor.ImageElement.NamespaceId, @in.ReadInt());
long numInodes = @in.ReadLong();
v.Visit(ImageVisitor.ImageElement.GenerationStamp, @in.ReadLong());
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.SequentialBlockId, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.GenerationStampV2, @in.ReadLong());
v.Visit(ImageVisitor.ImageElement.GenerationStampV1Limit, @in.ReadLong());
v.Visit(ImageVisitor.ImageElement.LastAllocatedBlockId, @in.ReadLong());
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.StoredTxids, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.TransactionId, @in.ReadLong());
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.LastInodeId, @in.ReadLong());
}
bool supportSnapshot = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot
, imageVersion);
if (supportSnapshot)
{
v.Visit(ImageVisitor.ImageElement.SnapshotCounter, @in.ReadInt());
int numSnapshots = @in.ReadInt();
v.Visit(ImageVisitor.ImageElement.NumSnapshotsTotal, numSnapshots);
for (int i = 0; i < numSnapshots; i++)
{
ProcessSnapshot(@in, v);
}
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FsimageCompression, imageVersion
))
{
bool isCompressed = @in.ReadBoolean();
v.Visit(ImageVisitor.ImageElement.IsCompressed, isCompressed.ToString());
if (isCompressed)
{
string codecClassName = Text.ReadString(@in);
v.Visit(ImageVisitor.ImageElement.CompressCodec, codecClassName);
CompressionCodecFactory codecFac = new CompressionCodecFactory(new Configuration(
));
CompressionCodec codec = codecFac.GetCodecByClassName(codecClassName);
if (codec == null)
{
throw new IOException("Image compression codec not supported: " + codecClassName);
}
@in = new DataInputStream(codec.CreateInputStream(@in));
}
}
ProcessINodes(@in, v, numInodes, skipBlocks, supportSnapshot);
subtreeMap.Clear();
dirNodeMap.Clear();
ProcessINodesUC(@in, v, skipBlocks);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.DelegationToken, imageVersion
))
{
ProcessDelegationTokens(@in, v);
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Caching, imageVersion))
{
ProcessCacheManagerState(@in, v);
}
v.LeaveEnclosingElement();
// FSImage
done = true;
}
finally
{
if (done)
{
v.Finish();
}
else
{
v.FinishAbnormally();
}
}
}
public virtual void TestRelease1_2_0()
{
NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LayoutVersion.Feature
.Concat, LayoutVersion.Feature.ReservedRel120.GetInfo().GetLayoutVersion()));
}
public virtual void TestRelease204()
{
NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LayoutVersion.Feature
.DelegationToken, LayoutVersion.Feature.ReservedRel20204.GetInfo().GetLayoutVersion
()));
}
