public virtual void TestNameNodeFeature() { LayoutVersion.LayoutFeature first = NameNodeLayoutVersion.Feature.RollingUpgrade; NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LastNonReservedCommonFeature , first.GetInfo().GetLayoutVersion())); NUnit.Framework.Assert.AreEqual(LastCommonFeature.GetInfo().GetLayoutVersion() - 1, first.GetInfo().GetLayoutVersion()); }
/// <exception cref="System.IO.IOException"/> private void ProcessDirectoryDiff(DataInputStream @in, ImageVisitor v, string currentINodeName ) { v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiff); int snapshotId = @in.ReadInt(); v.Visit(ImageVisitor.ImageElement.SnapshotDiffSnapshotid, snapshotId); v.Visit(ImageVisitor.ImageElement.SnapshotDirDiffChildrenSize, @in.ReadInt()); // process snapshotINode bool useRoot = @in.ReadBoolean(); if (!useRoot) { if (@in.ReadBoolean()) { v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotInodeDirectoryAttributes ); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.OptimizeSnapshotInodes, imageVersion)) { ProcessINodeDirectoryAttributes(@in, v, currentINodeName); } else { ProcessINode(@in, v, true, currentINodeName, true); } v.LeaveEnclosingElement(); } } // process createdList int createdSize = @in.ReadInt(); v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiffCreatedlist, ImageVisitor.ImageElement .SnapshotDirDiffCreatedlistSize, createdSize); for (int i = 0; i < createdSize; i++) { string createdNode = FSImageSerialization.ReadString(@in); v.Visit(ImageVisitor.ImageElement.SnapshotDirDiffCreatedInode, createdNode); } v.LeaveEnclosingElement(); // process deletedList int deletedSize = @in.ReadInt(); v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiffDeletedlist, ImageVisitor.ImageElement .SnapshotDirDiffDeletedlistSize, deletedSize); for (int i_1 = 0; i_1 < deletedSize; i_1++) { v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotDirDiffDeletedInode); ProcessINode(@in, v, false, currentINodeName, true); v.LeaveEnclosingElement(); } v.LeaveEnclosingElement(); v.LeaveEnclosingElement(); }
public virtual void TestSNAPSHOT() { foreach (LayoutVersion.Feature f in LayoutVersion.Feature.Values()) { int version = f.GetInfo().GetLayoutVersion(); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot, version)) { NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LayoutVersion.Feature .FsimageNameOptimization, version)); } } }
/// <summary> /// Given feature /// <paramref name="f"/> /// , ensures the layout version of that feature /// supports all the features supported by it's ancestor. /// </summary> private void ValidateFeatureList(LayoutVersion.LayoutFeature f) { LayoutVersion.FeatureInfo info = f.GetInfo(); int lv = info.GetLayoutVersion(); int ancestorLV = info.GetAncestorLayoutVersion(); ICollection <LayoutVersion.LayoutFeature> ancestorSet = NameNodeLayoutVersion.GetFeatures (ancestorLV); NUnit.Framework.Assert.IsNotNull(ancestorSet); foreach (LayoutVersion.LayoutFeature feature in ancestorSet) { NUnit.Framework.Assert.IsTrue("LV " + lv + " does nto support " + feature + " supported by the ancestor LV " + info.GetAncestorLayoutVersion(), NameNodeLayoutVersion.Supports(feature, lv)); } }
// DELEGATION_TOKENS /// <summary>Process the INodes under construction section of the fsimage.</summary> /// <param name="in">DataInputStream to process</param> /// <param name="v">Visitor to walk over inodes</param> /// <param name="skipBlocks">Walk over each block?</param> /// <exception cref="System.IO.IOException"/> private void ProcessINodesUC(DataInputStream @in, ImageVisitor v, bool skipBlocks ) { int numINUC = @in.ReadInt(); v.VisitEnclosingElement(ImageVisitor.ImageElement.InodesUnderConstruction, ImageVisitor.ImageElement .NumInodesUnderConstruction, numINUC); for (int i = 0; i < numINUC; i++) { v.VisitEnclosingElement(ImageVisitor.ImageElement.InodeUnderConstruction); byte[] name = FSImageSerialization.ReadBytes(@in); string n = Sharpen.Runtime.GetStringForBytes(name, "UTF8"); v.Visit(ImageVisitor.ImageElement.InodePath, n); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imageVersion )) { long inodeId = @in.ReadLong(); v.Visit(ImageVisitor.ImageElement.InodeId, inodeId); } v.Visit(ImageVisitor.ImageElement.Replication, @in.ReadShort()); v.Visit(ImageVisitor.ImageElement.ModificationTime, FormatDate(@in.ReadLong())); v.Visit(ImageVisitor.ImageElement.PreferredBlockSize, @in.ReadLong()); int numBlocks = @in.ReadInt(); ProcessBlocks(@in, v, numBlocks, skipBlocks); ProcessPermission(@in, v); v.Visit(ImageVisitor.ImageElement.ClientName, FSImageSerialization.ReadString(@in )); v.Visit(ImageVisitor.ImageElement.ClientMachine, FSImageSerialization.ReadString( @in)); // Skip over the datanode descriptors, which are still stored in the // file but are not used by the datanode or loaded into memory int numLocs = @in.ReadInt(); for (int j = 0; j < numLocs; j++) { @in.ReadShort(); @in.ReadLong(); @in.ReadLong(); @in.ReadLong(); @in.ReadInt(); FSImageSerialization.ReadString(@in); FSImageSerialization.ReadString(@in); WritableUtils.ReadEnum <DatanodeInfo.AdminStates>(@in); } v.LeaveEnclosingElement(); } // INodeUnderConstruction v.LeaveEnclosingElement(); }
// INode /// <exception cref="System.IO.IOException"/> private void ProcessINodeFileAttributes(DataInputStream @in, ImageVisitor v, string parentName) { string pathName = ReadINodePath(@in, parentName); v.Visit(ImageVisitor.ImageElement.InodePath, pathName); ProcessPermission(@in, v); v.Visit(ImageVisitor.ImageElement.ModificationTime, FormatDate(@in.ReadLong())); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FileAccessTime, imageVersion )) { v.Visit(ImageVisitor.ImageElement.AccessTime, FormatDate(@in.ReadLong())); } v.Visit(ImageVisitor.ImageElement.Replication, @in.ReadShort()); v.Visit(ImageVisitor.ImageElement.BlockSize, @in.ReadLong()); }
/// <exception cref="System.IO.IOException"/> private void ProcessFileDiff(DataInputStream @in, ImageVisitor v, string currentINodeName ) { int snapshotId = @in.ReadInt(); v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotFileDiff, ImageVisitor.ImageElement .SnapshotDiffSnapshotid, snapshotId); v.Visit(ImageVisitor.ImageElement.SnapshotFileSize, @in.ReadLong()); if (@in.ReadBoolean()) { v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotInodeFileAttributes); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.OptimizeSnapshotInodes, imageVersion)) { ProcessINodeFileAttributes(@in, v, currentINodeName); } else { ProcessINode(@in, v, true, currentINodeName, true); } v.LeaveEnclosingElement(); } v.LeaveEnclosingElement(); }
// Permissions /// <summary>Process the INode records stored in the fsimage.</summary> /// <param name="in">Datastream to process</param> /// <param name="v">Visitor to walk over INodes</param> /// <param name="numInodes">Number of INodes stored in file</param> /// <param name="skipBlocks">Process all the blocks within the INode?</param> /// <param name="supportSnapshot">Whether or not the imageVersion supports snapshot</param> /// <exception cref="VisitException"/> /// <exception cref="System.IO.IOException"/> private void ProcessINodes(DataInputStream @in, ImageVisitor v, long numInodes, bool skipBlocks, bool supportSnapshot) { v.VisitEnclosingElement(ImageVisitor.ImageElement.Inodes, ImageVisitor.ImageElement .NumInodes, numInodes); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FsimageNameOptimization, imageVersion)) { if (!supportSnapshot) { ProcessLocalNameINodes(@in, v, numInodes, skipBlocks); } else { ProcessLocalNameINodesWithSnapshot(@in, v, skipBlocks); } } else { // full path name ProcessFullNameINodes(@in, v, numInodes, skipBlocks); } v.LeaveEnclosingElement(); }
/// <summary>Process an INode</summary> /// <param name="in">image stream</param> /// <param name="v">visitor</param> /// <param name="skipBlocks">skip blocks or not</param> /// <param name="parentName">the name of its parent node</param> /// <param name="isSnapshotCopy">whether or not the inode is a snapshot copy</param> /// <exception cref="System.IO.IOException"/> private void ProcessINode(DataInputStream @in, ImageVisitor v, bool skipBlocks, string parentName, bool isSnapshotCopy) { bool supportSnapshot = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot , imageVersion); bool supportInodeId = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId , imageVersion); v.VisitEnclosingElement(ImageVisitor.ImageElement.Inode); string pathName = ReadINodePath(@in, parentName); v.Visit(ImageVisitor.ImageElement.InodePath, pathName); long inodeId = INodeId.GrandfatherInodeId; if (supportInodeId) { inodeId = @in.ReadLong(); v.Visit(ImageVisitor.ImageElement.InodeId, inodeId); } v.Visit(ImageVisitor.ImageElement.Replication, @in.ReadShort()); v.Visit(ImageVisitor.ImageElement.ModificationTime, FormatDate(@in.ReadLong())); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FileAccessTime, imageVersion )) { v.Visit(ImageVisitor.ImageElement.AccessTime, FormatDate(@in.ReadLong())); } v.Visit(ImageVisitor.ImageElement.BlockSize, @in.ReadLong()); int numBlocks = @in.ReadInt(); ProcessBlocks(@in, v, numBlocks, skipBlocks); if (numBlocks >= 0) { // File if (supportSnapshot) { // make sure subtreeMap only contains entry for directory Sharpen.Collections.Remove(subtreeMap, inodeId); // process file diffs ProcessFileDiffList(@in, v, parentName); if (isSnapshotCopy) { bool underConstruction = @in.ReadBoolean(); if (underConstruction) { v.Visit(ImageVisitor.ImageElement.ClientName, FSImageSerialization.ReadString(@in )); v.Visit(ImageVisitor.ImageElement.ClientMachine, FSImageSerialization.ReadString( @in)); } } } ProcessPermission(@in, v); } else { if (numBlocks == -1) { // Directory if (supportSnapshot && supportInodeId) { dirNodeMap[inodeId] = pathName; } v.Visit(ImageVisitor.ImageElement.NsQuota, numBlocks == -1 ? @in.ReadLong() : -1); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.DiskspaceQuota, imageVersion )) { v.Visit(ImageVisitor.ImageElement.DsQuota, numBlocks == -1 ? @in.ReadLong() : -1); } if (supportSnapshot) { bool snapshottable = @in.ReadBoolean(); if (!snapshottable) { bool withSnapshot = @in.ReadBoolean(); v.Visit(ImageVisitor.ImageElement.IsWithsnapshotDir, bool.ToString(withSnapshot)); } else { v.Visit(ImageVisitor.ImageElement.IsSnapshottableDir, bool.ToString(snapshottable )); } } ProcessPermission(@in, v); } else { if (numBlocks == -2) { v.Visit(ImageVisitor.ImageElement.Symlink, Text.ReadString(@in)); ProcessPermission(@in, v); } else { if (numBlocks == -3) { // reference node bool isWithName = @in.ReadBoolean(); int snapshotId = @in.ReadInt(); if (isWithName) { v.Visit(ImageVisitor.ImageElement.SnapshotLastSnapshotId, snapshotId); } else { v.Visit(ImageVisitor.ImageElement.SnapshotDstSnapshotId, snapshotId); } bool firstReferred = @in.ReadBoolean(); if (firstReferred) { // if a subtree is linked by multiple "parents", the corresponding dir // must be referred by a reference node. we put the reference node into // the subtreeMap here and let its value be false. when we later visit // the subtree for the first time, we change the value to true. subtreeMap[inodeId] = false; v.VisitEnclosingElement(ImageVisitor.ImageElement.SnapshotRefInode); ProcessINode(@in, v, skipBlocks, parentName, isSnapshotCopy); v.LeaveEnclosingElement(); } else { // referred inode v.Visit(ImageVisitor.ImageElement.SnapshotRefInodeId, @in.ReadLong()); } } } } } v.LeaveEnclosingElement(); }
/* (non-Javadoc) * @see ImageLoader#processImage(java.io.DataInputStream, ImageVisitor, boolean) */ /// <exception cref="System.IO.IOException"/> public override void LoadImage(DataInputStream @in, ImageVisitor v, bool skipBlocks ) { bool done = false; try { v.Start(); v.VisitEnclosingElement(ImageVisitor.ImageElement.FsImage); imageVersion = @in.ReadInt(); if (!CanLoadVersion(imageVersion)) { throw new IOException("Cannot process fslayout version " + imageVersion); } if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddLayoutFlags, imageVersion )) { LayoutFlags.Read(@in); } v.Visit(ImageVisitor.ImageElement.ImageVersion, imageVersion); v.Visit(ImageVisitor.ImageElement.NamespaceId, @in.ReadInt()); long numInodes = @in.ReadLong(); v.Visit(ImageVisitor.ImageElement.GenerationStamp, @in.ReadLong()); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.SequentialBlockId, imageVersion )) { v.Visit(ImageVisitor.ImageElement.GenerationStampV2, @in.ReadLong()); v.Visit(ImageVisitor.ImageElement.GenerationStampV1Limit, @in.ReadLong()); v.Visit(ImageVisitor.ImageElement.LastAllocatedBlockId, @in.ReadLong()); } if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.StoredTxids, imageVersion )) { v.Visit(ImageVisitor.ImageElement.TransactionId, @in.ReadLong()); } if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imageVersion )) { v.Visit(ImageVisitor.ImageElement.LastInodeId, @in.ReadLong()); } bool supportSnapshot = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot , imageVersion); if (supportSnapshot) { v.Visit(ImageVisitor.ImageElement.SnapshotCounter, @in.ReadInt()); int numSnapshots = @in.ReadInt(); v.Visit(ImageVisitor.ImageElement.NumSnapshotsTotal, numSnapshots); for (int i = 0; i < numSnapshots; i++) { ProcessSnapshot(@in, v); } } if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FsimageCompression, imageVersion )) { bool isCompressed = @in.ReadBoolean(); v.Visit(ImageVisitor.ImageElement.IsCompressed, isCompressed.ToString()); if (isCompressed) { string codecClassName = Text.ReadString(@in); v.Visit(ImageVisitor.ImageElement.CompressCodec, codecClassName); CompressionCodecFactory codecFac = new CompressionCodecFactory(new Configuration( )); CompressionCodec codec = codecFac.GetCodecByClassName(codecClassName); if (codec == null) { throw new IOException("Image compression codec not supported: " + codecClassName); } @in = new DataInputStream(codec.CreateInputStream(@in)); } } ProcessINodes(@in, v, numInodes, skipBlocks, supportSnapshot); subtreeMap.Clear(); dirNodeMap.Clear(); ProcessINodesUC(@in, v, skipBlocks); if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.DelegationToken, imageVersion )) { ProcessDelegationTokens(@in, v); } if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Caching, imageVersion)) { ProcessCacheManagerState(@in, v); } v.LeaveEnclosingElement(); // FSImage done = true; } finally { if (done) { v.Finish(); } else { v.FinishAbnormally(); } } }
public virtual void TestRelease1_2_0() { NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LayoutVersion.Feature .Concat, LayoutVersion.Feature.ReservedRel120.GetInfo().GetLayoutVersion())); }
public virtual void TestRelease204() { NUnit.Framework.Assert.IsTrue(NameNodeLayoutVersion.Supports(LayoutVersion.Feature .DelegationToken, LayoutVersion.Feature.ReservedRel20204.GetInfo().GetLayoutVersion ())); }