/* * corrupt files by removing and recreating the directory */ /// <exception cref="System.IO.IOException"/> private void CorruptNameNodeFiles() { // now corrupt/delete the directrory IList <URI> nameDirs = (IList <URI>)FSNamesystem.GetNamespaceDirs(config); IList <URI> nameEditsDirs = FSNamesystem.GetNamespaceEditsDirs(config); // get name dir and its length, then delete and recreate the directory FilePath dir = new FilePath(nameDirs[0].GetPath()); // has only one this.fsimageLength = new FilePath(new FilePath(dir, Storage.StorageDirCurrent), NNStorage.NameNodeFile .Image.GetName()).Length(); if (dir.Exists() && !(FileUtil.FullyDelete(dir))) { throw new IOException("Cannot remove directory: " + dir); } Log.Info("--removed dir " + dir + ";len was =" + this.fsimageLength); if (!dir.Mkdirs()) { throw new IOException("Cannot create directory " + dir); } dir = new FilePath(nameEditsDirs[0].GetPath()); //has only one this.editsLength = new FilePath(new FilePath(dir, Storage.StorageDirCurrent), NNStorage.NameNodeFile .Edits.GetName()).Length(); if (dir.Exists() && !(FileUtil.FullyDelete(dir))) { throw new IOException("Cannot remove directory: " + dir); } if (!dir.Mkdirs()) { throw new IOException("Cannot create directory " + dir); } Log.Info("--removed dir and recreated " + dir + ";len was =" + this.editsLength); }
internal SaveNamespaceContext(FSNamesystem sourceNamesystem, long txid, Canceler canceller) { this.sourceNamesystem = sourceNamesystem; this.txid = txid; this.canceller = canceller; }
/// <summary> /// Ensure that the given NameNode marks the specified DataNode as /// entirely dead/expired. /// </summary> /// <param name="nn">the NameNode to manipulate</param> /// <param name="dnName">the name of the DataNode</param> public static void NoticeDeadDatanode(NameNode nn, string dnName) { FSNamesystem namesystem = nn.GetNamesystem(); namesystem.WriteLock(); try { DatanodeManager dnm = namesystem.GetBlockManager().GetDatanodeManager(); HeartbeatManager hbm = dnm.GetHeartbeatManager(); DatanodeDescriptor[] dnds = hbm.GetDatanodes(); DatanodeDescriptor theDND = null; foreach (DatanodeDescriptor dnd in dnds) { if (dnd.GetXferAddr().Equals(dnName)) { theDND = dnd; } } NUnit.Framework.Assert.IsNotNull("Could not find DN with name: " + dnName, theDND ); lock (hbm) { DFSTestUtil.SetDatanodeDead(theDND); hbm.HeartbeatCheck(); } } finally { namesystem.WriteUnlock(); } }
public virtual void TestDelegationTokenRpc() { ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>(); FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>(); DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys. DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault , DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys ); sm.StartThreads(); Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol )).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(sm).Build(); server.Start(); UserGroupInformation current = UserGroupInformation.GetCurrentUser(); IPEndPoint addr = NetUtils.GetConnectAddress(server); string user = current.GetUserName(); Text owner = new Text(user); DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null ); Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(dtId, sm); SecurityUtil.SetTokenService(token, addr); Log.Info("Service for token is " + token.GetService()); current.AddToken(token); current.DoAs(new _PrivilegedExceptionAction_100(addr, server)); }
/// <exception cref="System.IO.IOException"/> internal static void DoMerge(CheckpointSignature sig, RemoteEditLogManifest manifest , bool loadImage, FSImage dstImage, FSNamesystem dstNamesystem) { NNStorage dstStorage = dstImage.GetStorage(); dstStorage.SetStorageInfo(sig); if (loadImage) { FilePath file = dstStorage.FindImageFile(NNStorage.NameNodeFile.Image, sig.mostRecentCheckpointTxId ); if (file == null) { throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId + " even though it should have " + "just been downloaded"); } dstNamesystem.WriteLock(); try { dstImage.ReloadFromImageFile(file, dstNamesystem); } finally { dstNamesystem.WriteUnlock(); } dstNamesystem.ImageLoadComplete(); } // error simulation code for junit test CheckpointFaultInjector.GetInstance().DuringMerge(); Checkpointer.RollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); // The following has the side effect of purging old fsimages/edit logs. dstImage.SaveFSImageInAllDirs(dstNamesystem, dstImage.GetLastAppliedTxId()); dstStorage.WriteAll(); }
/// <summary>Verify the support for decommissioning a datanode that is already dead.</summary> /// <remarks> /// Verify the support for decommissioning a datanode that is already dead. /// Under this scenario the datanode should immediately be marked as /// DECOMMISSIONED /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDecommissionDeadDN() { Logger log = Logger.GetLogger(typeof(DecommissionManager)); log.SetLevel(Level.Debug); DatanodeID dnID = cluster.GetDataNodes()[0].GetDatanodeId(); string dnName = dnID.GetXferAddr(); MiniDFSCluster.DataNodeProperties stoppedDN = cluster.StopDataNode(0); DFSTestUtil.WaitForDatanodeState(cluster, dnID.GetDatanodeUuid(), false, 30000); FSNamesystem fsn = cluster.GetNamesystem(); DatanodeManager dm = fsn.GetBlockManager().GetDatanodeManager(); DatanodeDescriptor dnDescriptor = dm.GetDatanode(dnID); DecommissionNode(fsn, localFileSys, dnName); dm.RefreshNodes(conf); BlockManagerTestUtil.RecheckDecommissionState(dm); NUnit.Framework.Assert.IsTrue(dnDescriptor.IsDecommissioned()); // Add the node back cluster.RestartDataNode(stoppedDN, true); cluster.WaitActive(); // Call refreshNodes on FSNamesystem with empty exclude file to remove the // datanode from decommissioning list and make it available again. WriteConfigFile(localFileSys, excludeFile, null); dm.RefreshNodes(conf); }
public virtual void TestWithFSNamesystemWriteLock() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; FSNamesystem fsn = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); fsn = cluster.GetNameNode().namesystem; fsn.WriteLock(); TestFSNamesystemMBean.MBeanClient client = new TestFSNamesystemMBean.MBeanClient( ); client.Start(); client.Join(20000); NUnit.Framework.Assert.IsTrue("JMX calls are blocked when FSNamesystem's writerlock" + "is owned by another thread", client.succeeded); client.Interrupt(); } finally { if (fsn != null && fsn.HasWriteLock()) { fsn.WriteUnlock(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void Test() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNameNode().namesystem; MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer(); ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState" ); string snapshotStats = (string)(mbs.GetAttribute(mxbeanName, "SnapshotStats")); IDictionary <string, object> stat = (IDictionary <string, object>)JSON.Parse(snapshotStats ); NUnit.Framework.Assert.IsTrue(stat.Contains("SnapshottableDirectories") && (long) stat["SnapshottableDirectories"] == fsn.GetNumSnapshottableDirs()); NUnit.Framework.Assert.IsTrue(stat.Contains("Snapshots") && (long)stat["Snapshots" ] == fsn.GetNumSnapshots()); object pendingDeletionBlocks = mbs.GetAttribute(mxbeanName, "PendingDeletionBlocks" ); NUnit.Framework.Assert.IsNotNull(pendingDeletionBlocks); NUnit.Framework.Assert.IsTrue(pendingDeletionBlocks is long); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestSufficientlySingleReplBlockUsesNewRack() { Configuration conf = GetConf(); short ReplicationFactor = 1; Path filePath = new Path("/testFile"); string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack2" }; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length ).Racks(racks).Build(); FSNamesystem ns = cluster.GetNameNode().GetNamesystem(); try { // Create a file with one block with a replication factor of 1 FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L); ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath); DFSTestUtil.WaitForReplication(cluster, b, 1, ReplicationFactor, 0); ReplicationFactor = 2; NameNodeAdapter.SetReplication(ns, "/testFile", ReplicationFactor); DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0); } finally { cluster.Shutdown(); } }
public virtual void TestReduceReplFactorRespectsRackPolicy() { Configuration conf = GetConf(); short ReplicationFactor = 3; Path filePath = new Path("/testFile"); string[] racks = new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length ).Racks(racks).Build(); FSNamesystem ns = cluster.GetNameNode().GetNamesystem(); try { // Create a file with one block FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L); ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath); DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0); // Decrease the replication factor, make sure the deleted replica // was not the one that lived on the rack with only one replica, // ie we should still have 2 racks after reducing the repl factor. ReplicationFactor = 2; NameNodeAdapter.SetReplication(ns, "/testFile", ReplicationFactor); DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0); } finally { cluster.Shutdown(); } }
public virtual void TestUnderReplicatedUsesNewRacks() { Configuration conf = GetConf(); short ReplicationFactor = 3; Path filePath = new Path("/testFile"); // All datanodes are on the same rack string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack1", "/rack1" }; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length ).Racks(racks).Build(); FSNamesystem ns = cluster.GetNameNode().GetNamesystem(); try { // Create a file with one block FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L); ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath); DFSTestUtil.WaitForReplication(cluster, b, 1, ReplicationFactor, 0); // Add new datanodes on a different rack and increase the // replication factor so the block is underreplicated and make // sure at least one of the hosts on the new rack is used. string[] newRacks = new string[] { "/rack2", "/rack2" }; cluster.StartDataNodes(conf, 2, true, null, newRacks); ReplicationFactor = 5; NameNodeAdapter.SetReplication(ns, "/testFile", ReplicationFactor); DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0); } finally { cluster.Shutdown(); } }
/// <summary> /// Confirm that FSImage files in all StorageDirectory are the same, /// and non-empty, and there are the expected number of them. /// </summary> /// <param name="fsn">- the FSNamesystem being checked.</param> /// <param name="numImageDirs">- the configured number of StorageDirectory of type IMAGE. /// </param> /// <returns>- the md5 hash of the most recent FSImage files, which must all be the same. /// </returns> /// <exception cref="System.Exception"> /// if image files are empty or different, /// if less than two StorageDirectory are provided, or if the /// actual number of StorageDirectory is less than configured. /// </exception> public static string CheckImages(FSNamesystem fsn, int numImageDirs) { NNStorage stg = fsn.GetFSImage().GetStorage(); //any failed StorageDirectory is removed from the storageDirs list NUnit.Framework.Assert.AreEqual("Some StorageDirectories failed Upgrade", numImageDirs , stg.GetNumStorageDirs(NNStorage.NameNodeDirType.Image)); NUnit.Framework.Assert.IsTrue("Not enough fsimage copies in MiniDFSCluster " + "to test parallel write" , numImageDirs > 1); // List of "current/" directory from each SD IList <FilePath> dirs = FSImageTestUtil.GetCurrentDirs(stg, NNStorage.NameNodeDirType .Image); // across directories, all files with same names should be identical hashes FSImageTestUtil.AssertParallelFilesAreIdentical(dirs, Sharpen.Collections.EmptySet <string>()); FSImageTestUtil.AssertSameNewestImage(dirs); // Return the hash of the newest image file Storage.StorageDirectory firstSd = stg.DirIterator(NNStorage.NameNodeDirType.Image ).Next(); FilePath latestImage = FSImageTestUtil.FindLatestImageFile(firstSd); string md5 = FSImageTestUtil.GetImageFileMD5IgnoringTxId(latestImage); System.Console.Error.WriteLine("md5 of " + latestImage + ": " + md5); return(md5); }
public virtual void TestChooseReplicaToDelete() { MiniDFSCluster cluster = null; FileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); fs = cluster.GetFileSystem(); FSNamesystem namesystem = cluster.GetNamesystem(); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300); cluster.StartDataNodes(conf, 1, true, null, null, null); DataNode lastDN = cluster.GetDataNodes()[3]; DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem .GetBlockPoolId()); string lastDNid = dnReg.GetDatanodeUuid(); Path fileName = new Path("/foo2"); DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L); DFSTestUtil.WaitReplication(fs, fileName, (short)4); // Wait for tolerable number of heartbeats plus one DatanodeDescriptor nodeInfo = null; long lastHeartbeat = 0; long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys .DfsNamenodeTolerateHeartbeatMultiplierDefault + 1); do { nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg); lastHeartbeat = nodeInfo.GetLastUpdateMonotonic(); }while (Time.MonotonicNow() - lastHeartbeat < waitTime); fs.SetReplication(fileName, (short)3); BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue ); // All replicas for deletion should be scheduled on lastDN. // And should not actually be deleted, because lastDN does not heartbeat. namesystem.ReadLock(); ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid ]; NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted" , SmallFileLength / SmallBlockSize, dnBlocks.Count); namesystem.ReadUnlock(); foreach (BlockLocation location in locs) { NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location .GetNames().Length); } } finally { if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> private void ParseConfAndFindOtherNN() { Configuration conf = GetConf(); nsId = DFSUtil.GetNamenodeNameServiceId(conf); if (!HAUtil.IsHAEnabled(conf, nsId)) { throw new HadoopIllegalArgumentException("HA is not enabled for this namenode."); } nnId = HAUtil.GetNameNodeId(conf, nsId); NameNode.InitializeGenericKeys(conf, nsId, nnId); if (!HAUtil.UsesSharedEditsDir(conf)) { throw new HadoopIllegalArgumentException("Shared edits storage is not enabled for this namenode." ); } Configuration otherNode = HAUtil.GetConfForOtherNode(conf); otherNNId = HAUtil.GetNameNodeId(otherNode, nsId); otherIpcAddr = NameNode.GetServiceAddress(otherNode, true); Preconditions.CheckArgument(otherIpcAddr.Port != 0 && !otherIpcAddr.Address.IsAnyLocalAddress (), "Could not determine valid IPC address for other NameNode (%s)" + ", got: %s" , otherNNId, otherIpcAddr); string scheme = DFSUtil.GetHttpClientScheme(conf); otherHttpAddr = DFSUtil.GetInfoServerWithDefaultHost(otherIpcAddr.GetHostName(), otherNode, scheme).ToURL(); dirsToFormat = FSNamesystem.GetNamespaceDirs(conf); editUrisToFormat = FSNamesystem.GetNamespaceEditsDirs(conf, false); sharedEditsUris = FSNamesystem.GetSharedEditsDirs(conf); }
/// <exception cref="System.IO.IOException"/> internal static BatchedRemoteIterator.BatchedListEntries <CachePoolEntry> ListCachePools (FSNamesystem fsn, CacheManager cacheManager, string prevKey) { FSPermissionChecker pc = GetFsPermissionChecker(fsn); return(cacheManager.ListCachePools(pc, prevKey)); }
public _Thread_371(FSNamesystem namesystem, AtomicReference <Exception> deferredException , CountDownLatch waitToEnterFlush) { this.namesystem = namesystem; this.deferredException = deferredException; this.waitToEnterFlush = waitToEnterFlush; }
public _Thread_467(FSNamesystem namesystem, AtomicReference <Exception> deferredException , CountDownLatch waitToEnterSync) { this.namesystem = namesystem; this.deferredException = deferredException; this.waitToEnterSync = waitToEnterSync; }
public virtual void TestReplQueuesActiveAfterStartupSafemode() { Configuration conf = new Configuration(); FSEditLog fsEditLog = Org.Mockito.Mockito.Mock <FSEditLog>(); FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>(); Org.Mockito.Mockito.When(fsImage.GetEditLog()).ThenReturn(fsEditLog); FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage); FSNamesystem fsn = Org.Mockito.Mockito.Spy(fsNamesystem); //Make shouldPopulaeReplQueues return true HAContext haContext = Org.Mockito.Mockito.Mock <HAContext>(); HAState haState = Org.Mockito.Mockito.Mock <HAState>(); Org.Mockito.Mockito.When(haContext.GetState()).ThenReturn(haState); Org.Mockito.Mockito.When(haState.ShouldPopulateReplQueues()).ThenReturn(true); Whitebox.SetInternalState(fsn, "haContext", haContext); //Make NameNode.getNameNodeMetrics() not return null NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode); fsn.EnterSafeMode(false); NUnit.Framework.Assert.IsTrue("FSNamesystem didn't enter safemode", fsn.IsInSafeMode ()); NUnit.Framework.Assert.IsTrue("Replication queues were being populated during very first " + "safemode", !fsn.IsPopulatingReplQueues()); fsn.LeaveSafeMode(); NUnit.Framework.Assert.IsTrue("FSNamesystem didn't leave safemode", !fsn.IsInSafeMode ()); NUnit.Framework.Assert.IsTrue("Replication queues weren't being populated even after leaving " + "safemode", fsn.IsPopulatingReplQueues()); fsn.EnterSafeMode(false); NUnit.Framework.Assert.IsTrue("FSNamesystem didn't enter safemode", fsn.IsInSafeMode ()); NUnit.Framework.Assert.IsTrue("Replication queues weren't being populated after entering " + "safemode 2nd time", fsn.IsPopulatingReplQueues()); }
public virtual void TestEnterSafeModeInSBNShouldNotThrowNPE() { Banner("Starting with NN0 active and NN1 standby, creating some blocks"); DFSTestUtil.CreateFile(fs, new Path("/test"), 3 * BlockSize, (short)3, 1L); // Roll edit log so that, when the SBN restarts, it will load // the namespace during startup and enter safemode. nn0.GetRpcServer().RollEditLog(); Banner("Creating some blocks that won't be in the edit log"); DFSTestUtil.CreateFile(fs, new Path("/test2"), 5 * BlockSize, (short)3, 1L); Banner("Deleting the original blocks"); fs.Delete(new Path("/test"), true); Banner("Restarting standby"); RestartStandby(); FSNamesystem namesystem = nn1.GetNamesystem(); string status = namesystem.GetSafemode(); NUnit.Framework.Assert.IsTrue("Bad safemode status: '" + status + "'", status.StartsWith ("Safe mode is ON.")); NameNodeAdapter.EnterSafeMode(nn1, false); NUnit.Framework.Assert.IsTrue("Failed to enter into safemode in standby", namesystem .IsInSafeMode()); NameNodeAdapter.EnterSafeMode(nn1, false); NUnit.Framework.Assert.IsTrue("Failed to enter into safemode in standby", namesystem .IsInSafeMode()); }
public static ReentrantReadWriteLock SpyOnFsLock(FSNamesystem fsn) { ReentrantReadWriteLock spy = Org.Mockito.Mockito.Spy(fsn.GetFsLockForTests()); fsn.SetFsLockForTests(spy); return(spy); }
/// <summary> /// Create a NameNodeResourceChecker, which will check the edits dirs and any /// additional dirs to check set in <code>conf</code>. /// </summary> /// <exception cref="System.IO.IOException"/> public NameNodeResourceChecker(Configuration conf) { this.conf = conf; volumes = new Dictionary <string, NameNodeResourceChecker.CheckedVolume>(); duReserved = conf.GetLong(DFSConfigKeys.DfsNamenodeDuReservedKey, DFSConfigKeys.DfsNamenodeDuReservedDefault ); ICollection <URI> extraCheckedVolumes = Util.StringCollectionAsURIs(conf.GetTrimmedStringCollection (DFSConfigKeys.DfsNamenodeCheckedVolumesKey)); ICollection <URI> localEditDirs = Collections2.Filter(FSNamesystem.GetNamespaceEditsDirs (conf), new _Predicate_121()); // Add all the local edits dirs, marking some as required if they are // configured as such. foreach (URI editsDirToCheck in localEditDirs) { AddDirToCheck(editsDirToCheck, FSNamesystem.GetRequiredNamespaceEditsDirs(conf).Contains (editsDirToCheck)); } // All extra checked volumes are marked "required" foreach (URI extraDirToCheck in extraCheckedVolumes) { AddDirToCheck(extraDirToCheck, true); } minimumRedundantVolumes = conf.GetInt(DFSConfigKeys.DfsNamenodeCheckedVolumesMinimumKey , DFSConfigKeys.DfsNamenodeCheckedVolumesMinimumDefault); }
public EditLogTailer(FSNamesystem namesystem, Configuration conf) { this.tailerThread = new EditLogTailer.EditLogTailerThread(this); this.conf = conf; this.namesystem = namesystem; this.editLog = namesystem.GetEditLog(); lastLoadTimeMs = Time.MonotonicNow(); logRollPeriodMs = conf.GetInt(DFSConfigKeys.DfsHaLogrollPeriodKey, DFSConfigKeys. DfsHaLogrollPeriodDefault) * 1000; if (logRollPeriodMs >= 0) { this.activeAddr = GetActiveNodeAddress(); Preconditions.CheckArgument(activeAddr.Port > 0, "Active NameNode must have an IPC port configured. " + "Got address '%s'", activeAddr); Log.Info("Will roll logs on active node at " + activeAddr + " every " + (logRollPeriodMs / 1000) + " seconds."); } else { Log.Info("Not going to trigger log rolls on active node because " + DFSConfigKeys .DfsHaLogrollPeriodKey + " is negative."); } sleepTimeMs = conf.GetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, DFSConfigKeys.DfsHaTaileditsPeriodDefault ) * 1000; Log.Debug("logRollPeriodMs=" + logRollPeriodMs + " sleepTime=" + sleepTimeMs); }
public virtual void TestCommitBlockSynchronization() { INodeFile file = MockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; ExtendedBlock lastBlock = new ExtendedBlock(); namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false , newTargets, null); // Repeat the call to make sure it does not throw namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false , newTargets, null); // Simulate 'completing' the block. BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1); completedBlockInfo.SetBlockCollection(file); completedBlockInfo.SetGenerationStamp(genStamp); Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock (Matchers.Any <Block>()); Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock(); // Repeat the call to make sure it does not throw namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false , newTargets, null); }
/// <exception cref="System.IO.IOException"/> private void SaveNameSystemSection(FsImageProto.FileSummary.Builder summary) { FSNamesystem fsn = context.GetSourceNamesystem(); OutputStream @out = sectionOutputStream; BlockIdManager blockIdManager = fsn.GetBlockIdManager(); FsImageProto.NameSystemSection.Builder b = FsImageProto.NameSystemSection.NewBuilder ().SetGenstampV1(blockIdManager.GetGenerationStampV1()).SetGenstampV1Limit(blockIdManager .GetGenerationStampV1Limit()).SetGenstampV2(blockIdManager.GetGenerationStampV2( )).SetLastAllocatedBlockId(blockIdManager.GetLastAllocatedBlockId()).SetTransactionId (context.GetTxId()); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock // from the actual saver thread, there's a potential of a // fairness-related deadlock. See the comments on HDFS-2223. b.SetNamespaceId(fsn.UnprotectedGetNamespaceInfo().GetNamespaceID()); if (fsn.IsRollingUpgrade()) { b.SetRollingUpgradeStartTime(fsn.GetRollingUpgradeInfo().GetStartTime()); } FsImageProto.NameSystemSection s = ((FsImageProto.NameSystemSection)b.Build()); s.WriteDelimitedTo(@out); CommitSection(summary, FSImageFormatProtobuf.SectionName.NsInfo); }
/// <summary> /// Delete a path from the name space /// Update the count at each ancestor directory with quota /// <br /> /// Note: This is to be used by /// <see cref="FSEditLog"/> /// only. /// <br /> /// </summary> /// <param name="src">a string representation of a path to an inode</param> /// <param name="mtime">the time the inode is removed</param> /// <exception cref="System.IO.IOException"/> internal static void DeleteForEditLog(FSDirectory fsd, string src, long mtime) { System.Diagnostics.Debug.Assert(fsd.HasWriteLock()); FSNamesystem fsn = fsd.GetFSNamesystem(); INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo(); IList <INode> removedINodes = new ChunkedArrayList <INode>(); INodesInPath iip = fsd.GetINodesInPath4Write(FSDirectory.NormalizePath(src), false ); if (!DeleteAllowed(iip, src)) { return; } IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>(); FSDirSnapshotOp.CheckSnapshot(iip.GetLastINode(), snapshottableDirs); long filesRemoved = UnprotectedDelete(fsd, iip, collectedBlocks, removedINodes, mtime ); fsn.RemoveSnapshottableDirs(snapshottableDirs); if (filesRemoved >= 0) { fsn.RemoveLeasesAndINodes(src, removedINodes, false); fsn.RemoveBlocksAndUpdateSafemodeTotal(collectedBlocks); } }
/// <summary>Remove a file/directory from the namespace.</summary> /// <remarks> /// Remove a file/directory from the namespace. /// <p> /// For large directories, deletion is incremental. The blocks under /// the directory are collected and deleted a small number at a time holding /// the /// <see cref="FSNamesystem"/> /// lock. /// <p> /// For small directory or file the deletion is done in one shot. /// </remarks> /// <exception cref="System.IO.IOException"/> internal static INode.BlocksMapUpdateInfo DeleteInternal(FSNamesystem fsn, string src, INodesInPath iip, bool logRetryCache) { System.Diagnostics.Debug.Assert(fsn.HasWriteLock()); if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* NameSystem.delete: " + src); } FSDirectory fsd = fsn.GetFSDirectory(); INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo(); IList <INode> removedINodes = new ChunkedArrayList <INode>(); long mtime = Time.Now(); // Unlink the target directory from directory tree long filesRemoved = Delete(fsd, iip, collectedBlocks, removedINodes, mtime); if (filesRemoved < 0) { return(null); } fsd.GetEditLog().LogDelete(src, mtime, logRetryCache); IncrDeletedFileCount(filesRemoved); fsn.RemoveLeasesAndINodes(src, removedINodes, true); if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* Namesystem.delete: " + src + " is removed"); } return(collectedBlocks); }
public virtual void TestInvalidateOverReplicatedBlock() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); try { FSNamesystem namesystem = cluster.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path(MiniDFSCluster.GetBaseDirectory(), "/foo1"); FSDataOutputStream @out = fs.Create(p, (short)2); @out.WriteBytes("HDFS-3119: " + p); @out.Hsync(); fs.SetReplication(p, (short)1); @out.Close(); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, p); NUnit.Framework.Assert.AreEqual("Expected only one live replica for the block", 1 , bm.CountNodes(block.GetLocalBlock()).LiveReplicas()); } finally { cluster.Shutdown(); } }
public virtual void TestRetryCacheRebuild() { DFSTestUtil.RunOperations(cluster, filesystem, conf, BlockSize, 0); FSNamesystem namesystem = cluster.GetNamesystem(); LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry> cacheSet = (LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry>)namesystem.GetRetryCache().GetCacheSet (); NUnit.Framework.Assert.AreEqual(25, cacheSet.Size()); IDictionary <RetryCache.CacheEntry, RetryCache.CacheEntry> oldEntries = new Dictionary <RetryCache.CacheEntry, RetryCache.CacheEntry>(); IEnumerator <RetryCache.CacheEntry> iter = cacheSet.GetEnumerator(); while (iter.HasNext()) { RetryCache.CacheEntry entry = iter.Next(); oldEntries[entry] = entry; } // restart NameNode cluster.RestartNameNode(); cluster.WaitActive(); namesystem = cluster.GetNamesystem(); // check retry cache NUnit.Framework.Assert.IsTrue(namesystem.HasRetryCache()); cacheSet = (LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry>)namesystem .GetRetryCache().GetCacheSet(); NUnit.Framework.Assert.AreEqual(25, cacheSet.Size()); iter = cacheSet.GetEnumerator(); while (iter.HasNext()) { RetryCache.CacheEntry entry = iter.Next(); NUnit.Framework.Assert.IsTrue(oldEntries.Contains(entry)); } }
public virtual void TestSaveImageWhileSyncInProgress() { Configuration conf = GetConf(); NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode); DFSTestUtil.FormatNameNode(conf); FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf); try { FSImage fsimage = namesystem.GetFSImage(); FSEditLog editLog = fsimage.GetEditLog(); JournalSet.JournalAndStream jas = editLog.GetJournals()[0]; EditLogFileOutputStream spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream )jas.GetCurrentStream()); jas.SetCurrentStreamForTests(spyElos); AtomicReference <Exception> deferredException = new AtomicReference <Exception>(); CountDownLatch waitToEnterFlush = new CountDownLatch(1); Sharpen.Thread doAnEditThread = new _Thread_371(namesystem, deferredException, waitToEnterFlush ); Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush); // Signal to main thread that the edit thread is in the racy section Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush(); doAnEditThread.Start(); // Wait for the edit thread to get to the logsync unsynchronized section Log.Info("Main thread: waiting to enter flush..."); waitToEnterFlush.Await(); NUnit.Framework.Assert.IsNull(deferredException.Get()); Log.Info("Main thread: detected that logSync is in unsynchronized section."); Log.Info("Trying to enter safe mode."); Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long" ); long st = Time.Now(); namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); long et = Time.Now(); Log.Info("Entered safe mode"); // Make sure we really waited for the flush to complete! NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000); // Once we're in safe mode, save namespace. namesystem.SaveNamespace(); Log.Info("Joining on edit thread..."); doAnEditThread.Join(); NUnit.Framework.Assert.IsNull(deferredException.Get()); // We did 3 edits: begin, txn, and end NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage. GetFinalizedEditsFileName(1, 3), 1)); // after the save, just the one "begin" NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage. GetInProgressEditsFileName(4), 4)); } finally { Log.Info("Closing nn"); if (namesystem != null) { namesystem.Close(); } } }
public virtual void TestArrayOutOfBoundsException() { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); Path FilePath = new Path("/tmp.txt"); long FileLen = 1L; DFSTestUtil.CreateFile(fs, FilePath, FileLen, (short)2, 1L); // get the block string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(0, 0); FilePath dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); NUnit.Framework.Assert.IsTrue("Data directory does not exist", dataDir.Exists()); ExtendedBlock blk = GetBlock(bpid, dataDir); if (blk == null) { storageDir = cluster.GetInstanceStorageDir(0, 1); dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); blk = GetBlock(bpid, dataDir); } NUnit.Framework.Assert.IsFalse("Data directory does not contain any blocks or there was an " + "IO error", blk == null); // start a third datanode cluster.StartDataNodes(conf, 1, true, null, null); AList <DataNode> datanodes = cluster.GetDataNodes(); NUnit.Framework.Assert.AreEqual(datanodes.Count, 3); DataNode dataNode = datanodes[2]; // report corrupted block by the third datanode DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(dataNode, blk .GetBlockPoolId()); FSNamesystem ns = cluster.GetNamesystem(); ns.WriteLock(); try { cluster.GetNamesystem().GetBlockManager().FindAndMarkBlockAsCorrupt(blk, new DatanodeInfo (dnR), "TEST", "STORAGE_ID"); } finally { ns.WriteUnlock(); } // open the file fs.Open(FilePath); //clean up fs.Delete(FilePath, false); } finally { if (cluster != null) { cluster.Shutdown(); } } }