/*
* corrupt files by removing and recreating the directory
*/
/// <exception cref="System.IO.IOException"/>
private void CorruptNameNodeFiles()
{
// now corrupt/delete the directrory
IList <URI> nameDirs = (IList <URI>)FSNamesystem.GetNamespaceDirs(config);
IList <URI> nameEditsDirs = FSNamesystem.GetNamespaceEditsDirs(config);
// get name dir and its length, then delete and recreate the directory
FilePath dir = new FilePath(nameDirs[0].GetPath());
// has only one
this.fsimageLength = new FilePath(new FilePath(dir, Storage.StorageDirCurrent), NNStorage.NameNodeFile
.Image.GetName()).Length();
if (dir.Exists() && !(FileUtil.FullyDelete(dir)))
{
throw new IOException("Cannot remove directory: " + dir);
}
Log.Info("--removed dir " + dir + ";len was =" + this.fsimageLength);
if (!dir.Mkdirs())
{
throw new IOException("Cannot create directory " + dir);
}
dir = new FilePath(nameEditsDirs[0].GetPath());
//has only one
this.editsLength = new FilePath(new FilePath(dir, Storage.StorageDirCurrent), NNStorage.NameNodeFile
.Edits.GetName()).Length();
if (dir.Exists() && !(FileUtil.FullyDelete(dir)))
{
throw new IOException("Cannot remove directory: " + dir);
}
if (!dir.Mkdirs())
{
throw new IOException("Cannot create directory " + dir);
}
Log.Info("--removed dir and recreated " + dir + ";len was =" + this.editsLength);
}
internal SaveNamespaceContext(FSNamesystem sourceNamesystem, long txid, Canceler
canceller)
{
this.sourceNamesystem = sourceNamesystem;
this.txid = txid;
this.canceller = canceller;
}
/// <summary>
/// Ensure that the given NameNode marks the specified DataNode as
/// entirely dead/expired.
/// </summary>
/// <param name="nn">the NameNode to manipulate</param>
/// <param name="dnName">the name of the DataNode</param>
public static void NoticeDeadDatanode(NameNode nn, string dnName)
{
FSNamesystem namesystem = nn.GetNamesystem();
namesystem.WriteLock();
try
{
DatanodeManager dnm = namesystem.GetBlockManager().GetDatanodeManager();
HeartbeatManager hbm = dnm.GetHeartbeatManager();
DatanodeDescriptor[] dnds = hbm.GetDatanodes();
DatanodeDescriptor theDND = null;
foreach (DatanodeDescriptor dnd in dnds)
{
if (dnd.GetXferAddr().Equals(dnName))
{
theDND = dnd;
}
}
NUnit.Framework.Assert.IsNotNull("Could not find DN with name: " + dnName, theDND
);
lock (hbm)
{
DFSTestUtil.SetDatanodeDead(theDND);
hbm.HeartbeatCheck();
}
}
finally
{
namesystem.WriteUnlock();
}
}
public virtual void TestDelegationTokenRpc()
{
ClientProtocol mockNN = Org.Mockito.Mockito.Mock <ClientProtocol>();
FSNamesystem mockNameSys = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager sm = new DelegationTokenSecretManager(DFSConfigKeys.
DfsNamenodeDelegationKeyUpdateIntervalDefault, DFSConfigKeys.DfsNamenodeDelegationKeyUpdateIntervalDefault
, DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeDefault, 3600000, mockNameSys
);
sm.StartThreads();
Org.Apache.Hadoop.Ipc.Server server = new RPC.Builder(conf).SetProtocol(typeof(ClientProtocol
)).SetInstance(mockNN).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string user = current.GetUserName();
Text owner = new Text(user);
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null
);
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, sm);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_100(addr, server));
}
/// <exception cref="System.IO.IOException"/>
internal static void DoMerge(CheckpointSignature sig, RemoteEditLogManifest manifest
, bool loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
{
NNStorage dstStorage = dstImage.GetStorage();
dstStorage.SetStorageInfo(sig);
if (loadImage)
{
FilePath file = dstStorage.FindImageFile(NNStorage.NameNodeFile.Image, sig.mostRecentCheckpointTxId
);
if (file == null)
{
throw new IOException("Couldn't find image file at txid " + sig.mostRecentCheckpointTxId
+ " even though it should have " + "just been downloaded");
}
dstNamesystem.WriteLock();
try
{
dstImage.ReloadFromImageFile(file, dstNamesystem);
}
finally
{
dstNamesystem.WriteUnlock();
}
dstNamesystem.ImageLoadComplete();
}
// error simulation code for junit test
CheckpointFaultInjector.GetInstance().DuringMerge();
Checkpointer.RollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
// The following has the side effect of purging old fsimages/edit logs.
dstImage.SaveFSImageInAllDirs(dstNamesystem, dstImage.GetLastAppliedTxId());
dstStorage.WriteAll();
}
/// <summary>Verify the support for decommissioning a datanode that is already dead.</summary>
/// <remarks>
/// Verify the support for decommissioning a datanode that is already dead.
/// Under this scenario the datanode should immediately be marked as
/// DECOMMISSIONED
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDecommissionDeadDN()
{
Logger log = Logger.GetLogger(typeof(DecommissionManager));
log.SetLevel(Level.Debug);
DatanodeID dnID = cluster.GetDataNodes()[0].GetDatanodeId();
string dnName = dnID.GetXferAddr();
MiniDFSCluster.DataNodeProperties stoppedDN = cluster.StopDataNode(0);
DFSTestUtil.WaitForDatanodeState(cluster, dnID.GetDatanodeUuid(), false, 30000);
FSNamesystem fsn = cluster.GetNamesystem();
DatanodeManager dm = fsn.GetBlockManager().GetDatanodeManager();
DatanodeDescriptor dnDescriptor = dm.GetDatanode(dnID);
DecommissionNode(fsn, localFileSys, dnName);
dm.RefreshNodes(conf);
BlockManagerTestUtil.RecheckDecommissionState(dm);
NUnit.Framework.Assert.IsTrue(dnDescriptor.IsDecommissioned());
// Add the node back
cluster.RestartDataNode(stoppedDN, true);
cluster.WaitActive();
// Call refreshNodes on FSNamesystem with empty exclude file to remove the
// datanode from decommissioning list and make it available again.
WriteConfigFile(localFileSys, excludeFile, null);
dm.RefreshNodes(conf);
}
public virtual void TestWithFSNamesystemWriteLock()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
FSNamesystem fsn = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
fsn = cluster.GetNameNode().namesystem;
fsn.WriteLock();
TestFSNamesystemMBean.MBeanClient client = new TestFSNamesystemMBean.MBeanClient(
);
client.Start();
client.Join(20000);
NUnit.Framework.Assert.IsTrue("JMX calls are blocked when FSNamesystem's writerlock"
+ "is owned by another thread", client.succeeded);
client.Interrupt();
}
finally
{
if (fsn != null && fsn.HasWriteLock())
{
fsn.WriteUnlock();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void Test()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNameNode().namesystem;
MBeanServer mbs = ManagementFactory.GetPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState"
);
string snapshotStats = (string)(mbs.GetAttribute(mxbeanName, "SnapshotStats"));
IDictionary <string, object> stat = (IDictionary <string, object>)JSON.Parse(snapshotStats
);
NUnit.Framework.Assert.IsTrue(stat.Contains("SnapshottableDirectories") && (long)
stat["SnapshottableDirectories"] == fsn.GetNumSnapshottableDirs());
NUnit.Framework.Assert.IsTrue(stat.Contains("Snapshots") && (long)stat["Snapshots"
] == fsn.GetNumSnapshots());
object pendingDeletionBlocks = mbs.GetAttribute(mxbeanName, "PendingDeletionBlocks"
);
NUnit.Framework.Assert.IsNotNull(pendingDeletionBlocks);
NUnit.Framework.Assert.IsTrue(pendingDeletionBlocks is long);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSufficientlySingleReplBlockUsesNewRack()
{
Configuration conf = GetConf();
short ReplicationFactor = 1;
Path filePath = new Path("/testFile");
string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack2" };
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length
).Racks(racks).Build();
FSNamesystem ns = cluster.GetNameNode().GetNamesystem();
try
{
// Create a file with one block with a replication factor of 1
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L);
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath);
DFSTestUtil.WaitForReplication(cluster, b, 1, ReplicationFactor, 0);
ReplicationFactor = 2;
NameNodeAdapter.SetReplication(ns, "/testFile", ReplicationFactor);
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestReduceReplFactorRespectsRackPolicy()
{
Configuration conf = GetConf();
short ReplicationFactor = 3;
Path filePath = new Path("/testFile");
string[] racks = new string[] { "/rack1", "/rack1", "/rack2", "/rack2" };
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length
).Racks(racks).Build();
FSNamesystem ns = cluster.GetNameNode().GetNamesystem();
try
{
// Create a file with one block
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L);
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath);
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
// Decrease the replication factor, make sure the deleted replica
// was not the one that lived on the rack with only one replica,
// ie we should still have 2 racks after reducing the repl factor.
ReplicationFactor = 2;
NameNodeAdapter.SetReplication(ns, "/testFile", ReplicationFactor);
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestUnderReplicatedUsesNewRacks()
{
Configuration conf = GetConf();
short ReplicationFactor = 3;
Path filePath = new Path("/testFile");
// All datanodes are on the same rack
string[] racks = new string[] { "/rack1", "/rack1", "/rack1", "/rack1", "/rack1" };
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length
).Racks(racks).Build();
FSNamesystem ns = cluster.GetNameNode().GetNamesystem();
try
{
// Create a file with one block
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, filePath, 1L, ReplicationFactor, 1L);
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath);
DFSTestUtil.WaitForReplication(cluster, b, 1, ReplicationFactor, 0);
// Add new datanodes on a different rack and increase the
// replication factor so the block is underreplicated and make
// sure at least one of the hosts on the new rack is used.
string[] newRacks = new string[] { "/rack2", "/rack2" };
cluster.StartDataNodes(conf, 2, true, null, newRacks);
ReplicationFactor = 5;
NameNodeAdapter.SetReplication(ns, "/testFile", ReplicationFactor);
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Confirm that FSImage files in all StorageDirectory are the same,
/// and non-empty, and there are the expected number of them.
/// </summary>
/// <param name="fsn">- the FSNamesystem being checked.</param>
/// <param name="numImageDirs">- the configured number of StorageDirectory of type IMAGE.
/// </param>
/// <returns>- the md5 hash of the most recent FSImage files, which must all be the same.
/// </returns>
/// <exception cref="System.Exception">
/// if image files are empty or different,
/// if less than two StorageDirectory are provided, or if the
/// actual number of StorageDirectory is less than configured.
/// </exception>
public static string CheckImages(FSNamesystem fsn, int numImageDirs)
{
NNStorage stg = fsn.GetFSImage().GetStorage();
//any failed StorageDirectory is removed from the storageDirs list
NUnit.Framework.Assert.AreEqual("Some StorageDirectories failed Upgrade", numImageDirs
, stg.GetNumStorageDirs(NNStorage.NameNodeDirType.Image));
NUnit.Framework.Assert.IsTrue("Not enough fsimage copies in MiniDFSCluster " + "to test parallel write"
, numImageDirs > 1);
// List of "current/" directory from each SD
IList <FilePath> dirs = FSImageTestUtil.GetCurrentDirs(stg, NNStorage.NameNodeDirType
.Image);
// across directories, all files with same names should be identical hashes
FSImageTestUtil.AssertParallelFilesAreIdentical(dirs, Sharpen.Collections.EmptySet
<string>());
FSImageTestUtil.AssertSameNewestImage(dirs);
// Return the hash of the newest image file
Storage.StorageDirectory firstSd = stg.DirIterator(NNStorage.NameNodeDirType.Image
).Next();
FilePath latestImage = FSImageTestUtil.FindLatestImageFile(firstSd);
string md5 = FSImageTestUtil.GetImageFileMD5IgnoringTxId(latestImage);
System.Console.Error.WriteLine("md5 of " + latestImage + ": " + md5);
return(md5);
}
public virtual void TestChooseReplicaToDelete()
{
MiniDFSCluster cluster = null;
FileSystem fs = null;
try
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300);
cluster.StartDataNodes(conf, 1, true, null, null, null);
DataNode lastDN = cluster.GetDataNodes()[3];
DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem
.GetBlockPoolId());
string lastDNid = dnReg.GetDatanodeUuid();
Path fileName = new Path("/foo2");
DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)4);
// Wait for tolerable number of heartbeats plus one
DatanodeDescriptor nodeInfo = null;
long lastHeartbeat = 0;
long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys
.DfsNamenodeTolerateHeartbeatMultiplierDefault + 1);
do
{
nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg);
lastHeartbeat = nodeInfo.GetLastUpdateMonotonic();
}while (Time.MonotonicNow() - lastHeartbeat < waitTime);
fs.SetReplication(fileName, (short)3);
BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue
);
// All replicas for deletion should be scheduled on lastDN.
// And should not actually be deleted, because lastDN does not heartbeat.
namesystem.ReadLock();
ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid
];
NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted"
, SmallFileLength / SmallBlockSize, dnBlocks.Count);
namesystem.ReadUnlock();
foreach (BlockLocation location in locs)
{
NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location
.GetNames().Length);
}
}
finally
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void ParseConfAndFindOtherNN()
{
Configuration conf = GetConf();
nsId = DFSUtil.GetNamenodeNameServiceId(conf);
if (!HAUtil.IsHAEnabled(conf, nsId))
{
throw new HadoopIllegalArgumentException("HA is not enabled for this namenode.");
}
nnId = HAUtil.GetNameNodeId(conf, nsId);
NameNode.InitializeGenericKeys(conf, nsId, nnId);
if (!HAUtil.UsesSharedEditsDir(conf))
{
throw new HadoopIllegalArgumentException("Shared edits storage is not enabled for this namenode."
);
}
Configuration otherNode = HAUtil.GetConfForOtherNode(conf);
otherNNId = HAUtil.GetNameNodeId(otherNode, nsId);
otherIpcAddr = NameNode.GetServiceAddress(otherNode, true);
Preconditions.CheckArgument(otherIpcAddr.Port != 0 && !otherIpcAddr.Address.IsAnyLocalAddress
(), "Could not determine valid IPC address for other NameNode (%s)" + ", got: %s"
, otherNNId, otherIpcAddr);
string scheme = DFSUtil.GetHttpClientScheme(conf);
otherHttpAddr = DFSUtil.GetInfoServerWithDefaultHost(otherIpcAddr.GetHostName(),
otherNode, scheme).ToURL();
dirsToFormat = FSNamesystem.GetNamespaceDirs(conf);
editUrisToFormat = FSNamesystem.GetNamespaceEditsDirs(conf, false);
sharedEditsUris = FSNamesystem.GetSharedEditsDirs(conf);
}
/// <exception cref="System.IO.IOException"/>
internal static BatchedRemoteIterator.BatchedListEntries <CachePoolEntry> ListCachePools
(FSNamesystem fsn, CacheManager cacheManager, string prevKey)
{
FSPermissionChecker pc = GetFsPermissionChecker(fsn);
return(cacheManager.ListCachePools(pc, prevKey));
}
public _Thread_371(FSNamesystem namesystem, AtomicReference <Exception> deferredException
, CountDownLatch waitToEnterFlush)
{
this.namesystem = namesystem;
this.deferredException = deferredException;
this.waitToEnterFlush = waitToEnterFlush;
}
public _Thread_467(FSNamesystem namesystem, AtomicReference <Exception> deferredException
, CountDownLatch waitToEnterSync)
{
this.namesystem = namesystem;
this.deferredException = deferredException;
this.waitToEnterSync = waitToEnterSync;
}
public virtual void TestReplQueuesActiveAfterStartupSafemode()
{
Configuration conf = new Configuration();
FSEditLog fsEditLog = Org.Mockito.Mockito.Mock <FSEditLog>();
FSImage fsImage = Org.Mockito.Mockito.Mock <FSImage>();
Org.Mockito.Mockito.When(fsImage.GetEditLog()).ThenReturn(fsEditLog);
FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage);
FSNamesystem fsn = Org.Mockito.Mockito.Spy(fsNamesystem);
//Make shouldPopulaeReplQueues return true
HAContext haContext = Org.Mockito.Mockito.Mock <HAContext>();
HAState haState = Org.Mockito.Mockito.Mock <HAState>();
Org.Mockito.Mockito.When(haContext.GetState()).ThenReturn(haState);
Org.Mockito.Mockito.When(haState.ShouldPopulateReplQueues()).ThenReturn(true);
Whitebox.SetInternalState(fsn, "haContext", haContext);
//Make NameNode.getNameNodeMetrics() not return null
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
fsn.EnterSafeMode(false);
NUnit.Framework.Assert.IsTrue("FSNamesystem didn't enter safemode", fsn.IsInSafeMode
());
NUnit.Framework.Assert.IsTrue("Replication queues were being populated during very first "
+ "safemode", !fsn.IsPopulatingReplQueues());
fsn.LeaveSafeMode();
NUnit.Framework.Assert.IsTrue("FSNamesystem didn't leave safemode", !fsn.IsInSafeMode
());
NUnit.Framework.Assert.IsTrue("Replication queues weren't being populated even after leaving "
+ "safemode", fsn.IsPopulatingReplQueues());
fsn.EnterSafeMode(false);
NUnit.Framework.Assert.IsTrue("FSNamesystem didn't enter safemode", fsn.IsInSafeMode
());
NUnit.Framework.Assert.IsTrue("Replication queues weren't being populated after entering "
+ "safemode 2nd time", fsn.IsPopulatingReplQueues());
}
public virtual void TestEnterSafeModeInSBNShouldNotThrowNPE()
{
Banner("Starting with NN0 active and NN1 standby, creating some blocks");
DFSTestUtil.CreateFile(fs, new Path("/test"), 3 * BlockSize, (short)3, 1L);
// Roll edit log so that, when the SBN restarts, it will load
// the namespace during startup and enter safemode.
nn0.GetRpcServer().RollEditLog();
Banner("Creating some blocks that won't be in the edit log");
DFSTestUtil.CreateFile(fs, new Path("/test2"), 5 * BlockSize, (short)3, 1L);
Banner("Deleting the original blocks");
fs.Delete(new Path("/test"), true);
Banner("Restarting standby");
RestartStandby();
FSNamesystem namesystem = nn1.GetNamesystem();
string status = namesystem.GetSafemode();
NUnit.Framework.Assert.IsTrue("Bad safemode status: '" + status + "'", status.StartsWith
("Safe mode is ON."));
NameNodeAdapter.EnterSafeMode(nn1, false);
NUnit.Framework.Assert.IsTrue("Failed to enter into safemode in standby", namesystem
.IsInSafeMode());
NameNodeAdapter.EnterSafeMode(nn1, false);
NUnit.Framework.Assert.IsTrue("Failed to enter into safemode in standby", namesystem
.IsInSafeMode());
}
public static ReentrantReadWriteLock SpyOnFsLock(FSNamesystem fsn)
{
ReentrantReadWriteLock spy = Org.Mockito.Mockito.Spy(fsn.GetFsLockForTests());
fsn.SetFsLockForTests(spy);
return(spy);
}
/// <summary>
/// Create a NameNodeResourceChecker, which will check the edits dirs and any
/// additional dirs to check set in <code>conf</code>.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public NameNodeResourceChecker(Configuration conf)
{
this.conf = conf;
volumes = new Dictionary <string, NameNodeResourceChecker.CheckedVolume>();
duReserved = conf.GetLong(DFSConfigKeys.DfsNamenodeDuReservedKey, DFSConfigKeys.DfsNamenodeDuReservedDefault
);
ICollection <URI> extraCheckedVolumes = Util.StringCollectionAsURIs(conf.GetTrimmedStringCollection
(DFSConfigKeys.DfsNamenodeCheckedVolumesKey));
ICollection <URI> localEditDirs = Collections2.Filter(FSNamesystem.GetNamespaceEditsDirs
(conf), new _Predicate_121());
// Add all the local edits dirs, marking some as required if they are
// configured as such.
foreach (URI editsDirToCheck in localEditDirs)
{
AddDirToCheck(editsDirToCheck, FSNamesystem.GetRequiredNamespaceEditsDirs(conf).Contains
(editsDirToCheck));
}
// All extra checked volumes are marked "required"
foreach (URI extraDirToCheck in extraCheckedVolumes)
{
AddDirToCheck(extraDirToCheck, true);
}
minimumRedundantVolumes = conf.GetInt(DFSConfigKeys.DfsNamenodeCheckedVolumesMinimumKey
, DFSConfigKeys.DfsNamenodeCheckedVolumesMinimumDefault);
}
public EditLogTailer(FSNamesystem namesystem, Configuration conf)
{
this.tailerThread = new EditLogTailer.EditLogTailerThread(this);
this.conf = conf;
this.namesystem = namesystem;
this.editLog = namesystem.GetEditLog();
lastLoadTimeMs = Time.MonotonicNow();
logRollPeriodMs = conf.GetInt(DFSConfigKeys.DfsHaLogrollPeriodKey, DFSConfigKeys.
DfsHaLogrollPeriodDefault) * 1000;
if (logRollPeriodMs >= 0)
{
this.activeAddr = GetActiveNodeAddress();
Preconditions.CheckArgument(activeAddr.Port > 0, "Active NameNode must have an IPC port configured. "
+ "Got address '%s'", activeAddr);
Log.Info("Will roll logs on active node at " + activeAddr + " every " + (logRollPeriodMs
/ 1000) + " seconds.");
}
else
{
Log.Info("Not going to trigger log rolls on active node because " + DFSConfigKeys
.DfsHaLogrollPeriodKey + " is negative.");
}
sleepTimeMs = conf.GetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, DFSConfigKeys.DfsHaTaileditsPeriodDefault
) * 1000;
Log.Debug("logRollPeriodMs=" + logRollPeriodMs + " sleepTime=" + sleepTimeMs);
}
public virtual void TestCommitBlockSynchronization()
{
INodeFile file = MockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = MakeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
// Simulate 'completing' the block.
BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short)1);
completedBlockInfo.SetBlockCollection(file);
completedBlockInfo.SetGenerationStamp(genStamp);
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(namesystemSpy).GetStoredBlock
(Matchers.Any <Block>());
Org.Mockito.Mockito.DoReturn(completedBlockInfo).When(file).GetLastBlock();
// Repeat the call to make sure it does not throw
namesystemSpy.CommitBlockSynchronization(lastBlock, genStamp, length, false, false
, newTargets, null);
}
/// <exception cref="System.IO.IOException"/>
private void SaveNameSystemSection(FsImageProto.FileSummary.Builder summary)
{
FSNamesystem fsn = context.GetSourceNamesystem();
OutputStream @out = sectionOutputStream;
BlockIdManager blockIdManager = fsn.GetBlockIdManager();
FsImageProto.NameSystemSection.Builder b = FsImageProto.NameSystemSection.NewBuilder
().SetGenstampV1(blockIdManager.GetGenerationStampV1()).SetGenstampV1Limit(blockIdManager
.GetGenerationStampV1Limit()).SetGenstampV2(blockIdManager.GetGenerationStampV2(
)).SetLastAllocatedBlockId(blockIdManager.GetLastAllocatedBlockId()).SetTransactionId
(context.GetTxId());
// We use the non-locked version of getNamespaceInfo here since
// the coordinating thread of saveNamespace already has read-locked
// the namespace for us. If we attempt to take another readlock
// from the actual saver thread, there's a potential of a
// fairness-related deadlock. See the comments on HDFS-2223.
b.SetNamespaceId(fsn.UnprotectedGetNamespaceInfo().GetNamespaceID());
if (fsn.IsRollingUpgrade())
{
b.SetRollingUpgradeStartTime(fsn.GetRollingUpgradeInfo().GetStartTime());
}
FsImageProto.NameSystemSection s = ((FsImageProto.NameSystemSection)b.Build());
s.WriteDelimitedTo(@out);
CommitSection(summary, FSImageFormatProtobuf.SectionName.NsInfo);
}
/// <summary>
/// Delete a path from the name space
/// Update the count at each ancestor directory with quota
/// <br />
/// Note: This is to be used by
/// <see cref="FSEditLog"/>
/// only.
/// <br />
/// </summary>
/// <param name="src">a string representation of a path to an inode</param>
/// <param name="mtime">the time the inode is removed</param>
/// <exception cref="System.IO.IOException"/>
internal static void DeleteForEditLog(FSDirectory fsd, string src, long mtime)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
FSNamesystem fsn = fsd.GetFSNamesystem();
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
IList <INode> removedINodes = new ChunkedArrayList <INode>();
INodesInPath iip = fsd.GetINodesInPath4Write(FSDirectory.NormalizePath(src), false
);
if (!DeleteAllowed(iip, src))
{
return;
}
IList <INodeDirectory> snapshottableDirs = new AList <INodeDirectory>();
FSDirSnapshotOp.CheckSnapshot(iip.GetLastINode(), snapshottableDirs);
long filesRemoved = UnprotectedDelete(fsd, iip, collectedBlocks, removedINodes, mtime
);
fsn.RemoveSnapshottableDirs(snapshottableDirs);
if (filesRemoved >= 0)
{
fsn.RemoveLeasesAndINodes(src, removedINodes, false);
fsn.RemoveBlocksAndUpdateSafemodeTotal(collectedBlocks);
}
}
/// <summary>Remove a file/directory from the namespace.</summary>
/// <remarks>
/// Remove a file/directory from the namespace.
/// <p>
/// For large directories, deletion is incremental. The blocks under
/// the directory are collected and deleted a small number at a time holding
/// the
/// <see cref="FSNamesystem"/>
/// lock.
/// <p>
/// For small directory or file the deletion is done in one shot.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static INode.BlocksMapUpdateInfo DeleteInternal(FSNamesystem fsn, string
src, INodesInPath iip, bool logRetryCache)
{
System.Diagnostics.Debug.Assert(fsn.HasWriteLock());
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* NameSystem.delete: " + src);
}
FSDirectory fsd = fsn.GetFSDirectory();
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
IList <INode> removedINodes = new ChunkedArrayList <INode>();
long mtime = Time.Now();
// Unlink the target directory from directory tree
long filesRemoved = Delete(fsd, iip, collectedBlocks, removedINodes, mtime);
if (filesRemoved < 0)
{
return(null);
}
fsd.GetEditLog().LogDelete(src, mtime, logRetryCache);
IncrDeletedFileCount(filesRemoved);
fsn.RemoveLeasesAndINodes(src, removedINodes, true);
if (NameNode.stateChangeLog.IsDebugEnabled())
{
NameNode.stateChangeLog.Debug("DIR* Namesystem.delete: " + src + " is removed");
}
return(collectedBlocks);
}
public virtual void TestInvalidateOverReplicatedBlock()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
try
{
FSNamesystem namesystem = cluster.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path(MiniDFSCluster.GetBaseDirectory(), "/foo1");
FSDataOutputStream @out = fs.Create(p, (short)2);
@out.WriteBytes("HDFS-3119: " + p);
@out.Hsync();
fs.SetReplication(p, (short)1);
@out.Close();
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, p);
NUnit.Framework.Assert.AreEqual("Expected only one live replica for the block", 1
, bm.CountNodes(block.GetLocalBlock()).LiveReplicas());
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestRetryCacheRebuild()
{
DFSTestUtil.RunOperations(cluster, filesystem, conf, BlockSize, 0);
FSNamesystem namesystem = cluster.GetNamesystem();
LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry> cacheSet = (LightWeightCache
<RetryCache.CacheEntry, RetryCache.CacheEntry>)namesystem.GetRetryCache().GetCacheSet
();
NUnit.Framework.Assert.AreEqual(25, cacheSet.Size());
IDictionary <RetryCache.CacheEntry, RetryCache.CacheEntry> oldEntries = new Dictionary
<RetryCache.CacheEntry, RetryCache.CacheEntry>();
IEnumerator <RetryCache.CacheEntry> iter = cacheSet.GetEnumerator();
while (iter.HasNext())
{
RetryCache.CacheEntry entry = iter.Next();
oldEntries[entry] = entry;
}
// restart NameNode
cluster.RestartNameNode();
cluster.WaitActive();
namesystem = cluster.GetNamesystem();
// check retry cache
NUnit.Framework.Assert.IsTrue(namesystem.HasRetryCache());
cacheSet = (LightWeightCache <RetryCache.CacheEntry, RetryCache.CacheEntry>)namesystem
.GetRetryCache().GetCacheSet();
NUnit.Framework.Assert.AreEqual(25, cacheSet.Size());
iter = cacheSet.GetEnumerator();
while (iter.HasNext())
{
RetryCache.CacheEntry entry = iter.Next();
NUnit.Framework.Assert.IsTrue(oldEntries.Contains(entry));
}
}
public virtual void TestSaveImageWhileSyncInProgress()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
JournalSet.JournalAndStream jas = editLog.GetJournals()[0];
EditLogFileOutputStream spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream
)jas.GetCurrentStream());
jas.SetCurrentStreamForTests(spyElos);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterFlush = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_371(namesystem, deferredException, waitToEnterFlush
);
Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush);
// Signal to main thread that the edit thread is in the racy section
Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush();
doAnEditThread.Start();
// Wait for the edit thread to get to the logsync unsynchronized section
Log.Info("Main thread: waiting to enter flush...");
waitToEnterFlush.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync is in unsynchronized section.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
public virtual void TestArrayOutOfBoundsException()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
Path FilePath = new Path("/tmp.txt");
long FileLen = 1L;
DFSTestUtil.CreateFile(fs, FilePath, FileLen, (short)2, 1L);
// get the block
string bpid = cluster.GetNamesystem().GetBlockPoolId();
FilePath storageDir = cluster.GetInstanceStorageDir(0, 0);
FilePath dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
NUnit.Framework.Assert.IsTrue("Data directory does not exist", dataDir.Exists());
ExtendedBlock blk = GetBlock(bpid, dataDir);
if (blk == null)
{
storageDir = cluster.GetInstanceStorageDir(0, 1);
dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
blk = GetBlock(bpid, dataDir);
}
NUnit.Framework.Assert.IsFalse("Data directory does not contain any blocks or there was an "
+ "IO error", blk == null);
// start a third datanode
cluster.StartDataNodes(conf, 1, true, null, null);
AList <DataNode> datanodes = cluster.GetDataNodes();
NUnit.Framework.Assert.AreEqual(datanodes.Count, 3);
DataNode dataNode = datanodes[2];
// report corrupted block by the third datanode
DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(dataNode, blk
.GetBlockPoolId());
FSNamesystem ns = cluster.GetNamesystem();
ns.WriteLock();
try
{
cluster.GetNamesystem().GetBlockManager().FindAndMarkBlockAsCorrupt(blk, new DatanodeInfo
(dnR), "TEST", "STORAGE_ID");
}
finally
{
ns.WriteUnlock();
}
// open the file
fs.Open(FilePath);
//clean up
fs.Delete(FilePath, false);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
