public virtual void TestGetNNUris()
{
HdfsConfiguration conf = new HdfsConfiguration();
string Ns1Nn1Addr = "ns1-nn1.example.com:8020";
string Ns1Nn2Addr = "ns1-nn2.example.com:8020";
string Ns2NnAddr = "ns2-nn.example.com:8020";
string Nn1Addr = "nn.example.com:8020";
string Nn1SrvcAddr = "nn.example.com:8021";
string Nn2Addr = "nn2.example.com:8020";
conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
), Ns1Nn1Addr);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
), Ns1Nn2Addr);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns2"
), Ns2NnAddr);
conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, "hdfs://" + Nn1Addr);
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://" + Nn2Addr);
ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(4, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn2Addr)));
// Make sure that non-HDFS URIs in fs.defaultFS don't get included.
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "viewfs://vfs-name.example.com"
);
uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(3, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
// Make sure that an HA URI being the default URI doesn't result in multiple
// entries being returned.
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(3, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
// Make sure that when a service RPC address is used that is distinct from
// the client RPC address, and that client RPC address is also used as the
// default URI, that the client URI does not end up in the set of URIs
// returned.
conf = new HdfsConfiguration();
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://" + Nn1Addr);
conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nn1Addr);
conf.Set(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, Nn1SrvcAddr);
uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(1, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1SrvcAddr)));
}
public virtual void TestGetNNServiceRpcAddressesForNsIds()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
conf.Set(DFSConfigKeys.DfsInternalNameservicesKey, "nn1");
// Test - configured list of namenodes are returned
string Nn1Address = "localhost:9000";
string Nn2Address = "localhost:9001";
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), Nn1Address
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), Nn2Address
);
IDictionary <string, IDictionary <string, IPEndPoint> > nnMap = DFSUtil.GetNNServiceRpcAddressesForCluster
(conf);
NUnit.Framework.Assert.AreEqual(1, nnMap.Count);
NUnit.Framework.Assert.IsTrue(nnMap.Contains("nn1"));
conf.Set(DFSConfigKeys.DfsInternalNameservicesKey, "nn3");
try
{
DFSUtil.GetNNServiceRpcAddressesForCluster(conf);
NUnit.Framework.Assert.Fail("Should fail for misconfiguration");
}
catch (IOException)
{
}
}
/// <summary>Verify secondary namenode port usage.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSecondaryNodePorts()
{
NameNode nn = null;
try
{
nn = StartNameNode();
// bind http server to the same port as name-node
Configuration conf2 = new HdfsConfiguration(config);
conf2.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, config.Get(DFSConfigKeys
.DfsNamenodeHttpAddressKey));
Log.Info("= Starting 1 on: " + conf2.Get(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey
));
bool started = CanStartSecondaryNode(conf2);
NUnit.Framework.Assert.IsFalse(started);
// should fail
// bind http server to a different port
conf2.Set(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey, ThisHost);
Log.Info("= Starting 2 on: " + conf2.Get(DFSConfigKeys.DfsNamenodeSecondaryHttpAddressKey
));
started = CanStartSecondaryNode(conf2);
NUnit.Framework.Assert.IsTrue(started);
}
finally
{
// should start now
StopNameNode(nn);
}
}
/// <summary>Verify BackupNode port usage.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestBackupNodePorts()
{
NameNode nn = null;
try
{
nn = StartNameNode();
Configuration backup_config = new HdfsConfiguration(config);
backup_config.Set(DFSConfigKeys.DfsNamenodeBackupAddressKey, ThisHost);
// bind http server to the same port as name-node
backup_config.Set(DFSConfigKeys.DfsNamenodeBackupHttpAddressKey, backup_config.Get
(DFSConfigKeys.DfsNamenodeHttpAddressKey));
Log.Info("= Starting 1 on: " + backup_config.Get(DFSConfigKeys.DfsNamenodeBackupHttpAddressKey
));
NUnit.Framework.Assert.IsFalse("Backup started on same port as Namenode", CanStartBackupNode
(backup_config));
// should fail
// bind http server to a different port
backup_config.Set(DFSConfigKeys.DfsNamenodeBackupHttpAddressKey, ThisHost);
Log.Info("= Starting 2 on: " + backup_config.Get(DFSConfigKeys.DfsNamenodeBackupHttpAddressKey
));
bool started = CanStartBackupNode(backup_config);
NUnit.Framework.Assert.IsTrue("Backup Namenode should've started", started);
}
finally
{
// should start now
StopNameNode(nn);
}
}
/// <exception cref="System.IO.IOException"/>
internal static void Setrep(int fromREP, int toREP, bool simulatedStorage)
{
Configuration conf = new HdfsConfiguration();
if (simulatedStorage)
{
SimulatedFSDataset.SetFactory(conf);
}
conf.Set(DFSConfigKeys.DfsReplicationKey, string.Empty + fromREP);
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L);
conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
(2));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(10).Build(
);
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem
);
try
{
Path root = TestDFSShell.Mkdir(fs, new Path("/test/setrep" + fromREP + "-" + toREP
));
Path f = TestDFSShell.WriteFile(fs, new Path(root, "foo"));
{
// Verify setrep for changing replication
string[] args = new string[] { "-setrep", "-w", string.Empty + toREP, string.Empty
+ f };
FsShell shell = new FsShell();
shell.SetConf(conf);
try
{
NUnit.Framework.Assert.AreEqual(0, shell.Run(args));
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue("-setrep " + e, false);
}
}
//get fs again since the old one may be closed
fs = cluster.GetFileSystem();
FileStatus file = fs.GetFileStatus(f);
long len = file.GetLen();
foreach (BlockLocation locations in fs.GetFileBlockLocations(file, 0, len))
{
NUnit.Framework.Assert.IsTrue(locations.GetHosts().Length == toREP);
}
TestDFSShell.Show("done setrep waiting: " + root);
}
finally
{
try
{
fs.Close();
}
catch (Exception)
{
}
cluster.Shutdown();
}
}
/// <summary>Bring up two clusters and assert that they are in different directories.
/// </summary>
/// <exception cref="System.Exception">on a failure</exception>
public virtual void TestDualClusters()
{
FilePath testDataCluster2 = new FilePath(testDataPath, Cluster2);
FilePath testDataCluster3 = new FilePath(testDataPath, Cluster3);
Configuration conf = new HdfsConfiguration();
string c2Path = testDataCluster2.GetAbsolutePath();
conf.Set(MiniDFSCluster.HdfsMinidfsBasedir, c2Path);
MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf).Build();
MiniDFSCluster cluster3 = null;
try
{
string dataDir2 = cluster2.GetDataDirectory();
NUnit.Framework.Assert.AreEqual(new FilePath(c2Path + "/data"), new FilePath(dataDir2
));
//change the data dir
conf.Set(MiniDFSCluster.HdfsMinidfsBasedir, testDataCluster3.GetAbsolutePath());
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
cluster3 = builder.Build();
string dataDir3 = cluster3.GetDataDirectory();
NUnit.Framework.Assert.IsTrue("Clusters are bound to the same directory: " + dataDir2
, !dataDir2.Equals(dataDir3));
}
finally
{
MiniDFSCluster.ShutdownCluster(cluster3);
MiniDFSCluster.ShutdownCluster(cluster2);
}
}
public virtual void TestSomeConfsNNSpecificSomeNSSpecific()
{
HdfsConfiguration conf = new HdfsConfiguration();
string key = DFSConfigKeys.DfsNamenodeSharedEditsDirKey;
conf.Set(key, "global-default");
conf.Set(key + ".ns1", "ns1-override");
conf.Set(key + ".ns1.nn1", "nn1-override");
// A namenode in another nameservice should get the global default.
Configuration newConf = new Configuration(conf);
NameNode.InitializeGenericKeys(newConf, "ns2", "nn1");
NUnit.Framework.Assert.AreEqual("global-default", newConf.Get(key));
// A namenode in another non-HA nameservice should get global default.
newConf = new Configuration(conf);
NameNode.InitializeGenericKeys(newConf, "ns2", null);
NUnit.Framework.Assert.AreEqual("global-default", newConf.Get(key));
// A namenode in the same nameservice should get the ns setting
newConf = new Configuration(conf);
NameNode.InitializeGenericKeys(newConf, "ns1", "nn2");
NUnit.Framework.Assert.AreEqual("ns1-override", newConf.Get(key));
// The nn with the nn-specific setting should get its own override
newConf = new Configuration(conf);
NameNode.InitializeGenericKeys(newConf, "ns1", "nn1");
NUnit.Framework.Assert.AreEqual("nn1-override", newConf.Get(key));
}
private Configuration SetupAddress(string key)
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1");
conf.Set(DFSUtil.AddKeySuffixes(key, "nn1"), "localhost:9000");
return(conf);
}
public virtual void TestGetOnlyNameServiceIdOrNull()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
NUnit.Framework.Assert.IsNull(DFSUtil.GetOnlyNameServiceIdOrNull(conf));
conf.Set(DFSConfigKeys.DfsNameservices, string.Empty);
NUnit.Framework.Assert.IsNull(DFSUtil.GetOnlyNameServiceIdOrNull(conf));
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
NUnit.Framework.Assert.AreEqual("ns1", DFSUtil.GetOnlyNameServiceIdOrNull(conf));
}
public static void SetUp()
{
ClearBaseDir();
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeHttpsPortKey, 0);
conf.Set(DFSConfigKeys.DfsDatanodeAddressKey, "localhost:0");
conf.Set(DFSConfigKeys.DfsDatanodeIpcAddressKey, "localhost:0");
conf.Set(DFSConfigKeys.DfsDatanodeHttpAddressKey, "localhost:0");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
}
/// <summary>
/// Test
/// <see cref="DFSUtil.GetNamenodeNameServiceId(Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// to ensure
/// exception is thrown when multiple rpc addresses match the local node's
/// address
/// </summary>
public virtual void TestGetNameServiceIdException()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), "localhost:9000"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), "localhost:9001"
);
DFSUtil.GetNamenodeNameServiceId(conf);
NUnit.Framework.Assert.Fail("Expected exception is not thrown");
}
public virtual void TestRegistrationWithDifferentSoftwareVersions()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDatanodeMinSupportedNamenodeVersionKey, "3.0.0");
conf.Set(DFSConfigKeys.DfsNamenodeMinSupportedDatanodeVersionKey, "3.0.0");
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
NamenodeProtocols rpcServer = cluster.GetNameNodeRpc();
long nnCTime = cluster.GetNamesystem().GetFSImage().GetStorage().GetCTime();
StorageInfo mockStorageInfo = Org.Mockito.Mockito.Mock <StorageInfo>();
Org.Mockito.Mockito.DoReturn(nnCTime).When(mockStorageInfo).GetCTime();
DatanodeRegistration mockDnReg = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.DoReturn(HdfsConstants.DatanodeLayoutVersion).When(mockDnReg)
.GetVersion();
Org.Mockito.Mockito.DoReturn("127.0.0.1").When(mockDnReg).GetIpAddr();
Org.Mockito.Mockito.DoReturn(123).When(mockDnReg).GetXferPort();
Org.Mockito.Mockito.DoReturn("fake-storage-id").When(mockDnReg).GetDatanodeUuid();
Org.Mockito.Mockito.DoReturn(mockStorageInfo).When(mockDnReg).GetStorageInfo();
// Should succeed when software versions are the same.
Org.Mockito.Mockito.DoReturn("3.0.0").When(mockDnReg).GetSoftwareVersion();
rpcServer.RegisterDatanode(mockDnReg);
// Should succeed when software version of DN is above minimum required by NN.
Org.Mockito.Mockito.DoReturn("4.0.0").When(mockDnReg).GetSoftwareVersion();
rpcServer.RegisterDatanode(mockDnReg);
// Should fail when software version of DN is below minimum required by NN.
Org.Mockito.Mockito.DoReturn("2.0.0").When(mockDnReg).GetSoftwareVersion();
try
{
rpcServer.RegisterDatanode(mockDnReg);
NUnit.Framework.Assert.Fail("Should not have been able to register DN with too-low version."
);
}
catch (IncorrectVersionException ive)
{
GenericTestUtils.AssertExceptionContains("The reported DataNode version is too low"
, ive);
Log.Info("Got expected exception", ive);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestEarlierVersionEditLog()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop10MultiblockTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-1.0");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
FilePath dataDir = new FilePath(dfsDir, "data");
GenericTestUtils.AssertExists(dataDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dataDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).NumDataNodes(1).StartupOption
(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/user/todd/4blocks");
// Read it without caring about the actual data within - we just need
// to make sure that the block states and locations are OK.
DFSTestUtil.ReadFile(fs, testPath);
// Ensure that we can append to it - if the blocks were in some funny
// state we'd get some kind of issue here.
FSDataOutputStream stm = fs.Append(testPath);
try
{
stm.Write(1);
}
finally
{
IOUtils.CloseStream(stm);
}
}
finally
{
cluster.Shutdown();
}
}
public static void SetupCluster()
{
DFSInputStream.tcpReadsDisabledForTesting = true;
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, string.Empty);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser
().GetShortUserName());
DomainSocket.DisableBindPathValidation();
SetupCluster(1, conf);
}
public virtual void GetNameServiceId()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameserviceId, "nn1");
NUnit.Framework.Assert.AreEqual("nn1", DFSUtil.GetNamenodeNameServiceId(conf));
}
/// <summary>Tests get/set working directory in DFS.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestHomeDirectory()
{
string[] homeBases = new string[] { "/home", "/home/user" };
Configuration conf = new HdfsConfiguration();
foreach (string homeBase in homeBases)
{
conf.Set(DFSConfigKeys.DfsUserHomeDirPrefixKey, homeBase);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fileSys = cluster.GetFileSystem();
try
{
// test home directory
Path home = fileSys.MakeQualified(new Path(homeBase + "/" + GetUserName(fileSys))
);
Path fsHome = fileSys.GetHomeDirectory();
NUnit.Framework.Assert.AreEqual(home, fsHome);
}
finally
{
fileSys.Close();
cluster.Shutdown();
}
}
}
private static Configuration CreateWebHDFSHAConfiguration(string logicalHostName,
string nnaddr1, string nnaddr2)
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeHttpAddressKey, "ns1", "nn1"
), nnaddr1);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeHttpAddressKey, "ns1", "nn2"
), nnaddr2);
conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + logicalHostName
, typeof(ConfiguredFailoverProxyProvider).FullName);
return(conf);
}
/// <exception cref="System.Exception"/>
private void DoTestFSOutputSummer(string checksumType)
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, checksumType);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumOfDatanodes
).Build();
fileSys = cluster.GetFileSystem();
try
{
Path file = new Path("try.dat");
Random rand = new Random(seed);
rand.NextBytes(expected);
WriteFile1(file);
WriteFile2(file);
WriteFile3(file);
}
finally
{
fileSys.Close();
cluster.Shutdown();
}
}
public static void SetupCluster()
{
if (DomainSocket.GetLoadingFailureReason() != null)
{
return;
}
sockDir = new TemporarySocketDirectory();
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestParallelShortCircuitReadUnCached._PORT.sock"
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
// Enabling data transfer encryption should have no effect when using
// short-circuit local reads. This is a regression test for HDFS-5353.
conf.SetBoolean(DFSConfigKeys.DfsEncryptDataTransferKey, true);
conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, true);
// We want to test reading from stale sockets.
conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 1);
conf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, 5 * 60 * 1000);
conf.SetInt(DFSConfigKeys.DfsClientSocketCacheCapacityKey, 32);
// Avoid using the FileInputStreamCache.
conf.SetInt(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheSizeKey, 0);
DomainSocket.DisableBindPathValidation();
DFSInputStream.tcpReadsDisabledForTesting = true;
SetupCluster(1, conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestWriteConf()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 4096);
System.Console.Out.WriteLine("Setting conf in: " + Runtime.IdentityHashCode(conf)
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
FileSystem fs = null;
OutputStream os = null;
try
{
fs = cluster.GetFileSystem();
Path filePath = new Path("/testWriteConf.xml");
os = fs.Create(filePath);
StringBuilder longString = new StringBuilder();
for (int i = 0; i < 100000; i++)
{
longString.Append("hello");
}
// 500KB
conf.Set("foobar", longString.ToString());
conf.WriteXml(os);
os.Close();
os = null;
fs.Close();
fs = null;
}
finally
{
IOUtils.Cleanup(null, os, fs);
cluster.Shutdown();
}
}
public virtual void TestSmallBlock()
{
Configuration conf = new HdfsConfiguration();
if (simulatedStorage)
{
SimulatedFSDataset.SetFactory(conf);
}
conf.Set(DFSConfigKeys.DfsBytesPerChecksumKey, "1");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fileSys = cluster.GetFileSystem();
try
{
Path file1 = new Path("smallblocktest.dat");
WriteFile(fileSys, file1);
CheckFile(fileSys, file1);
CleanupFile(fileSys, file1);
}
finally
{
fileSys.Close();
cluster.Shutdown();
}
}
public virtual void TestLoadLogsFromBuggyEarlierVersions()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop23BrokenAppendTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-with-buggy-append");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).NumDataNodes(0).WaitSafeMode
(false).StartupOption(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/tmp/io_data/test_io_0");
NUnit.Framework.Assert.AreEqual(2 * 1024 * 1024, fs.GetFileStatus(testPath).GetLen
());
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestGetSpnegoKeytabKey()
{
HdfsConfiguration conf = new HdfsConfiguration();
string defaultKey = "default.spengo.key";
conf.Unset(DFSConfigKeys.DfsWebAuthenticationKerberosKeytabKey);
NUnit.Framework.Assert.AreEqual("Test spnego key in config is null", defaultKey,
DFSUtil.GetSpnegoKeytabKey(conf, defaultKey));
conf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosKeytabKey, string.Empty);
NUnit.Framework.Assert.AreEqual("Test spnego key is empty", defaultKey, DFSUtil.GetSpnegoKeytabKey
(conf, defaultKey));
string spengoKey = "spengo.key";
conf.Set(DFSConfigKeys.DfsWebAuthenticationKerberosKeytabKey, spengoKey);
NUnit.Framework.Assert.AreEqual("Test spnego key is NOT null", DFSConfigKeys.DfsWebAuthenticationKerberosKeytabKey
, DFSUtil.GetSpnegoKeytabKey(conf, defaultKey));
}
public static void SetupCluster()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 4096);
conf.Set("fs.hdfs.impl.disable.cache", "true");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
fs = cluster.GetFileSystem();
}
public virtual void TestPipelineRecoveryOnRestartFailure()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientDatanodeRestartTimeoutKey, "5");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
Path file = new Path("dataprotocol3.dat");
DFSTestUtil.CreateFile(fileSys, file, 10240L, (short)2, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.GetDataNodes()[0];
string dnAddr1 = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args1 = new string[] { "-shutdownDatanode", dnAddr1, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args1));
Sharpen.Thread.Sleep(4000);
// This should succeed without restarting the node. The restart will
// expire and regular pipeline recovery will kick in.
@out.Close();
// At this point there is only one node in the cluster.
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
dn = cluster.GetDataNodes()[1];
string dnAddr2 = dn.GetDatanodeId().GetIpcAddr(false);
// issue shutdown to the datanode.
string[] args2 = new string[] { "-shutdownDatanode", dnAddr2, "upgrade" };
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(args2));
Sharpen.Thread.Sleep(4000);
try
{
// close should fail
@out.Close();
System.Diagnostics.Debug.Assert(false);
}
catch (IOException)
{
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestPacketTransmissionDelay()
{
// Make the first datanode to not relay heartbeat packet.
DataNodeFaultInjector dnFaultInjector = new _DataNodeFaultInjector_171();
DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.Get();
DataNodeFaultInjector.Set(dnFaultInjector);
// Setting the timeout to be 3 seconds. Normally heartbeat packet
// would be sent every 1.5 seconds if there is no data traffic.
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsClientSocketTimeoutKey, "3000");
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
FSDataOutputStream @out = fs.Create(new Path("noheartbeat.dat"), (short)2);
@out.Write(unchecked ((int)(0x31)));
@out.Hflush();
DFSOutputStream dfsOut = (DFSOutputStream)@out.GetWrappedStream();
// original pipeline
DatanodeInfo[] orgNodes = dfsOut.GetPipeline();
// Cause the second datanode to timeout on reading packet
Sharpen.Thread.Sleep(3500);
@out.Write(unchecked ((int)(0x32)));
@out.Hflush();
// new pipeline
DatanodeInfo[] newNodes = dfsOut.GetPipeline();
@out.Close();
bool contains = false;
for (int i = 0; i < newNodes.Length; i++)
{
if (orgNodes[0].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
throw new IOException("The first datanode should have been replaced.");
}
if (orgNodes[1].GetXferAddr().Equals(newNodes[i].GetXferAddr()))
{
contains = true;
}
}
NUnit.Framework.Assert.IsTrue(contains);
}
finally
{
DataNodeFaultInjector.Set(oldDnInjector);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestHANameNodesWithFederation()
{
HdfsConfiguration conf = new HdfsConfiguration();
string Ns1Nn1Host = "ns1-nn1.example.com:8020";
string Ns1Nn2Host = "ns1-nn2.example.com:8020";
string Ns2Nn1Host = "ns2-nn1.example.com:8020";
string Ns2Nn2Host = "ns2-nn2.example.com:8020";
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
// Two nameservices, each with two NNs.
conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "ns1-nn1,ns1-nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns2"), "ns2-nn1,ns2-nn2"
);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn1"
), Ns1Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn2"
), Ns1Nn2Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn1"
), Ns2Nn1Host);
conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn2"
), Ns2Nn2Host);
IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
(conf);
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns2"));
NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "ns3"));
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, map["ns1"]["ns1-nn1"].ToString());
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, map["ns1"]["ns1-nn2"].ToString());
NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, map["ns2"]["ns2-nn1"].ToString());
NUnit.Framework.Assert.AreEqual(Ns2Nn2Host, map["ns2"]["ns2-nn2"].ToString());
NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "ns1-nn1"));
NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns1", "ns1-nn2"));
NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
"ns2", "ns2-nn1"));
// No nameservice was given and we can't determine which service addr
// to use as two nameservices could share a namenode ID.
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, null,
"ns1-nn1"));
// Ditto for nameservice IDs, if multiple are defined
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeNameServiceId(conf));
NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetSecondaryNameServiceId(conf));
ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeRpcAddressKey
);
NUnit.Framework.Assert.AreEqual(2, uris.Count);
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns2")));
}
/// <summary>Verify namenode port usage.</summary>
/// <exception cref="System.Exception"/>
public virtual void RunTestNameNodePorts(bool withService)
{
NameNode nn = null;
try
{
nn = StartNameNode(withService);
// start another namenode on the same port
Configuration conf2 = new HdfsConfiguration(config);
conf2.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Util.FileAsURI(new FilePath(hdfsDir
, "name2")).ToString());
DFSTestUtil.FormatNameNode(conf2);
bool started = CanStartNameNode(conf2);
NUnit.Framework.Assert.IsFalse(started);
// should fail
// start on a different main port
FileSystem.SetDefaultUri(conf2, "hdfs://" + ThisHost);
started = CanStartNameNode(conf2);
NUnit.Framework.Assert.IsFalse(started);
// should fail again
// reset conf2 since NameNode modifies it
FileSystem.SetDefaultUri(conf2, "hdfs://" + ThisHost);
// different http port
conf2.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, ThisHost);
started = CanStartNameNode(conf2);
if (withService)
{
NUnit.Framework.Assert.IsFalse("Should've failed on service port", started);
// reset conf2 since NameNode modifies it
FileSystem.SetDefaultUri(conf2, "hdfs://" + ThisHost);
conf2.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, ThisHost);
// Set Service address
conf2.Set(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, ThisHost);
started = CanStartNameNode(conf2);
}
NUnit.Framework.Assert.IsTrue(started);
}
finally
{
StopNameNode(nn);
}
}
public virtual void TestRegistrationWithDifferentSoftwareVersionsDuringUpgrade()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDatanodeMinSupportedNamenodeVersionKey, "1.0.0");
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
NamenodeProtocols rpcServer = cluster.GetNameNodeRpc();
long nnCTime = cluster.GetNamesystem().GetFSImage().GetStorage().GetCTime();
StorageInfo mockStorageInfo = Org.Mockito.Mockito.Mock <StorageInfo>();
Org.Mockito.Mockito.DoReturn(nnCTime).When(mockStorageInfo).GetCTime();
DatanodeRegistration mockDnReg = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.DoReturn(HdfsConstants.DatanodeLayoutVersion).When(mockDnReg)
.GetVersion();
Org.Mockito.Mockito.DoReturn("fake-storage-id").When(mockDnReg).GetDatanodeUuid();
Org.Mockito.Mockito.DoReturn(mockStorageInfo).When(mockDnReg).GetStorageInfo();
// Should succeed when software versions are the same and CTimes are the
// same.
Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion()).When(mockDnReg).GetSoftwareVersion
();
Org.Mockito.Mockito.DoReturn("127.0.0.1").When(mockDnReg).GetIpAddr();
Org.Mockito.Mockito.DoReturn(123).When(mockDnReg).GetXferPort();
rpcServer.RegisterDatanode(mockDnReg);
// Should succeed when software versions are the same and CTimes are
// different.
Org.Mockito.Mockito.DoReturn(nnCTime + 1).When(mockStorageInfo).GetCTime();
rpcServer.RegisterDatanode(mockDnReg);
// Should fail when software version of DN is different from NN and CTimes
// are different.
Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion() + ".1").When(mockDnReg).GetSoftwareVersion
();
try
{
rpcServer.RegisterDatanode(mockDnReg);
NUnit.Framework.Assert.Fail("Should not have been able to register DN with different software"
+ " versions and CTimes");
}
catch (IncorrectVersionException ive)
{
GenericTestUtils.AssertExceptionContains("does not match CTime of NN", ive);
Log.Info("Got expected exception", ive);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestDeprecatedKeys()
{
Configuration conf = new HdfsConfiguration();
conf.Set("topology.script.file.name", "xyz");
string scriptFile = conf.Get(DFSConfigKeys.NetTopologyScriptFileNameKey);
NUnit.Framework.Assert.IsTrue(scriptFile.Equals("xyz"));
conf.SetInt("dfs.replication.interval", 1);
string alpha = DFSConfigKeys.DfsNamenodeReplicationIntervalKey;
int repInterval = conf.GetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 3);
NUnit.Framework.Assert.IsTrue(repInterval == 1);
}
