public virtual void SetupCluster()
{
conf = new Configuration();
conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
HAUtil.SetAllowStandbyReads(conf, true);
fsHelper = new FileSystemTestHelper();
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri());
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(1).Build();
cluster.WaitActive();
cluster.TransitionToActive(0);
fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 0, conf);
DFSTestUtil.CreateKey(TestKey, cluster, 1, conf);
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf);
dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf);
KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider
();
fs.GetClient().SetKeyProvider(nn0Provider);
}
public static void ClusterSetupAtBeginning()
{
cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology
(2)).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf);
fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf);
}
public static void ClusterSetupAtBeginning()
{
clusterConf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
cluster = new MiniDFSCluster.Builder(clusterConf).NnTopology(MiniDFSNNTopology.SimpleFederatedTopology
(2)).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), clusterConf);
fc2 = FileContext.GetFileContext(cluster.GetURI(1), clusterConf);
}
public static void ClusterSetupAtBegining()
{
SupportsBlocks = true;
Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
}
public static void ClusterSetupAtBegining()
{
Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
fc = FileContext.GetFileContext(cluster.GetURI(0), conf);
defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation
.GetCurrentUser().GetShortUserName()));
fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true);
}
/// <exception cref="System.IO.IOException"/>
private void RestartCluster(bool format)
{
if (cluster != null)
{
cluster.Shutdown();
cluster = null;
}
cluster = new MiniDFSCluster.Builder(Conf).Format(format).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
}
public virtual void TestIPFailoverProxyProviderLogicalUri()
{
// setup the config with the IP failover proxy provider class
Configuration config = new HdfsConfiguration(conf);
URI nnUri = cluster.GetURI(0);
config.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + nnUri.GetHost
(), typeof(IPFailoverProxyProvider).FullName);
NUnit.Framework.Assert.IsFalse("IPFailoverProxyProvider should not use logical URI."
, HAUtil.UseLogicalUri(config, nnUri));
}
public static void TestSetUp()
{
conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsListLimit, 2);
cluster = new MiniDFSCluster.Builder(conf).Build();
fs = cluster.GetFileSystem();
fc = FileContext.GetFileContext(cluster.GetURI(0), conf);
hftpfs = cluster.GetHftpFileSystem(0);
dfsClient = new DFSClient(NameNode.GetAddress(conf), conf);
file1 = new Path("filestatus.dat");
WriteFile(fs, file1, 1, fileSize, blockSize);
}
public static void ClusterSetupAtBegining()
{
SupportsBlocks = true;
Conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true);
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(2).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
Path defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation
.GetCurrentUser().GetShortUserName()));
fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true);
}
/// <summary>Regression test for HDFS-3357.</summary>
/// <remarks>
/// Regression test for HDFS-3357. Check that the datanode is respecting
/// its configured keepalive timeout.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRespectsKeepAliveTimeout()
{
Configuration clientConf = new Configuration(conf);
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 60000L;
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testDatanodeRespectsKeepAliveTimeout"
);
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
AssertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.ReadFile(fs, TestFile);
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
AssertXceiverCount(1);
// Sleep for a bit longer than the keepalive timeout
// and make sure the xceiver died.
Sharpen.Thread.Sleep(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveDefault + 50);
AssertXceiverCount(0);
// The socket is still in the cache, because we don't
// notice that it's closed until we try to read
// from it again.
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
// Take it out of the cache - reading should
// give an EOF.
Peer peer = peerCache.Get(dn.GetDatanodeId(), false);
NUnit.Framework.Assert.IsNotNull(peer);
NUnit.Framework.Assert.AreEqual(-1, peer.GetInputStream().Read());
}
public virtual void TestBinaryTokenFile()
{
Configuration conf = mrCluster.GetConfig();
// provide namenodes names for the job to get the delegation tokens for
string nnUri = dfsCluster.GetURI(0).ToString();
conf.Set(MRJobConfig.JobNamenodes, nnUri + "," + nnUri);
// using argument to pass the file name
string[] args = new string[] { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" };
int res = -1;
try
{
res = ToolRunner.Run(conf, new TestBinaryTokenFile.MySleepJob(this), args);
}
catch (Exception e)
{
System.Console.Out.WriteLine("Job failed with " + e.GetLocalizedMessage());
Sharpen.Runtime.PrintStackTrace(e, System.Console.Out);
NUnit.Framework.Assert.Fail("Job failed");
}
NUnit.Framework.Assert.AreEqual("dist job res is not 0:", 0, res);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Javax.Security.Auth.Login.LoginException"/>
private static void RestartCluster()
{
if (cluster != null)
{
cluster.Shutdown();
cluster = null;
}
cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(1).Format(false).Build();
cluster.WaitClusterUp();
fc = FileContext.GetFileContext(cluster.GetURI(0), Conf);
defaultWorkingDirectory = fc.MakeQualified(new Path("/user/" + UserGroupInformation
.GetCurrentUser().GetShortUserName()));
fc.Mkdir(defaultWorkingDirectory, FileContext.DefaultPerm, true);
}
public virtual void Setup()
{
conf = new HdfsConfiguration();
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
FilePath testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
Logger.GetLogger(typeof(EncryptionZoneManager)).SetLevel(Level.Trace);
fs = cluster.GetFileSystem();
fsWrapper = new FileSystemTestWrapper(cluster.GetFileSystem());
fcWrapper = new FileContextTestWrapper(FileContext.GetFileContext(cluster.GetURI(
), conf));
dfsAdmin = new HdfsAdmin(cluster.GetURI(), conf);
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
fs.GetClient().SetKeyProvider(cluster.GetNameNode().GetNamesystem().GetProvider()
);
DFSTestUtil.CreateKey(TestKey, cluster, conf);
}
/// <summary>Tests DataTransferProtocol with the given client configuration.</summary>
/// <param name="conf">client configuration</param>
/// <exception cref="System.IO.IOException">if there is an I/O error</exception>
private void DoTest(HdfsConfiguration conf)
{
fs = FileSystem.Get(cluster.GetURI(), conf);
FileSystemTestHelper.CreateFile(fs, Path, NumBlocks, BlockSize);
Assert.AssertArrayEquals(FileSystemTestHelper.GetFileData(NumBlocks, BlockSize),
Sharpen.Runtime.GetBytesForString(DFSTestUtil.ReadFile(fs, Path), "UTF-8"));
BlockLocation[] blockLocations = fs.GetFileBlockLocations(Path, 0, long.MaxValue);
NUnit.Framework.Assert.IsNotNull(blockLocations);
NUnit.Framework.Assert.AreEqual(NumBlocks, blockLocations.Length);
foreach (BlockLocation blockLocation in blockLocations)
{
NUnit.Framework.Assert.IsNotNull(blockLocation.GetHosts());
NUnit.Framework.Assert.AreEqual(3, blockLocation.GetHosts().Length);
}
}
/// <exception cref="System.Exception"/>
public TestFSRMStateStoreTester(TestFSRMStateStore _enclosing, MiniDFSCluster cluster
, bool adminCheckEnable)
{
this._enclosing = _enclosing;
Path workingDirPath = new Path("/yarn/Test");
this.adminCheckEnable = adminCheckEnable;
this.cluster = cluster;
FileSystem fs = cluster.GetFileSystem();
fs.Mkdirs(workingDirPath);
Path clusterURI = new Path(cluster.GetURI());
this.workingDirPathURI = new Path(clusterURI, workingDirPath);
fs.Close();
}
public static void Setup()
{
string currentUser = Runtime.GetProperty("user.name");
config.Set("fs.permissions.umask-mode", "u=rwx,g=,o=");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey
(currentUser), "*");
config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey
(currentUser), "*");
fsHelper = new FileSystemTestHelper();
// Set up java key store
string testRoot = fsHelper.GetTestRootDir();
testRootDir = new FilePath(testRoot).GetAbsoluteFile();
Path jksPath = new Path(testRootDir.ToString(), "test.jks");
config.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
+ "://file" + jksPath.ToUri());
ProxyUsers.RefreshSuperUserGroupsConfiguration(config);
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
nn = cluster.GetNameNode();
dfsAdmin = new HdfsAdmin(cluster.GetURI(), config);
// Use ephemeral ports in case tests are running in parallel
config.SetInt("nfs3.mountd.port", 0);
config.SetInt("nfs3.server.port", 0);
// Start NFS with allowed.hosts set to "* rw"
config.Set("dfs.nfs.exports.allowed.hosts", "* rw");
nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3(config);
nfs.StartServiceInternal(false);
nfsd = (RpcProgramNfs3)nfs.GetRpcProgram();
hdfs.GetClient().SetKeyProvider(nn.GetNamesystem().GetProvider());
DFSTestUtil.CreateKey(TestKey, cluster, config);
// Mock SecurityHandler which returns system user.name
securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(currentUser);
// Mock SecurityHandler which returns a dummy username "harry"
securityHandlerUnpriviledged = Org.Mockito.Mockito.Mock <SecurityHandler>();
Org.Mockito.Mockito.When(securityHandlerUnpriviledged.GetUser()).ThenReturn("harry"
);
}
protected override URI TestURI()
{
return(cluster.GetURI(0));
}