public virtual void TestSoftwareVersionDifferences()
{
// We expect no exception to be thrown when the software versions match.
NUnit.Framework.Assert.AreEqual(VersionInfo.GetVersion(), actor.RetrieveNamespaceInfo
().GetSoftwareVersion());
// We expect no exception to be thrown when the min NN version is below the
// reported NN version.
Org.Mockito.Mockito.DoReturn("4.0.0").When(fakeNsInfo).GetSoftwareVersion();
Org.Mockito.Mockito.DoReturn("3.0.0").When(mockDnConf).GetMinimumNameNodeVersion(
);
NUnit.Framework.Assert.AreEqual("4.0.0", actor.RetrieveNamespaceInfo().GetSoftwareVersion
());
// When the NN reports a version that's too low, throw an exception.
Org.Mockito.Mockito.DoReturn("3.0.0").When(fakeNsInfo).GetSoftwareVersion();
Org.Mockito.Mockito.DoReturn("4.0.0").When(mockDnConf).GetMinimumNameNodeVersion(
);
try
{
actor.RetrieveNamespaceInfo();
NUnit.Framework.Assert.Fail("Should have thrown an exception for NN with too-low version"
);
}
catch (IncorrectVersionException ive)
{
GenericTestUtils.AssertExceptionContains("The reported NameNode version is too low"
, ive);
Log.Info("Got expected exception", ive);
}
}
public virtual void TestRecoverAfterIncompleteRecovery()
{
SetupLoggers345();
// Shut down the logger that has length = 5
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
// Allow no logger to finalize
foreach (AsyncLogger spy in spies)
{
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spy).
FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(4L));
}
try
{
qjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed recovery since no finalization occurred"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("injected", ioe);
}
// Now bring back the logger that had 5, and run recovery again.
// We should recover to 4, even though there's a longer log.
cluster.GetJournalNode(0).StopAndJoin(0);
cluster.RestartJournalNode(2);
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, 4);
}
public virtual void TestLinkTargetNonSymlink()
{
FileContext fc = null;
Path notSymlink = new Path("/notasymlink");
try
{
fc = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
fc.Create(notSymlink, EnumSet.Of(CreateFlag.Create));
DFSClient client = new DFSClient(cluster.GetFileSystem().GetUri(), cluster.GetConfiguration
(0));
try
{
client.GetLinkTarget(notSymlink.ToString());
NUnit.Framework.Assert.Fail("Expected exception for resolving non-symlink");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("is not a symbolic link", e);
}
}
finally
{
if (fc != null)
{
fc.Delete(notSymlink, false);
}
}
}
public virtual void TestRenameWithIllegalName()
{
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, seed);
// Create snapshots for sub1
SnapshotTestHelper.CreateSnapshot(hdfs, sub1, "s1");
string name1 = HdfsConstants.DotSnapshotDir;
try
{
hdfs.RenameSnapshot(sub1, "s1", name1);
NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given for rename"
);
}
catch (RemoteException e)
{
string errorMsg = "\"" + HdfsConstants.DotSnapshotDir + "\" is a reserved name.";
GenericTestUtils.AssertExceptionContains(errorMsg, e);
}
string errorMsg_1 = "Snapshot name cannot contain \"" + Path.Separator + "\"";
string[] badNames = new string[] { "foo" + Path.Separator, Path.Separator + "foo"
, Path.Separator, "foo" + Path.Separator + "bar" };
foreach (string badName in badNames)
{
try
{
hdfs.RenameSnapshot(sub1, "s1", badName);
NUnit.Framework.Assert.Fail("Exception expected when an illegal name is given");
}
catch (RemoteException e)
{
GenericTestUtils.AssertExceptionContains(errorMsg_1, e);
}
}
}
public virtual void TestFailureToReadEditsOnTransitionToActive()
{
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path(TestDir1)));
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
// It should also upload it back to the active.
HATestUtil.WaitForCheckpoint(cluster, 0, ImmutableList.Of(0, 3));
CauseFailureOnEditLogRead();
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path(TestDir2)));
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(new Path(TestDir3)));
try
{
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
NUnit.Framework.Assert.Fail("Standby fully caught up, but should not have been able to"
);
}
catch (HATestUtil.CouldNotCatchUpException)
{
}
// Expected. The NN did not exit.
// Shutdown the active NN.
cluster.ShutdownNameNode(0);
try
{
// Transition the standby to active.
cluster.TransitionToActive(1);
NUnit.Framework.Assert.Fail("Standby transitioned to active, but should not have been able to"
);
}
catch (ExitUtil.ExitException ee)
{
GenericTestUtils.AssertExceptionContains("Error replaying edit log", ee);
}
}
public virtual void EnsureInvalidBlockTokensAreRejected()
{
cluster.TransitionToActive(0);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.WriteFile(fs, TestPath, TestData);
NUnit.Framework.Assert.AreEqual(TestData, DFSTestUtil.ReadFile(fs, TestPath));
DFSClient dfsClient = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
DFSClient spyDfsClient = Org.Mockito.Mockito.Spy(dfsClient);
Org.Mockito.Mockito.DoAnswer(new _Answer_121()).When(spyDfsClient).GetLocatedBlocks
(Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.AnyLong());
// This will make the token invalid, since the password
// won't match anymore
DFSClientAdapter.SetDFSClient((DistributedFileSystem)fs, spyDfsClient);
try
{
NUnit.Framework.Assert.AreEqual(TestData, DFSTestUtil.ReadFile(fs, TestPath));
NUnit.Framework.Assert.Fail("Shouldn't have been able to read a file with invalid block tokens"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block", ioe);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDFInvalidPath()
{
// Generate a path that doesn't exist
Random random = new Random(unchecked ((long)(0xDEADBEEFl)));
FilePath file = null;
byte[] bytes = new byte[64];
while (file == null)
{
random.NextBytes(bytes);
string invalid = new string("/" + bytes);
FilePath invalidFile = new FilePath(invalid);
if (!invalidFile.Exists())
{
file = invalidFile;
}
}
DF df = new DF(file, 0l);
try
{
df.GetMount();
}
catch (FileNotFoundException e)
{
// expected, since path does not exist
GenericTestUtils.AssertExceptionContains(file.GetName(), e);
}
}
/// <summary>Test setting some server options.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestServerOptions()
{
string TestPath = new FilePath(sockDir.GetDir(), "test_sock_server_options").GetAbsolutePath
();
DomainSocket serv = DomainSocket.BindAndListen(TestPath);
try
{
// Let's set a new receive buffer size
int bufSize = serv.GetAttribute(DomainSocket.ReceiveBufferSize);
int newBufSize = bufSize / 2;
serv.SetAttribute(DomainSocket.ReceiveBufferSize, newBufSize);
int nextBufSize = serv.GetAttribute(DomainSocket.ReceiveBufferSize);
Assert.Equal(newBufSize, nextBufSize);
// Let's set a server timeout
int newTimeout = 1000;
serv.SetAttribute(DomainSocket.ReceiveTimeout, newTimeout);
int nextTimeout = serv.GetAttribute(DomainSocket.ReceiveTimeout);
Assert.Equal(newTimeout, nextTimeout);
try
{
serv.Accept();
NUnit.Framework.Assert.Fail("expected the accept() to time out and fail");
}
catch (SocketTimeoutException e)
{
GenericTestUtils.AssertExceptionContains("accept(2) error: ", e);
}
}
finally
{
serv.Close();
NUnit.Framework.Assert.IsFalse(serv.IsOpen());
}
}
/// <exception cref="System.Exception"/>
public virtual void TestGracefulFailoverFailBecomingStandbyAndFailFence()
{
try
{
cluster.Start();
cluster.WaitForActiveLockHolder(0);
// Ask for failover when old node fails to transition to standby.
// This should trigger fencing, since the cedeActive() command
// still works, but leaves the breadcrumb in place.
cluster.SetFailToBecomeStandby(0, true);
cluster.SetFailToFence(0, true);
try
{
cluster.GetService(1).GetZKFCProxy(conf, 5000).GracefulFailover();
NUnit.Framework.Assert.Fail("Failover should have failed when old node wont fence"
);
}
catch (ServiceFailedException sfe)
{
GenericTestUtils.AssertExceptionContains("Unable to fence " + cluster.GetService(
0), sfe);
}
}
finally
{
cluster.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestListDotReserved()
{
// Create a base file for comparison
Path baseFileRaw = new Path("/.reserved/raw/base");
int len = 8192;
DFSTestUtil.CreateFile(fs, baseFileRaw, len, (short)1, unchecked ((int)(0xFEED)));
/*
* Ensure that you can't list /.reserved. Ever.
*/
try
{
fs.ListStatus(new Path("/.reserved"));
NUnit.Framework.Assert.Fail("expected FNFE");
}
catch (FileNotFoundException e)
{
GenericTestUtils.AssertExceptionContains("/.reserved does not exist", e);
}
try
{
fs.ListStatus(new Path("/.reserved/.inodes"));
NUnit.Framework.Assert.Fail("expected FNFE");
}
catch (FileNotFoundException e)
{
GenericTestUtils.AssertExceptionContains("/.reserved/.inodes does not exist", e);
}
FileStatus[] fileStatuses = fs.ListStatus(new Path("/.reserved/raw"));
NUnit.Framework.Assert.AreEqual("expected 1 entry", fileStatuses.Length, 1);
GenericTestUtils.AssertMatches(fileStatuses[0].GetPath().ToString(), "/.reserved/raw/base"
);
}
//cluster.shutdown();
/// <exception cref="System.Exception"/>
public virtual void TestMismatchedNNIsRejected()
{
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, MiniDFSCluster.GetBaseDirectory() +
"/TestNNWithQJM/image");
string defaultEditsDir = conf.Get(DFSConfigKeys.DfsNamenodeEditsDirKey);
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI("myjournal"
).ToString());
// Start a NN, so the storage is formatted -- both on-disk
// and QJM.
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs
(false).Build();
cluster.Shutdown();
// Reformat just the on-disk portion
Configuration onDiskOnly = new Configuration(conf);
onDiskOnly.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, defaultEditsDir);
NameNode.Format(onDiskOnly);
// Start the NN - should fail because the JNs are still formatted
// with the old namespace ID.
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).ManageNameDfsDirs(false
).Format(false).Build();
NUnit.Framework.Assert.Fail("New NN with different namespace should have been rejected"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Unable to start log segment 1: too few journals"
, ioe);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestGracefulFailoverToUnhealthy()
{
try
{
cluster.Start();
cluster.WaitForActiveLockHolder(0);
// Mark it unhealthy, wait for it to exit election
cluster.SetHealthy(1, false);
cluster.WaitForElectorState(1, ActiveStandbyElector.State.Init);
// Ask for failover, it should fail, because it's unhealthy
try
{
cluster.GetService(1).GetZKFCProxy(conf, 5000).GracefulFailover();
NUnit.Framework.Assert.Fail("Did not fail to graceful failover to unhealthy service!"
);
}
catch (ServiceFailedException sfe)
{
GenericTestUtils.AssertExceptionContains(cluster.GetService(1).ToString() + " is not currently healthy."
, sfe);
}
}
finally
{
cluster.Stop();
}
}
public virtual void TestRaceWhileNNStartup()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
NameNode namenode = cluster.GetNameNode();
NamenodeProtocols rpcServer = namenode.GetRpcServer();
Whitebox.SetInternalState(namenode, "rpcServer", null);
Path foo = new Path("/foo");
FileSystem webHdfs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem
.Scheme);
try
{
webHdfs.Mkdirs(foo);
NUnit.Framework.Assert.Fail("Expected RetriableException");
}
catch (RetriableException e)
{
GenericTestUtils.AssertExceptionContains("Namenode is in startup mode", e);
}
Whitebox.SetInternalState(namenode, "rpcServer", rpcServer);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
// Expect NoSuchPaddingException
/// <exception cref="System.Exception"/>
public virtual void TestUpdateArguments()
{
Assume.AssumeTrue(OpensslCipher.GetLoadingFailureReason() == null);
OpensslCipher cipher = OpensslCipher.GetInstance("AES/CTR/NoPadding");
Assert.True(cipher != null);
cipher.Init(OpensslCipher.EncryptMode, key, iv);
// Require direct buffers
ByteBuffer input = ByteBuffer.Allocate(1024);
ByteBuffer output = ByteBuffer.Allocate(1024);
try
{
cipher.Update(input, output);
NUnit.Framework.Assert.Fail("Input and output buffer should be direct buffer.");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Direct buffers are required", e);
}
// Output buffer length should be sufficient to store output data
input = ByteBuffer.AllocateDirect(1024);
output = ByteBuffer.AllocateDirect(1000);
try
{
cipher.Update(input, output);
NUnit.Framework.Assert.Fail("Output buffer length should be sufficient " + "to store output data"
);
}
catch (ShortBufferException e)
{
GenericTestUtils.AssertExceptionContains("Output buffer is not sufficient", e);
}
}
/// <summary>
/// Test for catching "no datanode" IOException, when to create a file
/// but datanode is not running for some reason.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCreateWithNoDN()
{
MiniDFSCluster cluster = null;
Configuration conf = WebHdfsTestUtil.CreateConf();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
cluster.WaitActive();
FileSystem fs = WebHdfsTestUtil.GetWebHdfsFileSystem(conf, WebHdfsFileSystem.Scheme
);
fs.Create(new Path("/testnodatanode"));
NUnit.Framework.Assert.Fail("No exception was thrown");
}
catch (IOException ex)
{
GenericTestUtils.AssertExceptionContains("Failed to find datanode", ex);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestFailureToTransitionCausesShutdown()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
// Set an illegal value for the trash emptier interval. This will cause
// the NN to fail to transition to the active state.
conf.SetLong(CommonConfigurationKeys.FsTrashIntervalKey, -1);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).CheckExitOnShutdown(false).Build();
cluster.WaitActive();
try
{
cluster.TransitionToActive(0);
NUnit.Framework.Assert.Fail("Transitioned to active but should not have been able to."
);
}
catch (ExitUtil.ExitException ee)
{
GenericTestUtils.AssertExceptionContains("Cannot start trash emptier with negative interval"
, ee);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
private void AssertCannotStartNameNodes()
{
// Make sure we can't currently start either NN.
try
{
cluster.RestartNameNode(0, false);
NUnit.Framework.Assert.Fail("Should not have been able to start NN1 without shared dir"
);
}
catch (IOException ioe)
{
Log.Info("Got expected exception", ioe);
GenericTestUtils.AssertExceptionContains("storage directory does not exist or is not accessible"
, ioe);
}
try
{
cluster.RestartNameNode(1, false);
NUnit.Framework.Assert.Fail("Should not have been able to start NN2 without shared dir"
);
}
catch (IOException ioe)
{
Log.Info("Got expected exception", ioe);
GenericTestUtils.AssertExceptionContains("storage directory does not exist or is not accessible"
, ioe);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestJournalLocking()
{
Assume.AssumeTrue(journal.GetStorage().GetStorageDir(0).IsLockSupported());
Storage.StorageDirectory sd = journal.GetStorage().GetStorageDir(0);
FilePath lockFile = new FilePath(sd.GetRoot(), Storage.StorageFileLock);
// Journal should be locked, since the format() call locks it.
GenericTestUtils.AssertExists(lockFile);
journal.NewEpoch(FakeNsinfo, 1);
try
{
new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular, mockErrorReporter
);
NUnit.Framework.Assert.Fail("Did not fail to create another journal in same dir");
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Cannot lock storage", ioe);
}
journal.Close();
// Journal should no longer be locked after the close() call.
// Hence, should be able to create a new Journal in the same dir.
Journal journal2 = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption
.Regular, mockErrorReporter);
journal2.NewEpoch(FakeNsinfo, 2);
journal2.Close();
}
/// <summary>Test skip.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSkip()
{
OutputStream @out = GetOutputStream(defaultBufferSize);
WriteData(@out);
// Default buffer size
InputStream @in = GetInputStream(defaultBufferSize);
byte[] result = new byte[dataLen];
int n1 = ReadAll(@in, result, 0, dataLen / 3);
Assert.Equal(n1, ((Seekable)@in).GetPos());
long skipped = @in.Skip(dataLen / 3);
int n2 = ReadAll(@in, result, 0, dataLen);
Assert.Equal(dataLen, n1 + skipped + n2);
byte[] readData = new byte[n2];
System.Array.Copy(result, 0, readData, 0, n2);
byte[] expectedData = new byte[n2];
System.Array.Copy(data, dataLen - n2, expectedData, 0, n2);
Assert.AssertArrayEquals(readData, expectedData);
try
{
skipped = @in.Skip(-3);
NUnit.Framework.Assert.Fail("Skip Negative length should fail.");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Negative skip length", e);
}
// Skip after EOF
skipped = @in.Skip(3);
Assert.Equal(skipped, 0);
@in.Close();
}
/// <summary>Test finalizing a segment after some batch of edits were missed.</summary>
/// <remarks>
/// Test finalizing a segment after some batch of edits were missed.
/// This should fail, since we validate the log before finalization.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestFinalizeWhenEditsAreMissed()
{
journal.NewEpoch(FakeNsinfo, 1);
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 3, QJMTestUtil.CreateTxnData(1, 3));
// Try to finalize up to txn 6, even though we only wrote up to txn 3.
try
{
journal.FinalizeLogSegment(MakeRI(3), 1, 6);
NUnit.Framework.Assert.Fail("did not fail to finalize");
}
catch (JournalOutOfSyncException e)
{
GenericTestUtils.AssertExceptionContains("but only written up to txid 3", e);
}
// Check that, even if we re-construct the journal by scanning the
// disk, we don't allow finalizing incorrectly.
journal.Close();
journal = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular
, mockErrorReporter);
try
{
journal.FinalizeLogSegment(MakeRI(4), 1, 6);
NUnit.Framework.Assert.Fail("did not fail to finalize");
}
catch (JournalOutOfSyncException e)
{
GenericTestUtils.AssertExceptionContains("disk only contains up to txid 3", e);
}
}
public virtual void TestWrappedReadForCompressedData()
{
byte[] buf = new byte[2];
InputStream mockStream = Org.Mockito.Mockito.Mock <InputStream>();
Org.Mockito.Mockito.When(mockStream.Read(buf, 0, 1)).ThenReturn(1);
Org.Mockito.Mockito.When(mockStream.Read(buf, 0, 2)).ThenThrow(new InternalError(
));
try
{
Assert.Equal("Check expected value", 1, IOUtils.WrappedReadForCompressedData
(mockStream, buf, 0, 1));
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Unexpected error while reading");
}
try
{
IOUtils.WrappedReadForCompressedData(mockStream, buf, 0, 2);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Error while reading compressed data", ioe
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestGracefulFailoverFailBecomingActive()
{
try
{
cluster.Start();
cluster.WaitForActiveLockHolder(0);
cluster.SetFailToBecomeActive(1, true);
// Ask for failover, it should fail and report back to user.
try
{
cluster.GetService(1).GetZKFCProxy(conf, 5000).GracefulFailover();
NUnit.Framework.Assert.Fail("Did not fail to graceful failover when target failed "
+ "to become active!");
}
catch (ServiceFailedException sfe)
{
GenericTestUtils.AssertExceptionContains("Couldn't make " + cluster.GetService(1)
+ " active", sfe);
GenericTestUtils.AssertExceptionContains("injected failure", sfe);
}
// No fencing
Assert.Equal(0, cluster.GetService(0).fenceCount);
Assert.Equal(0, cluster.GetService(1).fenceCount);
// Service 0 should go back to being active after the failed failover
cluster.WaitForActiveLockHolder(0);
}
finally
{
cluster.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMinBlockSizeLimit()
{
long blockSize = 4096;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeMinBlockSizeKey, blockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// Try with min block size
fs.Create(new Path("/testmblock1"), true, 4096, (short)3, blockSize);
try
{
// Try with min block size - 1
fs.Create(new Path("/testmblock2"), true, 4096, (short)3, blockSize - 1);
System.Diagnostics.Debug.Assert(false, "Expected IOException after creating a file with small"
+ " blocks ");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Specified block size is less", e);
}
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestMetadataVersionOutput()
{
InitConfig();
dfsCluster = new MiniDFSCluster.Builder(conf).ManageNameDfsDirs(false).NumDataNodes
(1).CheckExitOnShutdown(false).Build();
dfsCluster.WaitClusterUp();
dfsCluster.Shutdown(false);
InitConfig();
TextWriter origOut = System.Console.Out;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
TextWriter stdOut = new TextWriter(baos);
Runtime.SetOut(stdOut);
try
{
NameNode.CreateNameNode(new string[] { "-metadataVersion" }, conf);
}
catch (Exception e)
{
GenericTestUtils.AssertExceptionContains("ExitException", e);
}
/* Check if meta data version is printed correctly. */
string verNumStr = HdfsConstants.NamenodeLayoutVersion + string.Empty;
NUnit.Framework.Assert.IsTrue(baos.ToString("UTF-8").Contains("HDFS Image Version: "
+ verNumStr));
NUnit.Framework.Assert.IsTrue(baos.ToString("UTF-8").Contains("Software format version: "
+ verNumStr));
Runtime.SetOut(origOut);
}
public virtual void TestClientThatDoesNotSupportEncryption()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = GetFileSystem(conf);
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
fs.Close();
cluster.Shutdown();
SetEncryptionConfigKeys(conf);
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(false).ManageNameDfsDirs
(false).Format(false).StartupOption(HdfsServerConstants.StartupOption.Regular).Build
();
fs = GetFileSystem(conf);
DFSClient client = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
DFSClient spyClient = Org.Mockito.Mockito.Spy(client);
Org.Mockito.Mockito.DoReturn(false).When(spyClient).ShouldEncryptData();
DFSClientAdapter.SetDFSClient((DistributedFileSystem)fs, spyClient);
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(DataNode)));
try
{
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
if (resolverClazz != null && !resolverClazz.EndsWith("TestTrustedChannelResolver"
))
{
NUnit.Framework.Assert.Fail("Should not have been able to read without encryption enabled."
);
}
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block:", ioe);
}
finally
{
logs.StopCapturing();
}
fs.Close();
if (resolverClazz == null)
{
GenericTestUtils.AssertMatches(logs.GetOutput(), "Failed to read expected encryption handshake from client at"
);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
private void CheckException(Exception t)
{
GenericTestUtils.AssertExceptionContains("Injected", t);
if (t.ToString().Contains("AssertionError"))
{
throw new RuntimeException("Should never see AssertionError in fault test!", t);
}
}
public virtual void TestRegistrationWithDifferentSoftwareVersionsDuringUpgrade()
{
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDatanodeMinSupportedNamenodeVersionKey, "1.0.0");
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
NamenodeProtocols rpcServer = cluster.GetNameNodeRpc();
long nnCTime = cluster.GetNamesystem().GetFSImage().GetStorage().GetCTime();
StorageInfo mockStorageInfo = Org.Mockito.Mockito.Mock <StorageInfo>();
Org.Mockito.Mockito.DoReturn(nnCTime).When(mockStorageInfo).GetCTime();
DatanodeRegistration mockDnReg = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
Org.Mockito.Mockito.DoReturn(HdfsConstants.DatanodeLayoutVersion).When(mockDnReg)
.GetVersion();
Org.Mockito.Mockito.DoReturn("fake-storage-id").When(mockDnReg).GetDatanodeUuid();
Org.Mockito.Mockito.DoReturn(mockStorageInfo).When(mockDnReg).GetStorageInfo();
// Should succeed when software versions are the same and CTimes are the
// same.
Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion()).When(mockDnReg).GetSoftwareVersion
();
Org.Mockito.Mockito.DoReturn("127.0.0.1").When(mockDnReg).GetIpAddr();
Org.Mockito.Mockito.DoReturn(123).When(mockDnReg).GetXferPort();
rpcServer.RegisterDatanode(mockDnReg);
// Should succeed when software versions are the same and CTimes are
// different.
Org.Mockito.Mockito.DoReturn(nnCTime + 1).When(mockStorageInfo).GetCTime();
rpcServer.RegisterDatanode(mockDnReg);
// Should fail when software version of DN is different from NN and CTimes
// are different.
Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion() + ".1").When(mockDnReg).GetSoftwareVersion
();
try
{
rpcServer.RegisterDatanode(mockDnReg);
NUnit.Framework.Assert.Fail("Should not have been able to register DN with different software"
+ " versions and CTimes");
}
catch (IncorrectVersionException ive)
{
GenericTestUtils.AssertExceptionContains("does not match CTime of NN", ive);
Log.Info("Got expected exception", ive);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestNegativeGroupCaching()
{
string user = "******";
string failMessage = "Did not throw IOException: ";
conf.SetLong(CommonConfigurationKeys.HadoopSecurityGroupsNegativeCacheSecs, 2);
FakeTimer timer = new FakeTimer();
Groups groups = new Groups(conf, timer);
groups.CacheGroupsAdd(Arrays.AsList(myGroups));
groups.Refresh();
TestGroupsCaching.FakeGroupMapping.AddToBlackList(user);
// In the first attempt, the user will be put in the negative cache.
try
{
groups.GetGroups(user);
NUnit.Framework.Assert.Fail(failMessage + "Failed to obtain groups from FakeGroupMapping."
);
}
catch (IOException e)
{
// Expects to raise exception for the first time. But the user will be
// put into the negative cache
GenericTestUtils.AssertExceptionContains("No groups found for user", e);
}
// The second time, the user is in the negative cache.
try
{
groups.GetGroups(user);
NUnit.Framework.Assert.Fail(failMessage + "The user is in the negative cache.");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("No groups found for user", e);
}
// Brings back the backend user-group mapping service.
TestGroupsCaching.FakeGroupMapping.ClearBlackList();
// It should still get groups from the negative cache.
try
{
groups.GetGroups(user);
NUnit.Framework.Assert.Fail(failMessage + "The user is still in the negative cache, even "
+ "FakeGroupMapping has resumed.");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("No groups found for user", e);
}
// Let the elements in the negative cache expire.
timer.Advance(4 * 1000);
// The groups for the user is expired in the negative cache, a new copy of
// groups for the user is fetched.
Assert.Equal(Arrays.AsList(myGroups), groups.GetGroups(user));
}
public virtual void TestMoverCliWithFederation()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NnTopology
(MiniDFSNNTopology.SimpleFederatedTopology(3)).NumDataNodes(0).Build();
Configuration conf = new HdfsConfiguration();
DFSTestUtil.SetFederatedConfiguration(cluster, conf);
try
{
ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf);
NUnit.Framework.Assert.AreEqual(3, namenodes.Count);
try
{
Mover.Cli.GetNameNodePathsToMove(conf, "-p", "/foo");
NUnit.Framework.Assert.Fail("Expect exception for missing authority information");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("does not contain scheme and authority",
e);
}
try
{
Mover.Cli.GetNameNodePathsToMove(conf, "-p", "hdfs:///foo");
NUnit.Framework.Assert.Fail("Expect exception for missing authority information");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("does not contain scheme and authority",
e);
}
try
{
Mover.Cli.GetNameNodePathsToMove(conf, "-p", "wrong-hdfs://ns1/foo");
NUnit.Framework.Assert.Fail("Expect exception for wrong scheme");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Cannot resolve the path", e);
}
IEnumerator <URI> iter = namenodes.GetEnumerator();
URI nn1 = iter.Next();
URI nn2 = iter.Next();
IDictionary <URI, IList <Path> > movePaths = Mover.Cli.GetNameNodePathsToMove(conf,
"-p", nn1 + "/foo", nn1 + "/bar", nn2 + "/foo/bar");
NUnit.Framework.Assert.AreEqual(2, movePaths.Count);
CheckMovePaths(movePaths[nn1], new Path("/foo"), new Path("/bar"));
CheckMovePaths(movePaths[nn2], new Path("/foo/bar"));
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Test that attempting to connect to an invalid path doesn't work.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestInvalidOperations()
{
try
{
DomainSocket.Connect(new FilePath(sockDir.GetDir(), "test_sock_invalid_operation"
).GetAbsolutePath());
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("connect(2) error: ", e);
}
}
