public virtual void TestWrappedReadForCompressedData()
{
byte[] buf = new byte[2];
InputStream mockStream = Org.Mockito.Mockito.Mock <InputStream>();
Org.Mockito.Mockito.When(mockStream.Read(buf, 0, 1)).ThenReturn(1);
Org.Mockito.Mockito.When(mockStream.Read(buf, 0, 2)).ThenThrow(new InternalError(
));
try
{
Assert.Equal("Check expected value", 1, IOUtils.WrappedReadForCompressedData
(mockStream, buf, 0, 1));
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Unexpected error while reading");
}
try
{
IOUtils.WrappedReadForCompressedData(mockStream, buf, 0, 2);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Error while reading compressed data", ioe
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRollingUpgradeWithRollback()
{
try
{
StartCluster();
// Create files in DFS.
Path testFile1 = new Path("/" + GenericTestUtils.GetMethodName() + ".01.dat");
DFSTestUtil.CreateFile(fs, testFile1, FileSize, ReplFactor, Seed);
string fileContents1 = DFSTestUtil.ReadFile(fs, testFile1);
StartRollingUpgrade();
FilePath blockFile = GetBlockForFile(testFile1, true);
FilePath trashFile = GetTrashFileForBlock(blockFile, false);
DeleteAndEnsureInTrash(testFile1, blockFile, trashFile);
// Now perform a rollback to restore DFS to the pre-rollback state.
RollbackRollingUpgrade();
// Ensure that block was restored from trash
EnsureTrashRestored(blockFile, trashFile);
// Ensure that files exist and restored file contents are the same.
System.Diagnostics.Debug.Assert((fs.Exists(testFile1)));
string fileContents2 = DFSTestUtil.ReadFile(fs, testFile1);
Assert.AssertThat(fileContents1, IS.Is(fileContents2));
}
finally
{
ShutdownCluster();
}
}
public virtual void TestRecoverAfterIncompleteRecovery()
{
SetupLoggers345();
// Shut down the logger that has length = 5
cluster.GetJournalNode(2).StopAndJoin(0);
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
// Allow no logger to finalize
foreach (AsyncLogger spy in spies)
{
TestQuorumJournalManagerUnit.FutureThrows(new IOException("injected")).When(spy).
FinalizeLogSegment(Org.Mockito.Mockito.Eq(1L), Org.Mockito.Mockito.Eq(4L));
}
try
{
qjm.RecoverUnfinalizedSegments();
NUnit.Framework.Assert.Fail("Should have failed recovery since no finalization occurred"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("injected", ioe);
}
// Now bring back the logger that had 5, and run recovery again.
// We should recover to 4, even though there's a longer log.
cluster.GetJournalNode(0).StopAndJoin(0);
cluster.RestartJournalNode(2);
qjm = CreateSpyingQJM();
spies = qjm.GetLoggerSetForTests().GetLoggersForTests();
qjm.RecoverUnfinalizedSegments();
CheckRecovery(cluster, 1, 4);
}
/// <exception cref="System.Exception"/>
public virtual void TestUncacheQuiesces()
{
// Create a file
Path fileName = new Path("/testUncacheQuiesces");
int fileLen = 4096;
DFSTestUtil.CreateFile(fs, fileName, fileLen, (short)1, unchecked ((int)(0xFDFD)));
// Cache it
DistributedFileSystem dfs = cluster.GetFileSystem();
dfs.AddCachePool(new CachePoolInfo("pool"));
dfs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPool("pool").SetPath(fileName
).SetReplication((short)3).Build());
GenericTestUtils.WaitFor(new _Supplier_484(), 1000, 30000);
// Uncache it
dfs.RemoveCacheDirective(1);
GenericTestUtils.WaitFor(new _Supplier_495(), 1000, 30000);
// Make sure that no additional messages were sent
Sharpen.Thread.Sleep(10000);
MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(dn.GetMetrics().Name()
);
MetricsAsserts.AssertCounter("BlocksCached", 1l, dnMetrics);
MetricsAsserts.AssertCounter("BlocksUncached", 1l, dnMetrics);
}
public virtual void EnsureInvalidBlockTokensAreRejected()
{
cluster.TransitionToActive(0);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.WriteFile(fs, TestPath, TestData);
NUnit.Framework.Assert.AreEqual(TestData, DFSTestUtil.ReadFile(fs, TestPath));
DFSClient dfsClient = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
DFSClient spyDfsClient = Org.Mockito.Mockito.Spy(dfsClient);
Org.Mockito.Mockito.DoAnswer(new _Answer_121()).When(spyDfsClient).GetLocatedBlocks
(Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.AnyLong());
// This will make the token invalid, since the password
// won't match anymore
DFSClientAdapter.SetDFSClient((DistributedFileSystem)fs, spyDfsClient);
try
{
NUnit.Framework.Assert.AreEqual(TestData, DFSTestUtil.ReadFile(fs, TestPath));
NUnit.Framework.Assert.Fail("Shouldn't have been able to read a file with invalid block tokens"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block", ioe);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestGracefulFailoverToUnhealthy()
{
try
{
cluster.Start();
cluster.WaitForActiveLockHolder(0);
// Mark it unhealthy, wait for it to exit election
cluster.SetHealthy(1, false);
cluster.WaitForElectorState(1, ActiveStandbyElector.State.Init);
// Ask for failover, it should fail, because it's unhealthy
try
{
cluster.GetService(1).GetZKFCProxy(conf, 5000).GracefulFailover();
NUnit.Framework.Assert.Fail("Did not fail to graceful failover to unhealthy service!"
);
}
catch (ServiceFailedException sfe)
{
GenericTestUtils.AssertExceptionContains(cluster.GetService(1).ToString() + " is not currently healthy."
, sfe);
}
}
finally
{
cluster.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestGracefulFailoverFailBecomingStandbyAndFailFence()
{
try
{
cluster.Start();
cluster.WaitForActiveLockHolder(0);
// Ask for failover when old node fails to transition to standby.
// This should trigger fencing, since the cedeActive() command
// still works, but leaves the breadcrumb in place.
cluster.SetFailToBecomeStandby(0, true);
cluster.SetFailToFence(0, true);
try
{
cluster.GetService(1).GetZKFCProxy(conf, 5000).GracefulFailover();
NUnit.Framework.Assert.Fail("Failover should have failed when old node wont fence"
);
}
catch (ServiceFailedException sfe)
{
GenericTestUtils.AssertExceptionContains("Unable to fence " + cluster.GetService(
0), sfe);
}
}
finally
{
cluster.Stop();
}
}
public virtual void TestLinkTargetNonSymlink()
{
FileContext fc = null;
Path notSymlink = new Path("/notasymlink");
try
{
fc = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
fc.Create(notSymlink, EnumSet.Of(CreateFlag.Create));
DFSClient client = new DFSClient(cluster.GetFileSystem().GetUri(), cluster.GetConfiguration
(0));
try
{
client.GetLinkTarget(notSymlink.ToString());
NUnit.Framework.Assert.Fail("Expected exception for resolving non-symlink");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("is not a symbolic link", e);
}
}
finally
{
if (fc != null)
{
fc.Delete(notSymlink, false);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestMetadataVersionOutput()
{
InitConfig();
dfsCluster = new MiniDFSCluster.Builder(conf).ManageNameDfsDirs(false).NumDataNodes
(1).CheckExitOnShutdown(false).Build();
dfsCluster.WaitClusterUp();
dfsCluster.Shutdown(false);
InitConfig();
TextWriter origOut = System.Console.Out;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
TextWriter stdOut = new TextWriter(baos);
Runtime.SetOut(stdOut);
try
{
NameNode.CreateNameNode(new string[] { "-metadataVersion" }, conf);
}
catch (Exception e)
{
GenericTestUtils.AssertExceptionContains("ExitException", e);
}
/* Check if meta data version is printed correctly. */
string verNumStr = HdfsConstants.NamenodeLayoutVersion + string.Empty;
NUnit.Framework.Assert.IsTrue(baos.ToString("UTF-8").Contains("HDFS Image Version: "
+ verNumStr));
NUnit.Framework.Assert.IsTrue(baos.ToString("UTF-8").Contains("Software format version: "
+ verNumStr));
Runtime.SetOut(origOut);
}
public virtual void TestRamDiskShortCircuitRead()
{
StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, 2 * BlockSize - 1, true);
// 1 replica + delta, SCR read
string MethodName = GenericTestUtils.GetMethodName();
int Seed = unchecked ((int)(0xFADED));
Path path = new Path("/" + MethodName + ".dat");
MakeRandomTestFile(path, BlockSize, true, Seed);
EnsureFileReplicasOnStorageType(path, StorageType.RamDisk);
// Sleep for a short time to allow the lazy writer thread to do its job
Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
//assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
FSDataInputStream fis = fs.Open(path);
// Verify SCR read counters
try
{
fis = fs.Open(path);
byte[] buf = new byte[BufferLength];
fis.Read(0, buf, 0, BufferLength);
HdfsDataInputStream dfsis = (HdfsDataInputStream)fis;
NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead
());
NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead
());
}
finally
{
fis.Close();
fis = null;
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAppendSort()
{
GenericTestUtils.AssumeInNativeProfile();
Path file = new Path(RootPath, "testseqappendSort.seq");
fs.Delete(file, true);
Path sortedFile = new Path(RootPath, "testseqappendSort.seq.sort");
fs.Delete(sortedFile, true);
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, new JavaSerializationComparator
<long>(), typeof(long), typeof(string), conf);
SequenceFile.Writer.Option compressOption = SequenceFile.Writer.Compression(SequenceFile.CompressionType
.Block, new GzipCodec());
SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.
File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass
(typeof(string)), compressOption);
writer.Append(2L, "two");
writer.Append(1L, "one");
writer.Close();
writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer
.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer
.AppendIfExists(true), compressOption);
writer.Append(4L, "four");
writer.Append(3L, "three");
writer.Close();
// Sort file after append
sorter.Sort(file, sortedFile);
VerifyAll4Values(sortedFile);
fs.DeleteOnExit(file);
fs.DeleteOnExit(sortedFile);
}
public virtual void TestSoftwareVersionDifferences()
{
// We expect no exception to be thrown when the software versions match.
NUnit.Framework.Assert.AreEqual(VersionInfo.GetVersion(), actor.RetrieveNamespaceInfo
().GetSoftwareVersion());
// We expect no exception to be thrown when the min NN version is below the
// reported NN version.
Org.Mockito.Mockito.DoReturn("4.0.0").When(fakeNsInfo).GetSoftwareVersion();
Org.Mockito.Mockito.DoReturn("3.0.0").When(mockDnConf).GetMinimumNameNodeVersion(
);
NUnit.Framework.Assert.AreEqual("4.0.0", actor.RetrieveNamespaceInfo().GetSoftwareVersion
());
// When the NN reports a version that's too low, throw an exception.
Org.Mockito.Mockito.DoReturn("3.0.0").When(fakeNsInfo).GetSoftwareVersion();
Org.Mockito.Mockito.DoReturn("4.0.0").When(mockDnConf).GetMinimumNameNodeVersion(
);
try
{
actor.RetrieveNamespaceInfo();
NUnit.Framework.Assert.Fail("Should have thrown an exception for NN with too-low version"
);
}
catch (IncorrectVersionException ive)
{
GenericTestUtils.AssertExceptionContains("The reported NameNode version is too low"
, ive);
Log.Info("Got expected exception", ive);
}
}
public virtual void TestServerSaslNoClientSasl()
{
HdfsConfiguration clusterConf = CreateSecureConfig("authentication,integrity,privacy"
);
// Set short retry timeouts so this test runs faster
clusterConf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
StartCluster(clusterConf);
HdfsConfiguration clientConf = new HdfsConfiguration(clusterConf);
clientConf.Set(DFSConfigKeys.DfsDataTransferProtectionKey, string.Empty);
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(DataNode)));
try
{
DoTest(clientConf);
NUnit.Framework.Assert.Fail("Should fail if SASL data transfer protection is not "
+ "configured or not supported in client");
}
catch (IOException e)
{
GenericTestUtils.AssertMatches(e.Message, "could only be replicated to 0 nodes");
}
finally
{
logs.StopCapturing();
}
GenericTestUtils.AssertMatches(logs.GetOutput(), "Failed to read expected SASL data transfer protection "
+ "handshake from client at");
}
public virtual void Pipeline_01()
{
string MethodName = GenericTestUtils.GetMethodName();
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + MethodName);
}
Path filePath = new Path("/" + MethodName + ".dat");
DFSTestUtil.CreateFile(fs, filePath, FileSize, ReplFactor, rand.NextLong());
if (Log.IsDebugEnabled())
{
Log.Debug("Invoking append but doing nothing otherwise...");
}
FSDataOutputStream ofs = fs.Append(filePath);
ofs.WriteBytes("Some more stuff to write");
((DFSOutputStream)ofs.GetWrappedStream()).Hflush();
IList <LocatedBlock> lb = cluster.GetNameNodeRpc().GetBlockLocations(filePath.ToString
(), FileSize - 1, FileSize).GetLocatedBlocks();
string bpid = cluster.GetNamesystem().GetBlockPoolId();
foreach (DataNode dn in cluster.GetDataNodes())
{
Replica r = DataNodeTestUtils.FetchReplicaInfo(dn, bpid, lb[0].GetBlock().GetBlockId
());
NUnit.Framework.Assert.IsTrue("Replica on DN " + dn + " shouldn't be null", r !=
null);
NUnit.Framework.Assert.AreEqual("Should be RBW replica on " + dn + " after sequence of calls append()/write()/hflush()"
, HdfsServerConstants.ReplicaState.Rbw, r.GetState());
}
ofs.Close();
}
/// <summary>Test finalizing a segment after some batch of edits were missed.</summary>
/// <remarks>
/// Test finalizing a segment after some batch of edits were missed.
/// This should fail, since we validate the log before finalization.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestFinalizeWhenEditsAreMissed()
{
journal.NewEpoch(FakeNsinfo, 1);
journal.StartLogSegment(MakeRI(1), 1, NameNodeLayoutVersion.CurrentLayoutVersion);
journal.Journal(MakeRI(2), 1, 1, 3, QJMTestUtil.CreateTxnData(1, 3));
// Try to finalize up to txn 6, even though we only wrote up to txn 3.
try
{
journal.FinalizeLogSegment(MakeRI(3), 1, 6);
NUnit.Framework.Assert.Fail("did not fail to finalize");
}
catch (JournalOutOfSyncException e)
{
GenericTestUtils.AssertExceptionContains("but only written up to txid 3", e);
}
// Check that, even if we re-construct the journal by scanning the
// disk, we don't allow finalizing incorrectly.
journal.Close();
journal = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular
, mockErrorReporter);
try
{
journal.FinalizeLogSegment(MakeRI(4), 1, 6);
NUnit.Framework.Assert.Fail("did not fail to finalize");
}
catch (JournalOutOfSyncException e)
{
GenericTestUtils.AssertExceptionContains("disk only contains up to txid 3", e);
}
}
/// <summary>
/// Test that the NN initializes its under-replicated blocks queue
/// before it is ready to exit safemode (HDFS-1476)
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestInitializeReplQueuesEarly()
{
Log.Info("Starting testInitializeReplQueuesEarly");
// Spray the blocks around the cluster when we add DNs instead of
// concentrating all blocks on the first node.
BlockManagerTestUtil.SetWritingPrefersLocalNode(cluster.GetNamesystem().GetBlockManager
(), false);
cluster.StartDataNodes(conf, 2, true, HdfsServerConstants.StartupOption.Regular,
null);
cluster.WaitActive();
Log.Info("Creating files");
DFSTestUtil.CreateFile(fs, TestPath, 15 * BlockSize, (short)1, 1L);
Log.Info("Stopping all DataNodes");
IList <MiniDFSCluster.DataNodeProperties> dnprops = Lists.NewLinkedList();
dnprops.AddItem(cluster.StopDataNode(0));
dnprops.AddItem(cluster.StopDataNode(0));
dnprops.AddItem(cluster.StopDataNode(0));
cluster.GetConfiguration(0).SetFloat(DFSConfigKeys.DfsNamenodeReplQueueThresholdPctKey
, 1f / 15f);
Log.Info("Restarting NameNode");
cluster.RestartNameNode();
NameNode nn = cluster.GetNameNode();
string status = nn.GetNamesystem().GetSafemode();
NUnit.Framework.Assert.AreEqual("Safe mode is ON. The reported blocks 0 needs additional "
+ "15 blocks to reach the threshold 0.9990 of total blocks 15." + Newline + "The number of live datanodes 0 has reached the minimum number 0. "
+ "Safe mode will be turned off automatically once the thresholds " + "have been reached."
, status);
NUnit.Framework.Assert.IsFalse("Mis-replicated block queues should not be initialized "
+ "until threshold is crossed", NameNodeAdapter.SafeModeInitializedReplQueues(nn
));
Log.Info("Restarting one DataNode");
cluster.RestartDataNode(dnprops.Remove(0));
// Wait for block reports from all attached storages of
// the restarted DN to come in.
GenericTestUtils.WaitFor(new _Supplier_214(this), 10, 10000);
int safe = NameNodeAdapter.GetSafeModeSafeBlocks(nn);
NUnit.Framework.Assert.IsTrue("Expected first block report to make some blocks safe."
, safe > 0);
NUnit.Framework.Assert.IsTrue("Did not expect first block report to make all blocks safe."
, safe < 15);
NUnit.Framework.Assert.IsTrue(NameNodeAdapter.SafeModeInitializedReplQueues(nn));
// Ensure that UnderReplicatedBlocks goes up to 15 - safe. Misreplicated
// blocks are processed asynchronously so this may take a few seconds.
// Failure here will manifest as a test timeout.
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
long underReplicatedBlocks = nn.GetNamesystem().GetUnderReplicatedBlocks();
while (underReplicatedBlocks != (15 - safe))
{
Log.Info("UnderReplicatedBlocks expected=" + (15 - safe) + ", actual=" + underReplicatedBlocks
);
Sharpen.Thread.Sleep(100);
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
underReplicatedBlocks = nn.GetNamesystem().GetUnderReplicatedBlocks();
}
cluster.RestartDataNodes();
}
public virtual void Before()
{
BlockScanner.Conf.allowUnitTestSettings = true;
GenericTestUtils.SetLogLevel(BlockScanner.Log, Level.All);
GenericTestUtils.SetLogLevel(VolumeScanner.Log, Level.All);
GenericTestUtils.SetLogLevel(FsVolumeImpl.Log, Level.All);
}
public static void Init()
{
GenericTestUtils.AssumeInNativeProfile();
Configuration conf = new Configuration();
codec = CryptoCodec.GetInstance(conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestGracefulFailoverFailBecomingActive()
{
try
{
cluster.Start();
cluster.WaitForActiveLockHolder(0);
cluster.SetFailToBecomeActive(1, true);
// Ask for failover, it should fail and report back to user.
try
{
cluster.GetService(1).GetZKFCProxy(conf, 5000).GracefulFailover();
NUnit.Framework.Assert.Fail("Did not fail to graceful failover when target failed "
+ "to become active!");
}
catch (ServiceFailedException sfe)
{
GenericTestUtils.AssertExceptionContains("Couldn't make " + cluster.GetService(1)
+ " active", sfe);
GenericTestUtils.AssertExceptionContains("injected failure", sfe);
}
// No fencing
Assert.Equal(0, cluster.GetService(0).fenceCount);
Assert.Equal(0, cluster.GetService(1).fenceCount);
// Service 0 should go back to being active after the failed failover
cluster.WaitForActiveLockHolder(0);
}
finally
{
cluster.Stop();
}
}
/// <summary>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly.
/// </summary>
/// <remarks>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly. The datanode should time out sending the
/// chunks and the transceiver should die, even if it has a long keepalive.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSlowReader()
{
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 600000L;
Configuration clientConf = new Configuration(conf);
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testSlowReader");
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
// Restart the DN with a shorter write timeout.
MiniDFSCluster.DataNodeProperties props = cluster.StopDataNode(0);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketWriteTimeoutKey, WriteTimeout);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 120000);
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNode(props, true));
dn = cluster.GetDataNodes()[0];
// Wait for heartbeats to avoid a startup race where we
// try to write the block while the DN is still starting.
cluster.TriggerHeartbeats();
DFSTestUtil.CreateFile(fs, TestFile, 1024 * 1024 * 8L, (short)1, 0L);
FSDataInputStream stm = fs.Open(TestFile);
stm.Read();
AssertXceiverCount(1);
GenericTestUtils.WaitFor(new _Supplier_193(this), 500, 50000);
// DN should time out in sendChunks, and this should force
// the xceiver to exit.
IOUtils.CloseStream(stm);
}
/// <exception cref="System.Exception"/>
public virtual void TestMinBlockSizeLimit()
{
long blockSize = 4096;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodeMinBlockSizeKey, blockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// Try with min block size
fs.Create(new Path("/testmblock1"), true, 4096, (short)3, blockSize);
try
{
// Try with min block size - 1
fs.Create(new Path("/testmblock2"), true, 4096, (short)3, blockSize - 1);
System.Diagnostics.Debug.Assert(false, "Expected IOException after creating a file with small"
+ " blocks ");
}
catch (IOException e)
{
GenericTestUtils.AssertExceptionContains("Specified block size is less", e);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void BlockLengthHintIsPropagated()
{
string MethodName = GenericTestUtils.GetMethodName();
Path path = new Path("/" + MethodName + ".dat");
Configuration conf = new HdfsConfiguration();
TestWriteBlockGetsBlockLengthHint.FsDatasetChecker.SetFactory(conf);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockLength);
conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
cluster.WaitActive();
// FsDatasetChecker#createRbw asserts during block creation if the test
// fails.
DFSTestUtil.CreateFile(cluster.GetFileSystem(), path, 4096, ExpectedBlockLength,
ExpectedBlockLength, (short)1, unchecked ((int)(0x1BAD5EED)));
}
finally
{
// Buffer size.
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestListDotReserved()
{
// Create a base file for comparison
Path baseFileRaw = new Path("/.reserved/raw/base");
int len = 8192;
DFSTestUtil.CreateFile(fs, baseFileRaw, len, (short)1, unchecked ((int)(0xFEED)));
/*
* Ensure that you can't list /.reserved. Ever.
*/
try
{
fs.ListStatus(new Path("/.reserved"));
NUnit.Framework.Assert.Fail("expected FNFE");
}
catch (FileNotFoundException e)
{
GenericTestUtils.AssertExceptionContains("/.reserved does not exist", e);
}
try
{
fs.ListStatus(new Path("/.reserved/.inodes"));
NUnit.Framework.Assert.Fail("expected FNFE");
}
catch (FileNotFoundException e)
{
GenericTestUtils.AssertExceptionContains("/.reserved/.inodes does not exist", e);
}
FileStatus[] fileStatuses = fs.ListStatus(new Path("/.reserved/raw"));
NUnit.Framework.Assert.AreEqual("expected 1 entry", fileStatuses.Length, 1);
GenericTestUtils.AssertMatches(fileStatuses[0].GetPath().ToString(), "/.reserved/raw/base"
);
}
public virtual void TestSharedEditsMissingLogs()
{
RemoveStandbyNameDirs();
CheckpointSignature sig = nn0.GetRpcServer().RollEditLog();
NUnit.Framework.Assert.AreEqual(3, sig.GetCurSegmentTxId());
// Should have created edits_1-2 in shared edits dir
URI editsUri = cluster.GetSharedEditsDir(0, 1);
FilePath editsDir = new FilePath(editsUri);
FilePath editsSegment = new FilePath(new FilePath(editsDir, "current"), NNStorage
.GetFinalizedEditsFileName(1, 2));
GenericTestUtils.AssertExists(editsSegment);
// Delete the segment.
NUnit.Framework.Assert.IsTrue(editsSegment.Delete());
// Trying to bootstrap standby should now fail since the edit
// logs aren't available in the shared dir.
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(BootstrapStandby)));
try
{
int rc = BootstrapStandby.Run(new string[] { "-force" }, cluster.GetConfiguration
(1));
NUnit.Framework.Assert.AreEqual(BootstrapStandby.ErrCodeLogsUnavailable, rc);
}
finally
{
logs.StopCapturing();
}
GenericTestUtils.AssertMatches(logs.GetOutput(), "FATAL.*Unable to read transaction ids 1-3 from the configured shared"
);
}
// Expect NoSuchPaddingException
/// <exception cref="System.Exception"/>
public virtual void TestUpdateArguments()
{
Assume.AssumeTrue(OpensslCipher.GetLoadingFailureReason() == null);
OpensslCipher cipher = OpensslCipher.GetInstance("AES/CTR/NoPadding");
Assert.True(cipher != null);
cipher.Init(OpensslCipher.EncryptMode, key, iv);
// Require direct buffers
ByteBuffer input = ByteBuffer.Allocate(1024);
ByteBuffer output = ByteBuffer.Allocate(1024);
try
{
cipher.Update(input, output);
NUnit.Framework.Assert.Fail("Input and output buffer should be direct buffer.");
}
catch (ArgumentException e)
{
GenericTestUtils.AssertExceptionContains("Direct buffers are required", e);
}
// Output buffer length should be sufficient to store output data
input = ByteBuffer.AllocateDirect(1024);
output = ByteBuffer.AllocateDirect(1000);
try
{
cipher.Update(input, output);
NUnit.Framework.Assert.Fail("Output buffer length should be sufficient " + "to store output data"
);
}
catch (ShortBufferException e)
{
GenericTestUtils.AssertExceptionContains("Output buffer is not sufficient", e);
}
}
public virtual void TestLoadLogsFromBuggyEarlierVersions()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop23BrokenAppendTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-with-buggy-append");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).NumDataNodes(0).WaitSafeMode
(false).StartupOption(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/tmp/io_data/test_io_0");
NUnit.Framework.Assert.AreEqual(2 * 1024 * 1024, fs.GetFileStatus(testPath).GetLen
());
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestFailureToTransitionCausesShutdown()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
// Set an illegal value for the trash emptier interval. This will cause
// the NN to fail to transition to the active state.
conf.SetLong(CommonConfigurationKeys.FsTrashIntervalKey, -1);
cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
()).NumDataNodes(0).CheckExitOnShutdown(false).Build();
cluster.WaitActive();
try
{
cluster.TransitionToActive(0);
NUnit.Framework.Assert.Fail("Transitioned to active but should not have been able to."
);
}
catch (ExitUtil.ExitException ee)
{
GenericTestUtils.AssertExceptionContains("Cannot start trash emptier with negative interval"
, ee);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestJournalLocking()
{
Assume.AssumeTrue(journal.GetStorage().GetStorageDir(0).IsLockSupported());
Storage.StorageDirectory sd = journal.GetStorage().GetStorageDir(0);
FilePath lockFile = new FilePath(sd.GetRoot(), Storage.StorageFileLock);
// Journal should be locked, since the format() call locks it.
GenericTestUtils.AssertExists(lockFile);
journal.NewEpoch(FakeNsinfo, 1);
try
{
new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption.Regular, mockErrorReporter
);
NUnit.Framework.Assert.Fail("Did not fail to create another journal in same dir");
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Cannot lock storage", ioe);
}
journal.Close();
// Journal should no longer be locked after the close() call.
// Hence, should be able to create a new Journal in the same dir.
Journal journal2 = new Journal(conf, TestLogDir, Jid, HdfsServerConstants.StartupOption
.Regular, mockErrorReporter);
journal2.NewEpoch(FakeNsinfo, 2);
journal2.Close();
}
public virtual void TestPurgeLogs()
{
for (int txid = 1; txid <= 5; txid++)
{
QJMTestUtil.WriteSegment(cluster, qjm, txid, 1, true);
}
FilePath curDir = cluster.GetCurrentDir(0, QJMTestUtil.Jid);
GenericTestUtils.AssertGlobEquals(curDir, "edits_.*", NNStorage.GetFinalizedEditsFileName
(1, 1), NNStorage.GetFinalizedEditsFileName(2, 2), NNStorage.GetFinalizedEditsFileName
(3, 3), NNStorage.GetFinalizedEditsFileName(4, 4), NNStorage.GetFinalizedEditsFileName
(5, 5));
FilePath paxosDir = new FilePath(curDir, "paxos");
GenericTestUtils.AssertExists(paxosDir);
// Create new files in the paxos directory, which should get purged too.
NUnit.Framework.Assert.IsTrue(new FilePath(paxosDir, "1").CreateNewFile());
NUnit.Framework.Assert.IsTrue(new FilePath(paxosDir, "3").CreateNewFile());
GenericTestUtils.AssertGlobEquals(paxosDir, "\\d+", "1", "3");
// Create some temporary files of the sort that are used during recovery.
NUnit.Framework.Assert.IsTrue(new FilePath(curDir, "edits_inprogress_0000000000000000001.epoch=140"
).CreateNewFile());
NUnit.Framework.Assert.IsTrue(new FilePath(curDir, "edits_inprogress_0000000000000000002.empty"
).CreateNewFile());
qjm.PurgeLogsOlderThan(3);
// Log purging is asynchronous, so we have to wait for the calls
// to be sent and respond before verifying.
WaitForAllPendingCalls(qjm.GetLoggerSetForTests());
// Older edits should be purged
GenericTestUtils.AssertGlobEquals(curDir, "edits_.*", NNStorage.GetFinalizedEditsFileName
(3, 3), NNStorage.GetFinalizedEditsFileName(4, 4), NNStorage.GetFinalizedEditsFileName
(5, 5));
// Older paxos files should be purged
GenericTestUtils.AssertGlobEquals(paxosDir, "\\d+", "3");
}
/// <summary>
/// Test cancellation of ongoing checkpoints when failover happens
/// mid-checkpoint during image upload from standby to active NN.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCheckpointCancellationDuringUpload()
{
// don't compress, we want a big image
cluster.GetConfiguration(0).SetBoolean(DFSConfigKeys.DfsImageCompressKey, false);
cluster.GetConfiguration(1).SetBoolean(DFSConfigKeys.DfsImageCompressKey, false);
// Throttle SBN upload to make it hang during upload to ANN
cluster.GetConfiguration(1).SetLong(DFSConfigKeys.DfsImageTransferRateKey, 100);
cluster.RestartNameNode(0);
cluster.RestartNameNode(1);
nn0 = cluster.GetNameNode(0);
nn1 = cluster.GetNameNode(1);
cluster.TransitionToActive(0);
DoEdits(0, 100);
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
HATestUtil.WaitForCheckpoint(cluster, 1, ImmutableList.Of(104));
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
// Wait to make sure background TransferFsImageUpload thread was cancelled.
// This needs to be done before the next test in the suite starts, so that a
// file descriptor is not held open during the next cluster init.
cluster.Shutdown();
cluster = null;
GenericTestUtils.WaitFor(new _Supplier_312(), 1000, 30000);
// Assert that former active did not accept the canceled checkpoint file.
NUnit.Framework.Assert.AreEqual(0, nn0.GetFSImage().GetMostRecentCheckpointTxId()
);
}
