/// <summary>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly.
/// </summary>
/// <remarks>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly. The datanode should time out sending the
/// chunks and the transceiver should die, even if it has a long keepalive.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSlowReader()
{
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 600000L;
Configuration clientConf = new Configuration(conf);
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testSlowReader");
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
// Restart the DN with a shorter write timeout.
MiniDFSCluster.DataNodeProperties props = cluster.StopDataNode(0);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketWriteTimeoutKey, WriteTimeout);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 120000);
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNode(props, true));
dn = cluster.GetDataNodes()[0];
// Wait for heartbeats to avoid a startup race where we
// try to write the block while the DN is still starting.
cluster.TriggerHeartbeats();
DFSTestUtil.CreateFile(fs, TestFile, 1024 * 1024 * 8L, (short)1, 0L);
FSDataInputStream stm = fs.Open(TestFile);
stm.Read();
AssertXceiverCount(1);
GenericTestUtils.WaitFor(new _Supplier_193(this), 500, 50000);
// DN should time out in sendChunks, and this should force
// the xceiver to exit.
IOUtils.CloseStream(stm);
}
public virtual void TestDiffReportWithRenameOutside()
{
Path root = new Path("/");
Path dir1 = new Path(root, "dir1");
Path dir2 = new Path(root, "dir2");
Path foo = new Path(dir1, "foo");
Path fileInFoo = new Path(foo, "file");
Path bar = new Path(dir2, "bar");
Path fileInBar = new Path(bar, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed);
// create snapshot on /dir1
SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s0");
// move bar into dir1
Path newBar = new Path(dir1, "newBar");
hdfs.Rename(bar, newBar);
// move foo out of dir1 into dir2
Path newFoo = new Path(dir2, "new");
hdfs.Rename(foo, newFoo);
SnapshotTestHelper.CreateSnapshot(hdfs, dir1, "s1");
VerifyDiffReport(dir1, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Create, DFSUtil.String2Bytes(newBar.GetName())), new
SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes
(foo.GetName())));
}
public virtual void TestDiffReportWithRenameAndSnapshotDeletion()
{
Path root = new Path("/");
Path foo = new Path(root, "foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
// rename /foo to /foo2
Path foo2 = new Path(root, "foo2");
hdfs.Rename(foo, foo2);
// now /foo/bar becomes /foo2/bar
Path bar2 = new Path(foo2, "bar");
// delete snapshot s0 so that the withNameList inside of the WithCount node
// of foo becomes empty
hdfs.DeleteSnapshot(root, "s0");
// create snapshot s1 and rename bar again
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path bar3 = new Path(foo2, "bar-new");
hdfs.Rename(bar2, bar3);
// we always put modification on the file before rename
VerifyDiffReport(root, "s1", string.Empty, new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("foo2")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes("foo2/bar"), DFSUtil.String2Bytes
("foo2/bar-new")));
}
public virtual void TestBlocksRemovedWhileInSafeModeEditsArriveFirst()
{
Banner("Starting with NN0 active and NN1 standby, creating some blocks");
DFSTestUtil.CreateFile(fs, new Path("/test"), 10 * BlockSize, (short)3, 1L);
// Roll edit log so that, when the SBN restarts, it will load
// the namespace during startup.
nn0.GetRpcServer().RollEditLog();
Banner("Restarting standby");
RestartStandby();
// It will initially have all of the blocks necessary.
string status = nn1.GetNamesystem().GetSafemode();
NUnit.Framework.Assert.IsTrue("Bad safemode status: '" + status + "'", status.StartsWith
("Safe mode is ON. The reported blocks 10 has reached the threshold " + "0.9990 of total blocks 10. The number of live datanodes 3 has "
+ "reached the minimum number 0. In safe mode extension. " + "Safe mode will be turned off automatically"
));
// Delete those blocks while the SBN is in safe mode.
// Immediately roll the edit log before the actual deletions are sent
// to the DNs.
Banner("Removing the blocks without rolling the edit log");
fs.Delete(new Path("/test"), true);
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
// Should see removal of the blocks as well as their contribution to safe block count.
AssertSafeMode(nn1, 0, 0, 3, 0);
Banner("Triggering sending deletions to DNs and Deletion Reports");
BlockManagerTestUtil.ComputeAllPendingWork(nn0.GetNamesystem().GetBlockManager());
cluster.TriggerHeartbeats();
HATestUtil.WaitForDNDeletions(cluster);
cluster.TriggerDeletionReports();
// No change in assertion status here, but some of the consistency checks
// in safemode will fire here if we accidentally decrement safe block count
// below 0.
AssertSafeMode(nn1, 0, 0, 3, 0);
}
/// <summary>Test snapshot after file appending</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotAfterAppending()
{
Path file = new Path(dir, "file");
// 1. create snapshot --> create file --> append
SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString());
// 2. create snapshot --> modify the file --> append
hdfs.CreateSnapshot(dir, "s1");
hdfs.SetReplication(file, (short)(Replication - 1));
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize());
// 3. create snapshot --> append
hdfs.CreateSnapshot(dir, "s2");
DFSTestUtil.AppendFile(hdfs, file, Blocksize);
// check corresponding inodes
fileNode = (INodeFile)fsdir.GetINode(file.ToString());
NUnit.Framework.Assert.AreEqual(Replication - 1, fileNode.GetFileReplication());
NUnit.Framework.Assert.AreEqual(Blocksize * 4, fileNode.ComputeFileSize());
}
/// <summary>
/// If we have a UNIX domain socket configured,
/// and we have dfs.client.domain.socket.data.traffic set to true,
/// and short-circuit access fails, we should still be able to pass
/// data traffic over the UNIX domain socket.
/// </summary>
/// <remarks>
/// If we have a UNIX domain socket configured,
/// and we have dfs.client.domain.socket.data.traffic set to true,
/// and short-circuit access fails, we should still be able to pass
/// data traffic over the UNIX domain socket. Test this.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestFallbackFromShortCircuitToUnixDomainTraffic()
{
DFSInputStream.tcpReadsDisabledForTesting = true;
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
// The server is NOT configured with short-circuit local reads;
// the client is. Both support UNIX domain reads.
Configuration clientConf = CreateShortCircuitConf("testFallbackFromShortCircuitToUnixDomainTraffic"
, sockDir);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testFallbackFromShortCircuitToUnixDomainTraffic_clientContext"
);
clientConf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, true);
Configuration serverConf = new Configuration(clientConf);
serverConf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).NumDataNodes(1).Build
();
cluster.WaitActive();
FileSystem dfs = FileSystem.Get(cluster.GetURI(0), clientConf);
string TestFile = "/test_file";
int TestFileLen = 8193;
int Seed = unchecked ((int)(0xFADED));
DFSTestUtil.CreateFile(dfs, new Path(TestFile), TestFileLen, (short)1, Seed);
byte[] contents = DFSTestUtil.ReadFileBuffer(dfs, new Path(TestFile));
byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
NUnit.Framework.Assert.IsTrue(Arrays.Equals(contents, expected));
cluster.Shutdown();
sockDir.Close();
}
public virtual void TestEnterSafeModeInSBNShouldNotThrowNPE()
{
Banner("Starting with NN0 active and NN1 standby, creating some blocks");
DFSTestUtil.CreateFile(fs, new Path("/test"), 3 * BlockSize, (short)3, 1L);
// Roll edit log so that, when the SBN restarts, it will load
// the namespace during startup and enter safemode.
nn0.GetRpcServer().RollEditLog();
Banner("Creating some blocks that won't be in the edit log");
DFSTestUtil.CreateFile(fs, new Path("/test2"), 5 * BlockSize, (short)3, 1L);
Banner("Deleting the original blocks");
fs.Delete(new Path("/test"), true);
Banner("Restarting standby");
RestartStandby();
FSNamesystem namesystem = nn1.GetNamesystem();
string status = namesystem.GetSafemode();
NUnit.Framework.Assert.IsTrue("Bad safemode status: '" + status + "'", status.StartsWith
("Safe mode is ON."));
NameNodeAdapter.EnterSafeMode(nn1, false);
NUnit.Framework.Assert.IsTrue("Failed to enter into safemode in standby", namesystem
.IsInSafeMode());
NameNodeAdapter.EnterSafeMode(nn1, false);
NUnit.Framework.Assert.IsTrue("Failed to enter into safemode in standby", namesystem
.IsInSafeMode());
}
/// <summary>create a file with a length of <code>fileLen</code></summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
private void CreateFile(int index, long len)
{
FileSystem fs = cluster.GetFileSystem(index);
DFSTestUtil.CreateFile(fs, FilePath, len, replication, Random.NextLong());
DFSTestUtil.WaitReplication(fs, FilePath, replication);
}
/// <exception cref="System.Exception"/>
private void TestConcat()
{
Configuration config = GetProxiedFSConf();
config.SetLong(DFSConfigKeys.DfsBlockSizeKey, 1024);
if (!IsLocalFS())
{
FileSystem fs = FileSystem.Get(config);
fs.Mkdirs(GetProxiedFSTestDir());
Path path1 = new Path("/test/foo.txt");
Path path2 = new Path("/test/bar.txt");
Path path3 = new Path("/test/derp.txt");
DFSTestUtil.CreateFile(fs, path1, 1024, (short)3, 0);
DFSTestUtil.CreateFile(fs, path2, 1024, (short)3, 0);
DFSTestUtil.CreateFile(fs, path3, 1024, (short)3, 0);
fs.Close();
fs = GetHttpFSFileSystem();
fs.Concat(path1, new Path[] { path2, path3 });
fs.Close();
fs = FileSystem.Get(config);
NUnit.Framework.Assert.IsTrue(fs.Exists(path1));
NUnit.Framework.Assert.IsFalse(fs.Exists(path2));
NUnit.Framework.Assert.IsFalse(fs.Exists(path3));
fs.Close();
}
}
public virtual void TestChooseReplicaToDelete()
{
MiniDFSCluster cluster = null;
FileSystem fs = null;
try
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
fs = cluster.GetFileSystem();
FSNamesystem namesystem = cluster.GetNamesystem();
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300);
cluster.StartDataNodes(conf, 1, true, null, null, null);
DataNode lastDN = cluster.GetDataNodes()[3];
DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem
.GetBlockPoolId());
string lastDNid = dnReg.GetDatanodeUuid();
Path fileName = new Path("/foo2");
DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L);
DFSTestUtil.WaitReplication(fs, fileName, (short)4);
// Wait for tolerable number of heartbeats plus one
DatanodeDescriptor nodeInfo = null;
long lastHeartbeat = 0;
long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys
.DfsNamenodeTolerateHeartbeatMultiplierDefault + 1);
do
{
nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg);
lastHeartbeat = nodeInfo.GetLastUpdateMonotonic();
}while (Time.MonotonicNow() - lastHeartbeat < waitTime);
fs.SetReplication(fileName, (short)3);
BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue
);
// All replicas for deletion should be scheduled on lastDN.
// And should not actually be deleted, because lastDN does not heartbeat.
namesystem.ReadLock();
ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid
];
NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted"
, SmallFileLength / SmallBlockSize, dnBlocks.Count);
namesystem.ReadUnlock();
foreach (BlockLocation location in locs)
{
NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location
.GetNames().Length);
}
}
finally
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>create a file with a length of <code>filelen</code></summary>
/// <exception cref="System.IO.IOException"/>
private void CreateFile(string fileName, long filelen)
{
FileSystem fs = mc.GetFileSystem();
Path filePath = new Path(fileName);
DFSTestUtil.CreateFile(fs, filePath, filelen, (short)1, 0);
}
/// <summary>Create a file with one block and corrupt some/all of the block replicas.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
private void CreateAFileWithCorruptedBlockReplicas(Path filePath, short repl, int
corruptBlockCount)
{
DFSTestUtil.CreateFile(dfs, filePath, BlockSize, repl, 0);
DFSTestUtil.WaitReplication(dfs, filePath, repl);
// Locate the file blocks by asking name node
LocatedBlocks locatedblocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToString
(), 0L, BlockSize);
NUnit.Framework.Assert.AreEqual(repl, locatedblocks.Get(0).GetLocations().Length);
// The file only has one block
LocatedBlock lblock = locatedblocks.Get(0);
DatanodeInfo[] datanodeinfos = lblock.GetLocations();
ExtendedBlock block = lblock.GetBlock();
// corrupt some /all of the block replicas
for (int i = 0; i < corruptBlockCount; i++)
{
DatanodeInfo dninfo = datanodeinfos[i];
DataNode dn = cluster.GetDataNode(dninfo.GetIpcPort());
CorruptBlock(block, dn);
Log.Debug("Corrupted block " + block.GetBlockName() + " on data node " + dninfo);
}
}
public virtual void BlockLengthHintIsPropagated()
{
string MethodName = GenericTestUtils.GetMethodName();
Path path = new Path("/" + MethodName + ".dat");
Configuration conf = new HdfsConfiguration();
TestWriteBlockGetsBlockLengthHint.FsDatasetChecker.SetFactory(conf);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockLength);
conf.SetInt(DFSConfigKeys.DfsDatanodeScanPeriodHoursKey, -1);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
cluster.WaitActive();
// FsDatasetChecker#createRbw asserts during block creation if the test
// fails.
DFSTestUtil.CreateFile(cluster.GetFileSystem(), path, 4096, ExpectedBlockLength,
ExpectedBlockLength, (short)1, unchecked ((int)(0x1BAD5EED)));
}
finally
{
// Buffer size.
cluster.Shutdown();
}
}
/// <summary>Test that the client respects its keepalive timeout.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestClientResponsesKeepAliveTimeout()
{
Configuration clientConf = new Configuration(conf);
// Set a client socket cache expiry time much shorter than
// the datanode-side expiration time.
long ClientExpiryMs = 10L;
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testClientResponsesKeepAliveTimeout"
);
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
AssertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.ReadFile(fs, TestFile);
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
AssertXceiverCount(1);
// Sleep for a bit longer than the client keepalive timeout.
Sharpen.Thread.Sleep(ClientExpiryMs + 50);
// Taking out a peer which is expired should give a null.
Peer peer = peerCache.Get(dn.GetDatanodeId(), false);
NUnit.Framework.Assert.IsTrue(peer == null);
// The socket cache is now empty.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
}
public virtual void TestDataNodeMetrics()
{
Configuration conf = new HdfsConfiguration();
SimulatedFSDataset.SetFactory(conf);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
long LongFileLen = int.MaxValue + 1L;
DFSTestUtil.CreateFile(fs, new Path("/tmp.txt"), LongFileLen, (short)1, 1L);
IList <DataNode> datanodes = cluster.GetDataNodes();
NUnit.Framework.Assert.AreEqual(datanodes.Count, 1);
DataNode datanode = datanodes[0];
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name());
MetricsAsserts.AssertCounter("BytesWritten", LongFileLen, rb);
NUnit.Framework.Assert.IsTrue("Expected non-zero number of incremental block reports"
, MetricsAsserts.GetLongCounter("IncrementalBlockReportsNumOps", rb) > 0);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSetAndGetStoragePolicy()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(fs, bar, Size, Repl, 0);
StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of "
+ foo.ToString() + " is unspecified");
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo/bar", 0, "The storage policy of "
+ bar.ToString() + " is unspecified");
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /foo -policy WARM", 0, "Set storage policy WARM on "
+ foo.ToString());
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /foo/bar -policy COLD", 0, "Set storage policy COLD on "
+ bar.ToString());
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /fooz -policy WARM", 2, "File/Directory does not exist: /fooz"
);
BlockStoragePolicySuite suite = BlockStoragePolicySuite.CreateDefaultSuite();
BlockStoragePolicy warm = suite.GetPolicy("WARM");
BlockStoragePolicy cold = suite.GetPolicy("COLD");
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of "
+ foo.ToString() + ":\n" + warm);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo/bar", 0, "The storage policy of "
+ bar.ToString() + ":\n" + cold);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /fooz", 2, "File/Directory does not exist: /fooz"
);
}
public virtual void TestShortCircuitCacheShutdown()
{
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testShortCircuitCacheShutdown", sockDir
);
conf.Set(DFSConfigKeys.DfsClientContext, "testShortCircuitCacheShutdown");
Configuration serverConf = new Configuration(conf);
DFSInputStream.tcpReadsDisabledForTesting = true;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).NumDataNodes(1).Build
();
cluster.WaitActive();
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0
), conf);
string TestFile = "/test_file";
int TestFileLen = 4000;
int Seed = unchecked ((int)(0xFADEC));
DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
byte[] contents = DFSTestUtil.ReadFileBuffer(fs, new Path(TestFile));
byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
NUnit.Framework.Assert.IsTrue(Arrays.Equals(contents, expected));
ShortCircuitCache cache = fs.dfs.GetClientContext().GetShortCircuitCache();
cache.Close();
NUnit.Framework.Assert.IsTrue(cache.GetDfsClientShmManager().GetDomainSocketWatcher
().IsClosed());
cluster.Shutdown();
sockDir.Close();
}
/// <exception cref="System.Exception"/>
public virtual void CreateFiles(int nsIdx, int numFiles, int length)
{
for (int blockIdx = 0; blockIdx < numFiles; blockIdx++)
{
DFSTestUtil.CreateFile(dfs[nsIdx], GetPath(blockIdx), length, (short)1, 123L);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestXAttrSymlinks()
{
fs.Mkdirs(linkParent);
fs.Mkdirs(targetParent);
DFSTestUtil.CreateFile(fs, target, 1024, (short)3, unchecked ((long)(0xBEEFl)));
fs.CreateSymlink(target, link, false);
fs.SetXAttr(target, name1, value1);
fs.SetXAttr(target, name2, value2);
IDictionary <string, byte[]> xattrs = fs.GetXAttrs(link);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
fs.SetXAttr(link, name3, null);
xattrs = fs.GetXAttrs(target);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 3);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
Assert.AssertArrayEquals(new byte[0], xattrs[name3]);
fs.RemoveXAttr(link, name1);
xattrs = fs.GetXAttrs(target);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 2);
Assert.AssertArrayEquals(value2, xattrs[name2]);
Assert.AssertArrayEquals(new byte[0], xattrs[name3]);
fs.RemoveXAttr(target, name3);
xattrs = fs.GetXAttrs(link);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 1);
Assert.AssertArrayEquals(value2, xattrs[name2]);
fs.Delete(linkParent, true);
fs.Delete(targetParent, true);
}
public virtual void TestOpenManyFilesViaTcp()
{
int NumOpens = 500;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
MiniDFSCluster cluster = null;
FSDataInputStream[] streams = new FSDataInputStream[NumOpens];
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path TestPath = new Path("/testFile");
DFSTestUtil.CreateFile(dfs, TestPath, 131072, (short)1, 1);
for (int i = 0; i < NumOpens; i++)
{
streams[i] = dfs.Open(TestPath);
Log.Info("opening file " + i + "...");
NUnit.Framework.Assert.IsTrue(-1 != streams[i].Read());
streams[i].Unbuffer();
}
}
finally
{
foreach (FSDataInputStream stream in streams)
{
IOUtils.Cleanup(null, stream);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestBlocksRemovedWhileInSafeMode()
{
Banner("Starting with NN0 active and NN1 standby, creating some blocks");
DFSTestUtil.CreateFile(fs, new Path("/test"), 10 * BlockSize, (short)3, 1L);
// Roll edit log so that, when the SBN restarts, it will load
// the namespace during startup.
nn0.GetRpcServer().RollEditLog();
Banner("Restarting standby");
RestartStandby();
// It will initially have all of the blocks necessary.
AssertSafeMode(nn1, 10, 10, 3, 0);
// Delete those blocks while the SBN is in safe mode.
// This doesn't affect the SBN, since deletions are not
// ACKed when due to block removals.
Banner("Removing the blocks without rolling the edit log");
fs.Delete(new Path("/test"), true);
BlockManagerTestUtil.ComputeAllPendingWork(nn0.GetNamesystem().GetBlockManager());
Banner("Triggering deletions on DNs and Deletion Reports");
cluster.TriggerHeartbeats();
HATestUtil.WaitForDNDeletions(cluster);
cluster.TriggerDeletionReports();
AssertSafeMode(nn1, 10, 10, 3, 0);
// When we catch up to active namespace, it will restore back
// to 0 blocks.
Banner("Waiting for standby to catch up to active namespace");
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
AssertSafeMode(nn1, 0, 0, 3, 0);
}
/// <exception cref="System.Exception"/>
public virtual void TestUncacheQuiesces()
{
// Create a file
Path fileName = new Path("/testUncacheQuiesces");
int fileLen = 4096;
DFSTestUtil.CreateFile(fs, fileName, fileLen, (short)1, unchecked ((int)(0xFDFD)));
// Cache it
DistributedFileSystem dfs = cluster.GetFileSystem();
dfs.AddCachePool(new CachePoolInfo("pool"));
dfs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPool("pool").SetPath(fileName
).SetReplication((short)3).Build());
GenericTestUtils.WaitFor(new _Supplier_484(), 1000, 30000);
// Uncache it
dfs.RemoveCacheDirective(1);
GenericTestUtils.WaitFor(new _Supplier_495(), 1000, 30000);
// Make sure that no additional messages were sent
Sharpen.Thread.Sleep(10000);
MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(dn.GetMetrics().Name()
);
MetricsAsserts.AssertCounter("BlocksCached", 1l, dnMetrics);
MetricsAsserts.AssertCounter("BlocksUncached", 1l, dnMetrics);
}
/// <exception cref="System.Exception"/>
public virtual void TestSetrepIncWithUnderReplicatedBlocks()
{
// 1 min timeout
Configuration conf = new HdfsConfiguration();
short ReplicationFactor = 2;
string FileName = "/testFile";
Path FilePath = new Path(FileName);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor
+ 1).Build();
try
{
// create a file with one block with a replication factor of 2
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L);
DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor);
// remove one replica from the blocksMap so block becomes under-replicated
// but the block does not get put into the under-replicated blocks queue
BlockManager bm = cluster.GetNamesystem().GetBlockManager();
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, FilePath);
DatanodeDescriptor dn = bm.blocksMap.GetStorages(b.GetLocalBlock()).GetEnumerator
().Next().GetDatanodeDescriptor();
bm.AddToInvalidates(b.GetLocalBlock(), dn);
Sharpen.Thread.Sleep(5000);
bm.blocksMap.RemoveNode(b.GetLocalBlock(), dn);
// increment this file's replication factor
FsShell shell = new FsShell(conf);
NUnit.Framework.Assert.AreEqual(0, shell.Run(new string[] { "-setrep", "-w", Sharpen.Extensions.ToString
(1 + ReplicationFactor), FileName }));
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAdminAccessOnly()
{
Path zone = new Path("zone");
Path slashZone = new Path("/", zone);
fs.Mkdirs(slashZone);
dfsAdmin.CreateEncryptionZone(slashZone, TestKey);
Path @base = new Path("base");
Path reservedRaw = new Path("/.reserved/raw");
int len = 8192;
/* Test failure of create file in reserved/raw as non admin */
UserGroupInformation user = UserGroupInformation.CreateUserForTesting("user", new
string[] { "mygroup" });
user.DoAs(new _PrivilegedExceptionAction_233(this, reservedRaw, zone, @base, len)
);
/* Test failure of getFileStatus in reserved/raw as non admin */
Path ezRawEncFile = new Path(new Path(reservedRaw, zone), @base);
DFSTestUtil.CreateFile(fs, ezRawEncFile, len, (short)1, unchecked ((int)(0xFEED)));
user.DoAs(new _PrivilegedExceptionAction_252(this, ezRawEncFile));
/* Test failure of listStatus in reserved/raw as non admin */
user.DoAs(new _PrivilegedExceptionAction_267(this, ezRawEncFile));
fs.SetPermission(new Path("/"), new FsPermission((short)0x1ff));
/* Test failure of mkdir in reserved/raw as non admin */
user.DoAs(new _PrivilegedExceptionAction_283(this, reservedRaw));
}
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRollingUpgradeWithRollback()
{
try
{
StartCluster();
// Create files in DFS.
Path testFile1 = new Path("/" + GenericTestUtils.GetMethodName() + ".01.dat");
DFSTestUtil.CreateFile(fs, testFile1, FileSize, ReplFactor, Seed);
string fileContents1 = DFSTestUtil.ReadFile(fs, testFile1);
StartRollingUpgrade();
FilePath blockFile = GetBlockForFile(testFile1, true);
FilePath trashFile = GetTrashFileForBlock(blockFile, false);
DeleteAndEnsureInTrash(testFile1, blockFile, trashFile);
// Now perform a rollback to restore DFS to the pre-rollback state.
RollbackRollingUpgrade();
// Ensure that block was restored from trash
EnsureTrashRestored(blockFile, trashFile);
// Ensure that files exist and restored file contents are the same.
System.Diagnostics.Debug.Assert((fs.Exists(testFile1)));
string fileContents2 = DFSTestUtil.ReadFile(fs, testFile1);
Assert.AssertThat(fileContents1, IS.Is(fileContents2));
}
finally
{
ShutdownCluster();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestListDotReserved()
{
// Create a base file for comparison
Path baseFileRaw = new Path("/.reserved/raw/base");
int len = 8192;
DFSTestUtil.CreateFile(fs, baseFileRaw, len, (short)1, unchecked ((int)(0xFEED)));
/*
* Ensure that you can't list /.reserved. Ever.
*/
try
{
fs.ListStatus(new Path("/.reserved"));
NUnit.Framework.Assert.Fail("expected FNFE");
}
catch (FileNotFoundException e)
{
GenericTestUtils.AssertExceptionContains("/.reserved does not exist", e);
}
try
{
fs.ListStatus(new Path("/.reserved/.inodes"));
NUnit.Framework.Assert.Fail("expected FNFE");
}
catch (FileNotFoundException e)
{
GenericTestUtils.AssertExceptionContains("/.reserved/.inodes does not exist", e);
}
FileStatus[] fileStatuses = fs.ListStatus(new Path("/.reserved/raw"));
NUnit.Framework.Assert.AreEqual("expected 1 entry", fileStatuses.Length, 1);
GenericTestUtils.AssertMatches(fileStatuses[0].GetPath().ToString(), "/.reserved/raw/base"
);
}
public virtual void TestDiffReportWithRenameAndDelete()
{
Path root = new Path("/");
Path dir1 = new Path(root, "dir1");
Path dir2 = new Path(root, "dir2");
Path foo = new Path(dir1, "foo");
Path fileInFoo = new Path(foo, "file");
Path bar = new Path(dir2, "bar");
Path fileInBar = new Path(bar, "file");
DFSTestUtil.CreateFile(hdfs, fileInFoo, Blocksize, Replication, seed);
DFSTestUtil.CreateFile(hdfs, fileInBar, Blocksize, Replication, seed);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s0");
hdfs.Rename(fileInFoo, fileInBar, Options.Rename.Overwrite);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
VerifyDiffReport(root, "s0", "s1", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar/file")), new
SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType.Rename, DFSUtil.String2Bytes
("dir1/foo/file"), DFSUtil.String2Bytes("dir2/bar/file")));
// delete bar
hdfs.Delete(bar, true);
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
VerifyDiffReport(root, "s0", "s2", new SnapshotDiffReport.DiffReportEntry(SnapshotDiffReport.DiffType
.Modify, DFSUtil.String2Bytes(string.Empty)), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir1/foo")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Modify, DFSUtil.String2Bytes("dir2")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir2/bar")), new SnapshotDiffReport.DiffReportEntry
(SnapshotDiffReport.DiffType.Delete, DFSUtil.String2Bytes("dir1/foo/file")));
}
/// <summary>Basic read/write tests of raw files.</summary>
/// <remarks>
/// Basic read/write tests of raw files.
/// Create a non-encrypted file
/// Create an encryption zone
/// Verify that non-encrypted file contents and decrypted file in EZ are equal
/// Compare the raw encrypted bytes of the file with the decrypted version to
/// ensure they're different
/// Compare the raw and non-raw versions of the non-encrypted file to ensure
/// they're the same.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestReadWriteRaw()
{
// Create a base file for comparison
Path baseFile = new Path("/base");
int len = 8192;
DFSTestUtil.CreateFile(fs, baseFile, len, (short)1, unchecked ((int)(0xFEED)));
// Create the first enc file
Path zone = new Path("/zone");
fs.Mkdirs(zone);
dfsAdmin.CreateEncryptionZone(zone, TestKey);
Path encFile1 = new Path(zone, "myfile");
DFSTestUtil.CreateFile(fs, encFile1, len, (short)1, unchecked ((int)(0xFEED)));
// Read them back in and compare byte-by-byte
DFSTestUtil.VerifyFilesEqual(fs, baseFile, encFile1, len);
// Raw file should be different from encrypted file
Path encFile1Raw = new Path(zone, "/.reserved/raw/zone/myfile");
DFSTestUtil.VerifyFilesNotEqual(fs, encFile1Raw, encFile1, len);
// Raw file should be same as /base which is not in an EZ
Path baseFileRaw = new Path(zone, "/.reserved/raw/base");
DFSTestUtil.VerifyFilesEqual(fs, baseFile, baseFileRaw, len);
}
/// <summary>create a file with a length of <code>fileLen</code></summary>
/// <exception cref="System.IO.IOException"/>
private void CreateFile(string fileName, long fileLen)
{
filelist.AddItem(hftpURI + fileName);
Path filePath = new Path(fileName);
DFSTestUtil.CreateFile(fs, filePath, fileLen, (short)1, r.NextLong());
}
public virtual void TestShortCircuitTraceHooks()
{
Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows);
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix
, typeof(TestTracing.SetSpanReceiver).FullName);
conf.SetLong("dfs.blocksize", 100 * 1024);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT"
);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
dfs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L);
TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always);
FSDataInputStream stream = dfs.Open(TestPath);
byte[] buf = new byte[TestLength];
IOUtils.ReadFully(stream, buf, 0, TestLength);
stream.Close();
ts.Close();
string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" };
TestTracing.AssertSpanNamesFound(expectedSpanNames);
}
finally
{
dfs.Close();
cluster.Shutdown();
}
}