public static void SetupCluster()
{
if (DomainSocket.GetLoadingFailureReason() != null)
{
return;
}
sockDir = new TemporarySocketDirectory();
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestParallelShortCircuitReadUnCached._PORT.sock"
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
// Enabling data transfer encryption should have no effect when using
// short-circuit local reads. This is a regression test for HDFS-5353.
conf.SetBoolean(DFSConfigKeys.DfsEncryptDataTransferKey, true);
conf.SetBoolean(DFSConfigKeys.DfsBlockAccessTokenEnableKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, true);
// We want to test reading from stale sockets.
conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 1);
conf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, 5 * 60 * 1000);
conf.SetInt(DFSConfigKeys.DfsClientSocketCacheCapacityKey, 32);
// Avoid using the FileInputStreamCache.
conf.SetInt(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheSizeKey, 0);
DomainSocket.DisableBindPathValidation();
DFSInputStream.tcpReadsDisabledForTesting = true;
SetupCluster(1, conf);
}
public static void SetupCluster()
{
// This is a test of the normal (TCP) read path. For this reason, we turn
// off both short-circuit local reads and UNIX domain socket data traffic.
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false);
// dfs.domain.socket.path should be ignored because the previous two keys
// were set to false. This is a regression test for HDFS-4473.
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/will/not/be/created");
SetupCluster(DefaultReplicationFactor, conf);
}
public static void SetupCluster()
{
DFSInputStream.tcpReadsDisabledForTesting = true;
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, string.Empty);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser
().GetShortUserName());
DomainSocket.DisableBindPathValidation();
SetupCluster(1, conf);
}
public static void SetupCluster()
{
if (DomainSocket.GetLoadingFailureReason() != null)
{
return;
}
DFSInputStream.tcpReadsDisabledForTesting = true;
sockDir = new TemporarySocketDirectory();
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestParallelLocalRead.%d.sock"
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
DomainSocket.DisableBindPathValidation();
SetupCluster(1, conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestBlockReaderLocalLegacyWithAppend()
{
short ReplFactor = 1;
HdfsConfiguration conf = GetConfiguration(null);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path path = new Path("/testBlockReaderLocalLegacy");
DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0);
DFSTestUtil.WaitReplication(dfs, path, ReplFactor);
ClientDatanodeProtocol proxy;
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token;
ExtendedBlock originalBlock;
long originalGS;
{
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000
, false);
token = lb.GetBlockToken();
// get block and generation stamp
ExtendedBlock blk = new ExtendedBlock(lb.GetBlock());
originalBlock = new ExtendedBlock(blk);
originalGS = originalBlock.GetGenerationStamp();
// test getBlockLocalPathInfo
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token);
NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp());
}
{
// append one byte
FSDataOutputStream @out = dfs.Append(path);
@out.Write(1);
@out.Close();
}
{
// get new generation stamp
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
long newGS = lb.GetBlock().GetGenerationStamp();
NUnit.Framework.Assert.IsTrue(newGS > originalGS);
// getBlockLocalPathInfo using the original block.
NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp());
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token);
NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp());
}
cluster.Shutdown();
}
/// <exception cref="System.IO.IOException"/>
private static HdfsConfiguration GetConfiguration(TemporarySocketDirectory socketDir
)
{
HdfsConfiguration conf = new HdfsConfiguration();
if (socketDir == null)
{
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, string.Empty);
}
else
{
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(socketDir.GetDir(), "TestBlockReaderLocalLegacy.%d.sock"
).GetAbsolutePath());
}
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser
().GetShortUserName());
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false);
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
return(conf);
}
/// <summary>Tests replication in DFS.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void RunReplication(bool simulated)
{
Configuration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, false);
if (simulated)
{
SimulatedFSDataset.SetFactory(conf);
}
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes
).Racks(racks).Build();
cluster.WaitActive();
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, conf);
DatanodeInfo[] info = client.DatanodeReport(HdfsConstants.DatanodeReportType.Live
);
NUnit.Framework.Assert.AreEqual("Number of Datanodes ", numDatanodes, info.Length
);
FileSystem fileSys = cluster.GetFileSystem();
try
{
Path file1 = new Path("/smallblocktest.dat");
WriteFile(fileSys, file1, 3);
CheckFile(fileSys, file1, 3);
CleanupFile(fileSys, file1);
WriteFile(fileSys, file1, 10);
CheckFile(fileSys, file1, 10);
CleanupFile(fileSys, file1);
WriteFile(fileSys, file1, 4);
CheckFile(fileSys, file1, 4);
CleanupFile(fileSys, file1);
WriteFile(fileSys, file1, 1);
CheckFile(fileSys, file1, 1);
CleanupFile(fileSys, file1);
WriteFile(fileSys, file1, 2);
CheckFile(fileSys, file1, 2);
CleanupFile(fileSys, file1);
}
finally
{
fileSys.Close();
cluster.Shutdown();
}
}
public virtual void TestAppendAfterSoftLimit()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
conf.SetBoolean(DFSConfigKeys.DfsSupportAppendKey, true);
//Set small soft-limit for lease
long softLimit = 1L;
long hardLimit = 9999999L;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.SetLeasePeriod(softLimit, hardLimit);
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
FileSystem fs2 = new DistributedFileSystem();
fs2.Initialize(fs.GetUri(), conf);
Path testPath = new Path("/testAppendAfterSoftLimit");
byte[] fileContents = AppendTestUtil.InitBuffer(32);
// create a new file without closing
FSDataOutputStream @out = fs.Create(testPath);
@out.Write(fileContents);
//Wait for > soft-limit
Sharpen.Thread.Sleep(250);
try
{
FSDataOutputStream appendStream2 = fs2.Append(testPath);
appendStream2.Write(fileContents);
appendStream2.Close();
NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen
());
}
finally
{
fs.Close();
fs2.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
private void TestStatistics(bool isShortCircuit)
{
Assume.AssumeTrue(DomainSocket.GetLoadingFailureReason() == null);
HdfsConfiguration conf = new HdfsConfiguration();
TemporarySocketDirectory sockDir = null;
if (isShortCircuit)
{
DFSInputStream.tcpReadsDisabledForTesting = true;
sockDir = new TemporarySocketDirectory();
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestStatisticsForLocalRead.%d.sock"
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
DomainSocket.DisableBindPathValidation();
}
else
{
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
}
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn;
NUnit.Framework.Assert.AreEqual(TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, dfsIn.GetReadStatistics().GetTotalBytesRead());
NUnit.Framework.Assert.AreEqual(TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, dfsIn.GetReadStatistics().GetTotalLocalBytesRead());
if (isShortCircuit)
{
NUnit.Framework.Assert.AreEqual(TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, dfsIn.GetReadStatistics().GetTotalShortCircuitBytesRead());
}
else
{
NUnit.Framework.Assert.AreEqual(0, dfsIn.GetReadStatistics().GetTotalShortCircuitBytesRead
());
}
fsIn.Close();
fsIn = null;
}
finally
{
DFSInputStream.tcpReadsDisabledForTesting = false;
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (sockDir != null)
{
sockDir.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest
test, bool checksum, long readahead)
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum
);
conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead);
test.SetConfiguration(conf);
FileInputStream dataIn = null;
FileInputStream metaIn = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
BlockReaderLocal blockReaderLocal = null;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
ShortCircuitShm shm = null;
RandomAccessFile raf = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
fsIn.Close();
fsIn = null;
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
FilePath metaFile = cluster.GetBlockMetadataFile(0, block);
ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache
();
cluster.Shutdown();
cluster = null;
test.Setup(dataFile, checksum);
FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile)
, new FileInputStream(metaFile) };
dataIn = streams[0];
metaIn = streams[1];
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID.
RandomUUID().ToString()), "rw");
raf.SetLength(8192);
FileInputStream shmStream = new FileInputStream(raf.GetFD());
shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream);
ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache
, Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block))
);
blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename
(TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy
(new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build();
dataIn = null;
metaIn = null;
test.DoTest(blockReaderLocal, original);
// BlockReaderLocal should not alter the file position.
NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position());
NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position());
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (dataIn != null)
{
dataIn.Close();
}
if (metaIn != null)
{
metaIn.Close();
}
if (blockReaderLocal != null)
{
blockReaderLocal.Close();
}
if (shm != null)
{
shm.Free();
}
if (raf != null)
{
raf.Close();
}
}
}
public virtual void TestMultipleFilesSmallerThanOneBlock()
{
Configuration conf = new HdfsConfiguration();
int BlockSize = 6 * 1024;
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
// Make it relinquish locks. When run serially, the result should
// be identical.
conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
DFSAdmin admin = new DFSAdmin(conf);
string nnAddr = conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey);
string webhdfsuri = WebHdfsFileSystem.Scheme + "://" + nnAddr;
System.Console.Out.WriteLine("webhdfsuri=" + webhdfsuri);
FileSystem webhdfs = new Path(webhdfsuri).GetFileSystem(conf);
try
{
Path dir = new Path("/test");
bool exceededQuota = false;
ContentSummary c;
// 1kb file
// 6kb block
// 192kb quota
int FileSize = 1024;
int QuotaSize = 32 * (int)fs.GetDefaultBlockSize(dir);
NUnit.Framework.Assert.AreEqual(6 * 1024, fs.GetDefaultBlockSize(dir));
NUnit.Framework.Assert.AreEqual(192 * 1024, QuotaSize);
// Create the dir and set the quota. We need to enable the quota before
// writing the files as setting the quota afterwards will over-write
// the cached disk space used for quota verification with the actual
// amount used as calculated by INode#spaceConsumedInTree.
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(QuotaSize)
, dir.ToString());
// We can create at most 59 files because block allocation is
// conservative and initially assumes a full block is used, so we
// need to leave at least 3 * BLOCK_SIZE free space when allocating
// the last block: (58 * 3 * 1024) (3 * 6 * 1024) = 192kb
for (int i = 0; i < 59; i++)
{
Path file = new Path("/test/test" + i);
DFSTestUtil.CreateFile(fs, file, FileSize, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file, (short)3);
}
// Should account for all 59 files (almost QUOTA_SIZE)
c = fs.GetContentSummary(dir);
CheckContentSummary(c, webhdfs.GetContentSummary(dir));
NUnit.Framework.Assert.AreEqual("Invalid space consumed", 59 * FileSize * 3, c.GetSpaceConsumed
());
NUnit.Framework.Assert.AreEqual("Invalid space consumed", QuotaSize - (59 * FileSize
* 3), 3 * (fs.GetDefaultBlockSize(dir) - FileSize));
// Now check that trying to create another file violates the quota
try
{
Path file = new Path("/test/test59");
DFSTestUtil.CreateFile(fs, file, FileSize, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file, (short)3);
}
catch (QuotaExceededException)
{
exceededQuota = true;
}
NUnit.Framework.Assert.IsTrue("Quota not exceeded", exceededQuota);
NUnit.Framework.Assert.AreEqual(2, cluster.GetNamesystem().GetFSDirectory().GetYieldCount
());
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestBlockAllocationAdjustsUsageConservatively()
{
Configuration conf = new HdfsConfiguration();
int BlockSize = 6 * 1024;
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetBoolean(DFSConfigKeys.DfsWebhdfsEnabledKey, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
DFSAdmin admin = new DFSAdmin(conf);
string nnAddr = conf.Get(DFSConfigKeys.DfsNamenodeHttpAddressKey);
string webhdfsuri = WebHdfsFileSystem.Scheme + "://" + nnAddr;
System.Console.Out.WriteLine("webhdfsuri=" + webhdfsuri);
FileSystem webhdfs = new Path(webhdfsuri).GetFileSystem(conf);
try
{
Path dir = new Path("/test");
Path file1 = new Path("/test/test1");
Path file2 = new Path("/test/test2");
bool exceededQuota = false;
int QuotaSize = 3 * BlockSize;
// total space usage including
// repl.
int FileSize = BlockSize / 2;
ContentSummary c;
// Create the directory and set the quota
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
RunCommand(admin, false, "-setSpaceQuota", Sharpen.Extensions.ToString(QuotaSize)
, dir.ToString());
// Creating a file should use half the quota
DFSTestUtil.CreateFile(fs, file1, FileSize, (short)3, 1L);
DFSTestUtil.WaitReplication(fs, file1, (short)3);
c = fs.GetContentSummary(dir);
CheckContentSummary(c, webhdfs.GetContentSummary(dir));
NUnit.Framework.Assert.AreEqual("Quota is half consumed", QuotaSize / 2, c.GetSpaceConsumed
());
// We can not create the 2nd file because even though the total spaced
// used by two files (2 * 3 * 512/2) would fit within the quota (3 * 512)
// when a block for a file is created the space used is adjusted
// conservatively (3 * block size, ie assumes a full block is written)
// which will violate the quota (3 * block size) since we've already
// used half the quota for the first file.
try
{
DFSTestUtil.CreateFile(fs, file2, FileSize, (short)3, 1L);
}
catch (QuotaExceededException)
{
exceededQuota = true;
}
NUnit.Framework.Assert.IsTrue("Quota not exceeded", exceededQuota);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestReadSelectNonStaleDatanode()
{
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
long staleInterval = 30 * 1000 * 60;
conf.SetLong(DFSConfigKeys.DfsNamenodeStaleDatanodeIntervalKey, staleInterval);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes
).Racks(racks).Build();
cluster.WaitActive();
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, conf);
IList <DatanodeDescriptor> nodeInfoList = cluster.GetNameNode().GetNamesystem().GetBlockManager
().GetDatanodeManager().GetDatanodeListForReport(HdfsConstants.DatanodeReportType
.Live);
NUnit.Framework.Assert.AreEqual("Unexpected number of datanodes", numDatanodes, nodeInfoList
.Count);
FileSystem fileSys = cluster.GetFileSystem();
FSDataOutputStream stm = null;
try
{
// do the writing but do not close the FSDataOutputStream
// in order to mimic the ongoing writing
Path fileName = new Path("/file1");
stm = fileSys.Create(fileName, true, fileSys.GetConf().GetInt(CommonConfigurationKeys
.IoFileBufferSizeKey, 4096), (short)3, blockSize);
stm.Write(new byte[(blockSize * 3) / 2]);
// We do not close the stream so that
// the writing seems to be still ongoing
stm.Hflush();
LocatedBlocks blocks = client.GetNamenode().GetBlockLocations(fileName.ToString()
, 0, blockSize);
DatanodeInfo[] nodes = blocks.Get(0).GetLocations();
NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
DataNode staleNode = null;
DatanodeDescriptor staleNodeInfo = null;
// stop the heartbeat of the first node
staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
NUnit.Framework.Assert.IsNotNull(staleNode);
// set the first node as stale
staleNodeInfo = cluster.GetNameNode().GetNamesystem().GetBlockManager().GetDatanodeManager
().GetDatanode(staleNode.GetDatanodeId());
DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, -(staleInterval + 1));
LocatedBlocks blocksAfterStale = client.GetNamenode().GetBlockLocations(fileName.
ToString(), 0, blockSize);
DatanodeInfo[] nodesAfterStale = blocksAfterStale.Get(0).GetLocations();
NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
());
// restart the staleNode's heartbeat
DataNodeTestUtils.SetHeartbeatsDisabledForTests(staleNode, false);
// reset the first node as non-stale, so as to avoid two stale nodes
DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, 0);
LocatedBlock lastBlock = client.GetLocatedBlocks(fileName.ToString(), 0, long.MaxValue
).GetLastLocatedBlock();
nodes = lastBlock.GetLocations();
NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
// stop the heartbeat of the first node for the last block
staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
NUnit.Framework.Assert.IsNotNull(staleNode);
// set the node as stale
DatanodeDescriptor dnDesc = cluster.GetNameNode().GetNamesystem().GetBlockManager
().GetDatanodeManager().GetDatanode(staleNode.GetDatanodeId());
DFSTestUtil.ResetLastUpdatesWithOffset(dnDesc, -(staleInterval + 1));
LocatedBlock lastBlockAfterStale = client.GetLocatedBlocks(fileName.ToString(), 0
, long.MaxValue).GetLastLocatedBlock();
nodesAfterStale = lastBlockAfterStale.GetLocations();
NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
());
}
finally
{
if (stm != null)
{
stm.Close();
}
client.Close();
cluster.Shutdown();
}
}
/// <summary>Tests all FsEditLogOps that are converted to inotify events.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException"/>
public virtual void TestBasic()
{
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true);
// so that we can get an atime change
conf.SetLong(DFSConfigKeys.DfsNamenodeAccesstimePrecisionKey, 1);
MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf);
builder.GetDfsBuilder().NumDataNodes(2);
MiniQJMHACluster cluster = builder.Build();
try
{
cluster.GetDfsCluster().WaitActive();
cluster.GetDfsCluster().TransitionToActive(0);
DFSClient client = new DFSClient(cluster.GetDfsCluster().GetNameNode(0).GetNameNodeAddress
(), conf);
FileSystem fs = cluster.GetDfsCluster().GetFileSystem(0);
DFSTestUtil.CreateFile(fs, new Path("/file"), BlockSize, (short)1, 0L);
DFSTestUtil.CreateFile(fs, new Path("/file3"), BlockSize, (short)1, 0L);
DFSTestUtil.CreateFile(fs, new Path("/file5"), BlockSize, (short)1, 0L);
DFSInotifyEventInputStream eis = client.GetInotifyEventStream();
client.Rename("/file", "/file4", null);
// RenameOp -> RenameEvent
client.Rename("/file4", "/file2");
// RenameOldOp -> RenameEvent
// DeleteOp, AddOp -> UnlinkEvent, CreateEvent
OutputStream os = client.Create("/file2", true, (short)2, BlockSize);
os.Write(new byte[BlockSize]);
os.Close();
// CloseOp -> CloseEvent
// AddOp -> AppendEvent
os = client.Append("/file2", BlockSize, EnumSet.Of(CreateFlag.Append), null, null
);
os.Write(new byte[BlockSize]);
os.Close();
// CloseOp -> CloseEvent
Sharpen.Thread.Sleep(10);
// so that the atime will get updated on the next line
client.Open("/file2").Read(new byte[1]);
// TimesOp -> MetadataUpdateEvent
// SetReplicationOp -> MetadataUpdateEvent
client.SetReplication("/file2", (short)1);
// ConcatDeleteOp -> AppendEvent, UnlinkEvent, CloseEvent
client.Concat("/file2", new string[] { "/file3" });
client.Delete("/file2", false);
// DeleteOp -> UnlinkEvent
client.Mkdirs("/dir", null, false);
// MkdirOp -> CreateEvent
// SetPermissionsOp -> MetadataUpdateEvent
client.SetPermission("/dir", FsPermission.ValueOf("-rw-rw-rw-"));
// SetOwnerOp -> MetadataUpdateEvent
client.SetOwner("/dir", "username", "groupname");
client.CreateSymlink("/dir", "/dir2", false);
// SymlinkOp -> CreateEvent
client.SetXAttr("/file5", "user.field", Sharpen.Runtime.GetBytesForString("value"
), EnumSet.Of(XAttrSetFlag.Create));
// SetXAttrOp -> MetadataUpdateEvent
// RemoveXAttrOp -> MetadataUpdateEvent
client.RemoveXAttr("/file5", "user.field");
// SetAclOp -> MetadataUpdateEvent
client.SetAcl("/file5", AclEntry.ParseAclSpec("user::rwx,user:foo:rw-,group::r--,other::---"
, true));
client.RemoveAcl("/file5");
// SetAclOp -> MetadataUpdateEvent
client.Rename("/file5", "/dir");
// RenameOldOp -> RenameEvent
EventBatch batch = null;
// RenameOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
long txid = batch.GetTxid();
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Rename);
Event.RenameEvent re = (Event.RenameEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.AreEqual("/file4", re.GetDstPath());
NUnit.Framework.Assert.AreEqual("/file", re.GetSrcPath());
NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0);
long eventsBehind = eis.GetTxidsBehindEstimate();
// RenameOldOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Rename);
Event.RenameEvent re2 = (Event.RenameEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(re2.GetDstPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(re2.GetSrcPath().Equals("/file4"));
NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0);
// AddOp with overwrite
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
Event.CreateEvent ce = (Event.CreateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce.GetiNodeType() == Event.CreateEvent.INodeType.File
);
NUnit.Framework.Assert.IsTrue(ce.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ce.GetCtime() > 0);
NUnit.Framework.Assert.IsTrue(ce.GetReplication() > 0);
NUnit.Framework.Assert.IsTrue(ce.GetSymlinkTarget() == null);
NUnit.Framework.Assert.IsTrue(ce.GetOverwrite());
NUnit.Framework.Assert.AreEqual(BlockSize, ce.GetDefaultBlockSize());
// CloseOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Close);
Event.CloseEvent ce2 = (Event.CloseEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce2.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ce2.GetFileSize() > 0);
NUnit.Framework.Assert.IsTrue(ce2.GetTimestamp() > 0);
// AppendOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Append);
Event.AppendEvent append2 = (Event.AppendEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.AreEqual("/file2", append2.GetPath());
NUnit.Framework.Assert.IsFalse(append2.ToNewBlock());
// CloseOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Close);
NUnit.Framework.Assert.IsTrue(((Event.CloseEvent)batch.GetEvents()[0]).GetPath().
Equals("/file2"));
// TimesOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(mue.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Times);
// SetReplicationOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue2 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue2.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(mue2.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Replication);
NUnit.Framework.Assert.IsTrue(mue2.GetReplication() == 1);
// ConcatDeleteOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(3, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Append);
NUnit.Framework.Assert.IsTrue(((Event.AppendEvent)batch.GetEvents()[0]).GetPath()
.Equals("/file2"));
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[1].GetEventType() == Event.EventType
.Unlink);
Event.UnlinkEvent ue2 = (Event.UnlinkEvent)batch.GetEvents()[1];
NUnit.Framework.Assert.IsTrue(ue2.GetPath().Equals("/file3"));
NUnit.Framework.Assert.IsTrue(ue2.GetTimestamp() > 0);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[2].GetEventType() == Event.EventType
.Close);
Event.CloseEvent ce3 = (Event.CloseEvent)batch.GetEvents()[2];
NUnit.Framework.Assert.IsTrue(ce3.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ce3.GetTimestamp() > 0);
// DeleteOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Unlink);
Event.UnlinkEvent ue = (Event.UnlinkEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ue.GetPath().Equals("/file2"));
NUnit.Framework.Assert.IsTrue(ue.GetTimestamp() > 0);
// MkdirOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
Event.CreateEvent ce4 = (Event.CreateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce4.GetiNodeType() == Event.CreateEvent.INodeType.Directory
);
NUnit.Framework.Assert.IsTrue(ce4.GetPath().Equals("/dir"));
NUnit.Framework.Assert.IsTrue(ce4.GetCtime() > 0);
NUnit.Framework.Assert.IsTrue(ce4.GetReplication() == 0);
NUnit.Framework.Assert.IsTrue(ce4.GetSymlinkTarget() == null);
// SetPermissionsOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue3 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue3.GetPath().Equals("/dir"));
NUnit.Framework.Assert.IsTrue(mue3.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Perms);
NUnit.Framework.Assert.IsTrue(mue3.GetPerms().ToString().Contains("rw-rw-rw-"));
// SetOwnerOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue4 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue4.GetPath().Equals("/dir"));
NUnit.Framework.Assert.IsTrue(mue4.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Owner);
NUnit.Framework.Assert.IsTrue(mue4.GetOwnerName().Equals("username"));
NUnit.Framework.Assert.IsTrue(mue4.GetGroupName().Equals("groupname"));
// SymlinkOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Create);
Event.CreateEvent ce5 = (Event.CreateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(ce5.GetiNodeType() == Event.CreateEvent.INodeType.Symlink
);
NUnit.Framework.Assert.IsTrue(ce5.GetPath().Equals("/dir2"));
NUnit.Framework.Assert.IsTrue(ce5.GetCtime() > 0);
NUnit.Framework.Assert.IsTrue(ce5.GetReplication() == 0);
NUnit.Framework.Assert.IsTrue(ce5.GetSymlinkTarget().Equals("/dir"));
// SetXAttrOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue5 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue5.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue5.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Xattrs);
NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs().Count == 1);
NUnit.Framework.Assert.IsTrue(mue5.GetxAttrs()[0].GetName().Contains("field"));
NUnit.Framework.Assert.IsTrue(!mue5.IsxAttrsRemoved());
// RemoveXAttrOp
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue6 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue6.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue6.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Xattrs);
NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs().Count == 1);
NUnit.Framework.Assert.IsTrue(mue6.GetxAttrs()[0].GetName().Contains("field"));
NUnit.Framework.Assert.IsTrue(mue6.IsxAttrsRemoved());
// SetAclOp (1)
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue7 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue7.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue7.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Acls);
NUnit.Framework.Assert.IsTrue(mue7.GetAcls().Contains(AclEntry.ParseAclEntry("user::rwx"
, true)));
// SetAclOp (2)
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Metadata);
Event.MetadataUpdateEvent mue8 = (Event.MetadataUpdateEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(mue8.GetPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(mue8.GetMetadataType() == Event.MetadataUpdateEvent.MetadataType
.Acls);
NUnit.Framework.Assert.IsTrue(mue8.GetAcls() == null);
// RenameOp (2)
batch = WaitForNextEvents(eis);
NUnit.Framework.Assert.AreEqual(1, batch.GetEvents().Length);
txid = CheckTxid(batch, txid);
NUnit.Framework.Assert.IsTrue(batch.GetEvents()[0].GetEventType() == Event.EventType
.Rename);
Event.RenameEvent re3 = (Event.RenameEvent)batch.GetEvents()[0];
NUnit.Framework.Assert.IsTrue(re3.GetDstPath().Equals("/dir/file5"));
NUnit.Framework.Assert.IsTrue(re3.GetSrcPath().Equals("/file5"));
NUnit.Framework.Assert.IsTrue(re.GetTimestamp() > 0);
// Returns null when there are no further events
NUnit.Framework.Assert.IsTrue(eis.Poll() == null);
// make sure the estimate hasn't changed since the above assertion
// tells us that we are fully caught up to the current namesystem state
// and we should not have been behind at all when eventsBehind was set
// either, since there were few enough events that they should have all
// been read to the client during the first poll() call
NUnit.Framework.Assert.IsTrue(eis.GetTxidsBehindEstimate() == eventsBehind);
}
finally
{
cluster.Shutdown();
}
}
