public virtual void TestGetBlockLocations()
{
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
Path p = new Path(BaseDir, "file2.dat");
string src = p.ToString();
FSDataOutputStream @out = TestFileCreation.CreateFile(hdfs, p, 3);
// write a half block
int len = (int)(((uint)BlockSize) >> 1);
WriteFile(p, @out, len);
for (int i = 1; i < NumBlocks;)
{
// verify consistency
LocatedBlocks lb = namenode.GetBlockLocations(src, 0, len);
IList <LocatedBlock> blocks = lb.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(i, blocks.Count);
Block b = blocks[blocks.Count - 1].GetBlock().GetLocalBlock();
NUnit.Framework.Assert.IsTrue(b is BlockInfoContiguousUnderConstruction);
if (++i < NumBlocks)
{
// write one more block
WriteFile(p, @out, BlockSize);
len += BlockSize;
}
}
// close file
@out.Close();
}
void CreateTestFiles()
{
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_CONFIG", Path.Combine(tempTestPath, "config.yaml"));
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_RULES", Path.Combine(tempTestPath, "Rules"));
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_LAYOUT", Path.Combine(tempTestPath, "default.html"));
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_HOME_PAGE", Path.Combine(tempTestPath, "index.html"));
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_STYLE_SHEET", Path.Combine(tempTestPath, "stylesheet.css"));
}
/// <summary>Test that that writes to an incomplete block are available to a reader</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestUnfinishedBlockRead()
{
// create a new file in the root, write data, do no close
Path file1 = new Path("/unfinished-block");
FSDataOutputStream stm = TestFileCreation.CreateFile(fileSystem, file1, 1);
// write partial block and sync
int partialBlockSize = blockSize / 2;
WriteFileAndSync(stm, partialBlockSize);
// Make sure a client can read it before it is closed
CheckCanRead(fileSystem, file1, partialBlockSize);
stm.Close();
}
public virtual void TestBlockCreation()
{
Path file1 = new Path(BaseDir, "file1.dat");
FSDataOutputStream @out = TestFileCreation.CreateFile(hdfs, file1, 3);
for (int idx = 0; idx < NumBlocks; idx++)
{
// write one block
WriteFile(file1, @out, BlockSize);
// verify consistency
VerifyFileBlocks(file1.ToString(), true);
}
// close file
@out.Close();
// verify consistency
VerifyFileBlocks(file1.ToString(), false);
}
/// <summary>
/// test case: if the BlockSender decides there is only one packet to send,
/// the previous computation of the pktSize based on transferToAllowed
/// would result in too small a buffer to do the buffer-copy needed
/// for partial chunks.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestUnfinishedBlockPacketBufferOverrun()
{
// check that / exists
Path path = new Path("/");
System.Console.Out.WriteLine("Path : \"" + path.ToString() + "\"");
// create a new file in the root, write data, do no close
Path file1 = new Path("/unfinished-block");
FSDataOutputStream stm = TestFileCreation.CreateFile(fileSystem, file1, 1);
// write partial block and sync
int bytesPerChecksum = conf.GetInt("io.bytes.per.checksum", 512);
int partialBlockSize = bytesPerChecksum - 1;
WriteFileAndSync(stm, partialBlockSize);
// Make sure a client can read it before it is closed
CheckCanRead(fileSystem, file1, partialBlockSize);
stm.Close();
}
public virtual void TestLeaseExpireEmptyFiles()
{
Sharpen.Thread.UncaughtExceptionHandler oldUEH = Sharpen.Thread.GetDefaultUncaughtExceptionHandler
();
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new _UncaughtExceptionHandler_43
(this));
System.Console.Out.WriteLine("testLeaseExpireEmptyFiles start");
long leasePeriod = 1000;
int DatanodeNum = 3;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
// create cluster
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeNum
).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
// create a new file.
TestFileCreation.CreateFile(dfs, new Path("/foo"), DatanodeNum);
TestFileCreation.CreateFile(dfs, new Path("/foo2"), DatanodeNum);
TestFileCreation.CreateFile(dfs, new Path("/foo3"), DatanodeNum);
// set the soft and hard limit to be 1 second so that the
// namenode triggers lease recovery
cluster.SetLeasePeriod(leasePeriod, leasePeriod);
// wait for the lease to expire
try
{
Sharpen.Thread.Sleep(5 * leasePeriod);
}
catch (Exception)
{
}
NUnit.Framework.Assert.IsFalse(isConcurrentModificationException);
}
finally
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(oldUEH);
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual void WriteFile(Path file, FSDataOutputStream stm, int size)
{
long blocksBefore = stm.GetPos() / BlockSize;
TestFileCreation.WriteFile(stm, BlockSize);
// need to make sure the full block is completely flushed to the DataNodes
// (see FSOutputSummer#flush)
stm.Flush();
int blocksAfter = 0;
// wait until the block is allocated by DataStreamer
BlockLocation[] locatedBlocks;
while (blocksAfter <= blocksBefore)
{
locatedBlocks = DFSClientAdapter.GetDFSClient(hdfs).GetBlockLocations(file.ToString
(), 0L, BlockSize * NumBlocks);
blocksAfter = locatedBlocks == null ? 0 : locatedBlocks.Length;
}
}
void CreateTestFiles()
{
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_HOME_RESULT", Path.Combine(tempTestPath, "index.html"));
TestFileCreation.CreateFileFromResourceName("TEST_DEFAULT_STYLE_RESULT", Path.Combine(tempTestPath, "style.css"));
}
public virtual void TestWhileOpenRenameToNonExistentDirectory()
{
Configuration conf = new HdfsConfiguration();
int MaxIdleTime = 2000;
// 2s
conf.SetInt("ipc.client.connection.maxidletime", MaxIdleTime);
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, 1);
System.Console.Out.WriteLine("Test 4************************************");
// create cluster
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = null;
try
{
cluster.WaitActive();
fs = cluster.GetFileSystem();
int nnport = cluster.GetNameNodePort();
// create file1.
Path dir1 = new Path("/user/dir1");
Path file1 = new Path(dir1, "file1");
FSDataOutputStream stm1 = TestFileCreation.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("testFileCreationDeleteParent: " + "Created file " +
file1);
TestFileCreation.WriteFile(stm1);
stm1.Hflush();
Path dir2 = new Path("/user/dir2");
fs.Rename(file1, dir2);
// restart cluster with the same namenode port as before.
// This ensures that leases are persisted in fsimage.
cluster.Shutdown();
try
{
Sharpen.Thread.Sleep(2 * MaxIdleTime);
}
catch (Exception)
{
}
cluster = new MiniDFSCluster.Builder(conf).NameNodePort(nnport).Format(false).Build
();
cluster.WaitActive();
// restart cluster yet again. This triggers the code to read in
// persistent leases from fsimage.
cluster.Shutdown();
try
{
Sharpen.Thread.Sleep(5000);
}
catch (Exception)
{
}
cluster = new MiniDFSCluster.Builder(conf).NameNodePort(nnport).Format(false).Build
();
cluster.WaitActive();
fs = cluster.GetFileSystem();
Path newfile = new Path("/user", "dir2");
NUnit.Framework.Assert.IsTrue(!fs.Exists(file1));
NUnit.Framework.Assert.IsTrue(fs.Exists(newfile));
CheckFullFile(fs, newfile);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestWhileOpenRenameParent()
{
Configuration conf = new HdfsConfiguration();
int MaxIdleTime = 2000;
// 2s
conf.SetInt("ipc.client.connection.maxidletime", MaxIdleTime);
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, 1);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, TestFileCreation.blockSize);
// create cluster
System.Console.Out.WriteLine("Test 1*****************************");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = null;
try
{
cluster.WaitActive();
fs = cluster.GetFileSystem();
// Normally, the in-progress edit log would be finalized by
// FSEditLog#endCurrentLogSegment. For testing purposes, we
// disable that here.
FSEditLog spyLog = Org.Mockito.Mockito.Spy(cluster.GetNameNode().GetFSImage().GetEditLog
());
Org.Mockito.Mockito.DoNothing().When(spyLog).EndCurrentLogSegment(Org.Mockito.Mockito
.AnyBoolean());
DFSTestUtil.SetEditLogForTesting(cluster.GetNamesystem(), spyLog);
int nnport = cluster.GetNameNodePort();
// create file1.
Path dir1 = new Path("/user/a+b/dir1");
Path file1 = new Path(dir1, "file1");
FSDataOutputStream stm1 = TestFileCreation.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("testFileCreationDeleteParent: " + "Created file " +
file1);
TestFileCreation.WriteFile(stm1);
stm1.Hflush();
// create file2.
Path dir2 = new Path("/user/dir2");
Path file2 = new Path(dir2, "file2");
FSDataOutputStream stm2 = TestFileCreation.CreateFile(fs, file2, 1);
System.Console.Out.WriteLine("testFileCreationDeleteParent: " + "Created file " +
file2);
TestFileCreation.WriteFile(stm2);
stm2.Hflush();
// move dir1 while file1 is open
Path dir3 = new Path("/user/dir3");
fs.Mkdirs(dir3);
fs.Rename(dir1, dir3);
// create file3
Path file3 = new Path(dir3, "file3");
FSDataOutputStream stm3 = fs.Create(file3);
fs.Rename(file3, new Path(dir3, "bozo"));
// Get a new block for the file.
TestFileCreation.WriteFile(stm3, TestFileCreation.blockSize + 1);
stm3.Hflush();
// Stop the NameNode before closing the files.
// This will ensure that the write leases are still active and present
// in the edit log. Simiarly, there should be a pending ADD_BLOCK_OP
// for file3, since we just added a block to that file.
cluster.GetNameNode().Stop();
// Restart cluster with the same namenode port as before.
cluster.Shutdown();
try
{
Sharpen.Thread.Sleep(2 * MaxIdleTime);
}
catch (Exception)
{
}
cluster = new MiniDFSCluster.Builder(conf).NameNodePort(nnport).Format(false).Build
();
cluster.WaitActive();
// restart cluster yet again. This triggers the code to read in
// persistent leases from the edit log.
cluster.Shutdown();
try
{
Sharpen.Thread.Sleep(5000);
}
catch (Exception)
{
}
cluster = new MiniDFSCluster.Builder(conf).NameNodePort(nnport).Format(false).Build
();
cluster.WaitActive();
fs = cluster.GetFileSystem();
Path newfile = new Path("/user/dir3/dir1", "file1");
NUnit.Framework.Assert.IsTrue(!fs.Exists(file1));
NUnit.Framework.Assert.IsTrue(fs.Exists(file2));
NUnit.Framework.Assert.IsTrue(fs.Exists(newfile));
CheckFullFile(fs, newfile);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
