/// <summary>
/// Write to one file, then kill one datanode in the pipeline and then
/// close the file.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void SimpleTest(int datanodeToKill)
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 2000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 2);
conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 5000);
int myMaxNodes = 5;
System.Console.Out.WriteLine("SimpleTest starting with DataNode to Kill " + datanodeToKill
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(myMaxNodes
).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
short repl = 3;
Path filename = new Path("simpletest.dat");
try
{
// create a file and write one block of data
System.Console.Out.WriteLine("SimpleTest creating file " + filename);
FSDataOutputStream stm = CreateFile(fs, filename, repl);
DFSOutputStream dfstream = (DFSOutputStream)(stm.GetWrappedStream());
// these are test settings
dfstream.SetChunksPerPacket(5);
dfstream.SetArtificialSlowdown(3000);
long myseed = AppendTestUtil.NextLong();
byte[] buffer = AppendTestUtil.RandomBytes(myseed, fileSize);
int mid = fileSize / 4;
stm.Write(buffer, 0, mid);
DatanodeInfo[] targets = dfstream.GetPipeline();
int count = 5;
while (count-- > 0 && targets == null)
{
try
{
System.Console.Out.WriteLine("SimpleTest: Waiting for pipeline to be created.");
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
targets = dfstream.GetPipeline();
}
if (targets == null)
{
int victim = AppendTestUtil.NextInt(myMaxNodes);
System.Console.Out.WriteLine("SimpleTest stopping datanode random " + victim);
cluster.StopDataNode(victim);
}
else
{
int victim = datanodeToKill;
System.Console.Out.WriteLine("SimpleTest stopping datanode " + targets[victim]);
cluster.StopDataNode(targets[victim].GetXferAddr());
}
System.Console.Out.WriteLine("SimpleTest stopping datanode complete");
// write some more data to file, close and verify
stm.Write(buffer, mid, fileSize - mid);
stm.Close();
CheckFile(fs, filename, repl, numBlocks, fileSize, myseed);
}
catch (Exception e)
{
System.Console.Out.WriteLine("Simple Workload exception " + e);
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.IsTrue(e.ToString(), false);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
private void InitBuffer(int size)
{
long seed = AppendTestUtil.NextLong();
toWrite = AppendTestUtil.RandomBytes(seed, size);
}
/// <summary>
/// Test that writing to files is good even when datanodes in the pipeline
/// dies.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void ComplexTest()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 2000);
conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 2);
conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 2);
conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 5000);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes
).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
TestDatanodeDeath.Modify modThread = null;
try
{
// Create threads and make them run workload concurrently.
workload = new TestDatanodeDeath.Workload[numThreads];
for (int i = 0; i < numThreads; i++)
{
workload[i] = new TestDatanodeDeath.Workload(AppendTestUtil.NextLong(), fs, i, numberOfFiles
, replication, 0);
workload[i].Start();
}
// Create a thread that kills existing datanodes and creates new ones.
modThread = new TestDatanodeDeath.Modify(this, conf, cluster);
modThread.Start();
// wait for all transactions to get over
for (int i_1 = 0; i_1 < numThreads; i_1++)
{
try
{
System.Console.Out.WriteLine("Waiting for thread " + i_1 + " to complete...");
workload[i_1].Join();
// if most of the threads are done, then stop restarting datanodes.
if (i_1 >= numThreads / 2)
{
modThread.Close();
}
}
catch (Exception)
{
i_1--;
}
}
}
finally
{
// retry
if (modThread != null)
{
modThread.Close();
try
{
modThread.Join();
}
catch (Exception)
{
}
}
fs.Close();
cluster.Shutdown();
}
}
