/// <summary> /// Write to one file, then kill one datanode in the pipeline and then /// close the file. /// </summary> /// <exception cref="System.IO.IOException"/> private void SimpleTest(int datanodeToKill) { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 2000); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 2); conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 5000); int myMaxNodes = 5; System.Console.Out.WriteLine("SimpleTest starting with DataNode to Kill " + datanodeToKill ); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(myMaxNodes ).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); short repl = 3; Path filename = new Path("simpletest.dat"); try { // create a file and write one block of data System.Console.Out.WriteLine("SimpleTest creating file " + filename); FSDataOutputStream stm = CreateFile(fs, filename, repl); DFSOutputStream dfstream = (DFSOutputStream)(stm.GetWrappedStream()); // these are test settings dfstream.SetChunksPerPacket(5); dfstream.SetArtificialSlowdown(3000); long myseed = AppendTestUtil.NextLong(); byte[] buffer = AppendTestUtil.RandomBytes(myseed, fileSize); int mid = fileSize / 4; stm.Write(buffer, 0, mid); DatanodeInfo[] targets = dfstream.GetPipeline(); int count = 5; while (count-- > 0 && targets == null) { try { System.Console.Out.WriteLine("SimpleTest: Waiting for pipeline to be created."); Sharpen.Thread.Sleep(1000); } catch (Exception) { } targets = dfstream.GetPipeline(); } if (targets == null) { int victim = AppendTestUtil.NextInt(myMaxNodes); System.Console.Out.WriteLine("SimpleTest stopping datanode random " + victim); cluster.StopDataNode(victim); } else { int victim = datanodeToKill; System.Console.Out.WriteLine("SimpleTest stopping datanode " + targets[victim]); cluster.StopDataNode(targets[victim].GetXferAddr()); } System.Console.Out.WriteLine("SimpleTest stopping datanode complete"); // write some more data to file, close and verify stm.Write(buffer, mid, fileSize - mid); stm.Close(); CheckFile(fs, filename, repl, numBlocks, fileSize, myseed); } catch (Exception e) { System.Console.Out.WriteLine("Simple Workload exception " + e); Sharpen.Runtime.PrintStackTrace(e); NUnit.Framework.Assert.IsTrue(e.ToString(), false); } finally { fs.Close(); cluster.Shutdown(); } }
private void InitBuffer(int size) { long seed = AppendTestUtil.NextLong(); toWrite = AppendTestUtil.RandomBytes(seed, size); }
/// <summary> /// Test that writing to files is good even when datanodes in the pipeline /// dies. /// </summary> /// <exception cref="System.IO.IOException"/> private void ComplexTest() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 2000); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 2); conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, 2); conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 5000); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes ).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); TestDatanodeDeath.Modify modThread = null; try { // Create threads and make them run workload concurrently. workload = new TestDatanodeDeath.Workload[numThreads]; for (int i = 0; i < numThreads; i++) { workload[i] = new TestDatanodeDeath.Workload(AppendTestUtil.NextLong(), fs, i, numberOfFiles , replication, 0); workload[i].Start(); } // Create a thread that kills existing datanodes and creates new ones. modThread = new TestDatanodeDeath.Modify(this, conf, cluster); modThread.Start(); // wait for all transactions to get over for (int i_1 = 0; i_1 < numThreads; i_1++) { try { System.Console.Out.WriteLine("Waiting for thread " + i_1 + " to complete..."); workload[i_1].Join(); // if most of the threads are done, then stop restarting datanodes. if (i_1 >= numThreads / 2) { modThread.Close(); } } catch (Exception) { i_1--; } } } finally { // retry if (modThread != null) { modThread.Close(); try { modThread.Join(); } catch (Exception) { } } fs.Close(); cluster.Shutdown(); } }