/// <exception cref="System.IO.IOException"/> internal static void Setrep(int fromREP, int toREP, bool simulatedStorage) { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } conf.Set(DFSConfigKeys.DfsReplicationKey, string.Empty + fromREP); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString (2)); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(10).Build( ); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); try { Path root = TestDFSShell.Mkdir(fs, new Path("/test/setrep" + fromREP + "-" + toREP )); Path f = TestDFSShell.WriteFile(fs, new Path(root, "foo")); { // Verify setrep for changing replication string[] args = new string[] { "-setrep", "-w", string.Empty + toREP, string.Empty + f }; FsShell shell = new FsShell(); shell.SetConf(conf); try { NUnit.Framework.Assert.AreEqual(0, shell.Run(args)); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("-setrep " + e, false); } } //get fs again since the old one may be closed fs = cluster.GetFileSystem(); FileStatus file = fs.GetFileStatus(f); long len = file.GetLen(); foreach (BlockLocation locations in fs.GetFileBlockLocations(file, 0, len)) { NUnit.Framework.Assert.IsTrue(locations.GetHosts().Length == toREP); } TestDFSShell.Show("done setrep waiting: " + root); } finally { try { fs.Close(); } catch (Exception) { } cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> private void DfsPreadTest(Configuration conf, bool disableTransferTo, bool verifyChecksum ) { conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 4096); conf.SetLong(DFSConfigKeys.DfsClientReadPrefetchSizeKey, 4096); // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0); if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } if (disableTransferTo) { conf.SetBoolean("dfs.datanode.transferTo.allowed", false); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); FileSystem fileSys = cluster.GetFileSystem(); fileSys.SetVerifyChecksum(verifyChecksum); try { Path file1 = new Path("preadtest.dat"); WriteFile(fileSys, file1); PReadFile(fileSys, file1); DatanodeRestartTest(cluster, fileSys, file1); CleanupFile(fileSys, file1); } finally { fileSys.Close(); cluster.Shutdown(); } }
private SimulatedFSDataset GetSimulatedFSDataset() { SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf); fsdataset.AddBlockPool(bpid, conf); return(fsdataset); }
// ok - as expected public virtual void CheckInvalidBlock(ExtendedBlock b) { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); NUnit.Framework.Assert.IsFalse(fsdataset.IsValidBlock(b)); try { fsdataset.GetLength(b); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } // ok - as expected try { fsdataset.GetBlockInputStream(b); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } // ok - as expected try { fsdataset.FinalizeBlock(b); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } }
public virtual void TestInjectionEmpty() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); int bytesAdded = AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } // Inject blocks into an empty fsdataset // - injecting the blocks we got above. SimulatedFSDataset sfsdataset = GetSimulatedFSDataset(); sfsdataset.InjectBlocks(bpid, blockReport); blockReport = sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b_1 in blockReport) { NUnit.Framework.Assert.IsNotNull(b_1); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes() ); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength (new ExtendedBlock(bpid, b_1))); } NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset .GetRemaining()); }
public virtual void TestSmallBlock() { Configuration conf = new HdfsConfiguration(); if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } conf.Set(DFSConfigKeys.DfsBytesPerChecksumKey, "1"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fileSys = cluster.GetFileSystem(); try { Path file1 = new Path("smallblocktest.dat"); WriteFile(fileSys, file1); CheckFile(fileSys, file1); CleanupFile(fileSys, file1); } finally { fileSys.Close(); cluster.Shutdown(); } }
public virtual void TestDataNodeMetrics() { Configuration conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); try { FileSystem fs = cluster.GetFileSystem(); long LongFileLen = int.MaxValue + 1L; DFSTestUtil.CreateFile(fs, new Path("/tmp.txt"), LongFileLen, (short)1, 1L); IList <DataNode> datanodes = cluster.GetDataNodes(); NUnit.Framework.Assert.AreEqual(datanodes.Count, 1); DataNode datanode = datanodes[0]; MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name()); MetricsAsserts.AssertCounter("BytesWritten", LongFileLen, rb); NUnit.Framework.Assert.IsTrue("Expected non-zero number of incremental block reports" , MetricsAsserts.GetLongCounter("IncrementalBlockReportsNumOps", rb) > 0); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestGetMetaData() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); try { NUnit.Framework.Assert.IsTrue(fsdataset.GetMetaDataInputStream(b) == null); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } // ok - as expected AddSomeBlocks(fsdataset); // Only need to add one but .... b = new ExtendedBlock(bpid, 1, 0, 0); InputStream metaInput = fsdataset.GetMetaDataInputStream(b); DataInputStream metaDataInput = new DataInputStream(metaInput); short version = metaDataInput.ReadShort(); NUnit.Framework.Assert.AreEqual(BlockMetadataHeader.Version, version); DataChecksum checksum = DataChecksum.NewDataChecksum(metaDataInput); NUnit.Framework.Assert.AreEqual(DataChecksum.Type.Null, checksum.GetChecksumType( )); NUnit.Framework.Assert.AreEqual(0, checksum.GetChecksumSize()); }
public virtual void TestInValidBlocks() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); CheckInvalidBlock(b); // Now check invlaid after adding some blocks AddSomeBlocks(fsdataset); b = new ExtendedBlock(bpid, Numblocks + 99, 5, 0); CheckInvalidBlock(b); }
public virtual void TestInjectionNonEmpty() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); int bytesAdded = AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } fsdataset = null; // Inject blocks into an non-empty fsdataset // - injecting the blocks we got above. SimulatedFSDataset sfsdataset = GetSimulatedFSDataset(); // Add come blocks whose block ids do not conflict with // the ones we are going to inject. bytesAdded += AddSomeBlocks(sfsdataset, Numblocks + 1); sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); sfsdataset.InjectBlocks(bpid, blockReport); blockReport = sfsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks * 2, blockReport.GetNumberOfBlocks()); foreach (Block b_1 in blockReport) { NUnit.Framework.Assert.IsNotNull(b_1); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), b_1.GetNumBytes() ); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b_1.GetBlockId()), sfsdataset.GetLength (new ExtendedBlock(bpid, b_1))); } NUnit.Framework.Assert.AreEqual(bytesAdded, sfsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(sfsdataset.GetCapacity() - bytesAdded, sfsdataset .GetRemaining()); // Now test that the dataset cannot be created if it does not have sufficient cap conf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, 10); try { sfsdataset = GetSimulatedFSDataset(); sfsdataset.AddBlockPool(bpid, conf); sfsdataset.InjectBlocks(bpid, blockReport); NUnit.Framework.Assert.IsTrue("Expected an IO exception", false); } catch (IOException) { } }
public virtual void TestFSDatasetFactory() { Configuration conf = new Configuration(); FsDatasetSpi.Factory <object> f = FsDatasetSpi.Factory.GetFactory(conf); NUnit.Framework.Assert.AreEqual(typeof(FsDatasetFactory), f.GetType()); NUnit.Framework.Assert.IsFalse(f.IsSimulated()); SimulatedFSDataset.SetFactory(conf); FsDatasetSpi.Factory <object> s = FsDatasetSpi.Factory.GetFactory(conf); NUnit.Framework.Assert.AreEqual(typeof(SimulatedFSDataset.Factory), s.GetType()); NUnit.Framework.Assert.IsTrue(s.IsSimulated()); }
public virtual void TestWriteRead() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); AddSomeBlocks(fsdataset); for (int i = 1; i <= Numblocks; ++i) { ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0); NUnit.Framework.Assert.IsTrue(fsdataset.IsValidBlock(b)); NUnit.Framework.Assert.AreEqual(BlockIdToLen(i), fsdataset.GetLength(b)); CheckBlockDataAndSize(fsdataset, b, BlockIdToLen(i)); } }
public virtual void TestStorageUsage() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); NUnit.Framework.Assert.AreEqual(fsdataset.GetDfsUsed(), 0); NUnit.Framework.Assert.AreEqual(fsdataset.GetRemaining(), fsdataset.GetCapacity() ); int bytesAdded = AddSomeBlocks(fsdataset); NUnit.Framework.Assert.AreEqual(bytesAdded, fsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(fsdataset.GetCapacity() - bytesAdded, fsdataset.GetRemaining ()); }
/// <exception cref="System.IO.IOException"/> internal virtual void CheckBlockDataAndSize(SimulatedFSDataset fsdataset, ExtendedBlock b, long expectedLen) { InputStream input = fsdataset.GetBlockInputStream(b); long lengthRead = 0; int data; while ((data = input.Read()) != -1) { NUnit.Framework.Assert.AreEqual(SimulatedFSDataset.DefaultDatabyte, data); lengthRead++; } NUnit.Framework.Assert.AreEqual(expectedLen, lengthRead); }
public virtual void TestGetBlockReport() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); BlockListAsLongs blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(0, blockReport.GetNumberOfBlocks()); AddSomeBlocks(fsdataset); blockReport = fsdataset.GetBlockReport(bpid); NUnit.Framework.Assert.AreEqual(Numblocks, blockReport.GetNumberOfBlocks()); foreach (Block b in blockReport) { NUnit.Framework.Assert.IsNotNull(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(b.GetBlockId()), b.GetNumBytes()); } }
/// <summary>Tests replication in DFS.</summary> /// <exception cref="System.IO.IOException"/> public virtual void RunReplication(bool simulated) { Configuration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeReplicationConsiderloadKey, false); if (simulated) { SimulatedFSDataset.SetFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes ).Racks(racks).Build(); cluster.WaitActive(); IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); DFSClient client = new DFSClient(addr, conf); DatanodeInfo[] info = client.DatanodeReport(HdfsConstants.DatanodeReportType.Live ); NUnit.Framework.Assert.AreEqual("Number of Datanodes ", numDatanodes, info.Length ); FileSystem fileSys = cluster.GetFileSystem(); try { Path file1 = new Path("/smallblocktest.dat"); WriteFile(fileSys, file1, 3); CheckFile(fileSys, file1, 3); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 10); CheckFile(fileSys, file1, 10); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 4); CheckFile(fileSys, file1, 4); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 1); CheckFile(fileSys, file1, 1); CleanupFile(fileSys, file1); WriteFile(fileSys, file1, 2); CheckFile(fileSys, file1, 2); CleanupFile(fileSys, file1); } finally { fileSys.Close(); cluster.Shutdown(); } }
public virtual void Setup() { conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); Configuration[] overlays = new Configuration[NumDatanodes]; for (int i = 0; i < overlays.Length; i++) { overlays[i] = new Configuration(); if (i == RoNodeIndex) { overlays[i].SetEnum(SimulatedFSDataset.ConfigPropertyState, i == RoNodeIndex ? DatanodeStorage.State .ReadOnlyShared : DatanodeStorage.State.Normal); } } cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).DataNodeConfOverlays (overlays).Build(); fs = cluster.GetFileSystem(); blockManager = cluster.GetNameNode().GetNamesystem().GetBlockManager(); datanodeManager = blockManager.GetDatanodeManager(); client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), cluster .GetConfiguration(0)); for (int i_1 = 0; i_1 < NumDatanodes; i_1++) { DataNode dataNode = cluster.GetDataNodes()[i_1]; ValidateStorageState(BlockManagerTestUtil.GetStorageReportsForDatanode(datanodeManager .GetDatanode(dataNode.GetDatanodeId())), i_1 == RoNodeIndex ? DatanodeStorage.State .ReadOnlyShared : DatanodeStorage.State.Normal); } // Create a 1 block file DFSTestUtil.CreateFile(fs, Path, BlockSize, BlockSize, BlockSize, (short)1, seed); LocatedBlock locatedBlock = GetLocatedBlock(); extendedBlock = locatedBlock.GetBlock(); block = extendedBlock.GetLocalBlock(); Assert.AssertThat(locatedBlock.GetLocations().Length, CoreMatchers.Is(1)); normalDataNode = locatedBlock.GetLocations()[0]; readOnlyDataNode = datanodeManager.GetDatanode(cluster.GetDataNodes()[RoNodeIndex ].GetDatanodeId()); Assert.AssertThat(normalDataNode, CoreMatchers.Is(CoreMatchers.Not(readOnlyDataNode ))); ValidateNumberReplicas(1); // Inject the block into the datanode with READ_ONLY_SHARED storage cluster.InjectBlocks(0, RoNodeIndex, Collections.Singleton(block)); // There should now be 2 *locations* for the block // Must wait until the NameNode has processed the block report for the injected blocks WaitForLocations(2); }
public virtual void TestDataNodeTimeSpend() { Configuration conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); try { FileSystem fs = cluster.GetFileSystem(); IList <DataNode> datanodes = cluster.GetDataNodes(); NUnit.Framework.Assert.AreEqual(datanodes.Count, 1); DataNode datanode = datanodes[0]; MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name()); long LongFileLen = 1024 * 1024 * 10; long startWriteValue = MetricsAsserts.GetLongCounter("TotalWriteTime", rb); long startReadValue = MetricsAsserts.GetLongCounter("TotalReadTime", rb); for (int x = 0; x < 50; x++) { DFSTestUtil.CreateFile(fs, new Path("/time.txt." + x), LongFileLen, (short)1, Time .MonotonicNow()); } for (int x_1 = 0; x_1 < 50; x_1++) { string s = DFSTestUtil.ReadFile(fs, new Path("/time.txt." + x_1)); } MetricsRecordBuilder rbNew = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name ()); long endWriteValue = MetricsAsserts.GetLongCounter("TotalWriteTime", rbNew); long endReadValue = MetricsAsserts.GetLongCounter("TotalReadTime", rbNew); NUnit.Framework.Assert.IsTrue(endReadValue > startReadValue); NUnit.Framework.Assert.IsTrue(endWriteValue > startWriteValue); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestInvalidate() { SimulatedFSDataset fsdataset = GetSimulatedFSDataset(); int bytesAdded = AddSomeBlocks(fsdataset); Block[] deleteBlocks = new Block[2]; deleteBlocks[0] = new Block(1, 0, 0); deleteBlocks[1] = new Block(2, 0, 0); fsdataset.Invalidate(bpid, deleteBlocks); CheckInvalidBlock(new ExtendedBlock(bpid, deleteBlocks[0])); CheckInvalidBlock(new ExtendedBlock(bpid, deleteBlocks[1])); long sizeDeleted = BlockIdToLen(1) + BlockIdToLen(2); NUnit.Framework.Assert.AreEqual(bytesAdded - sizeDeleted, fsdataset.GetDfsUsed()); NUnit.Framework.Assert.AreEqual(fsdataset.GetCapacity() - bytesAdded + sizeDeleted , fsdataset.GetRemaining()); // Now make sure the rest of the blocks are valid for (int i = 3; i <= Numblocks; ++i) { Block b = new Block(i, 0, 0); NUnit.Framework.Assert.IsTrue(fsdataset.IsValidBlock(new ExtendedBlock(bpid, b))); } }
/// <exception cref="System.IO.IOException"/> internal virtual int AddSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId ) { int bytesAdded = 0; for (int i = startingBlockId; i < startingBlockId + Numblocks; ++i) { ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual // data written ReplicaInPipelineInterface bInfo = fsdataset.CreateRbw(StorageType.Default, b, false ).GetReplica(); ReplicaOutputStreams @out = bInfo.CreateStreams(true, DataChecksum.NewDataChecksum (DataChecksum.Type.Crc32, 512)); try { OutputStream dataOut = @out.GetDataOut(); NUnit.Framework.Assert.AreEqual(0, fsdataset.GetLength(b)); for (int j = 1; j <= BlockIdToLen(i); ++j) { dataOut.Write(j); NUnit.Framework.Assert.AreEqual(j, bInfo.GetBytesOnDisk()); // correct length even as we write bytesAdded++; } } finally { @out.Close(); } b.SetNumBytes(BlockIdToLen(i)); fsdataset.FinalizeBlock(b); NUnit.Framework.Assert.AreEqual(BlockIdToLen(i), fsdataset.GetLength(b)); } return(bytesAdded); }
public virtual void TestFileLimit() { Configuration conf = new HdfsConfiguration(); int maxObjects = 5; conf.SetLong(DFSConfigKeys.DfsNamenodeMaxObjectsKey, maxObjects); conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000L); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); int currentNodes = 0; if (simulatedStorage) { SimulatedFSDataset.SetFactory(conf); } MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); FSNamesystem namesys = cluster.GetNamesystem(); try { // // check that / exists // Path path = new Path("/"); NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory ()); currentNodes = 1; // root inode // verify that we can create the specified number of files. We leave // one for the "/". Each file takes an inode and a block. // for (int i = 0; i < maxObjects / 2; i++) { Path file = new Path("/filestatus" + i); CreateFile(fs, file); System.Console.Out.WriteLine("Created file " + file); currentNodes += 2; } // two more objects for this creation. // verify that creating another file fails bool hitException = false; try { Path file = new Path("/filestatus"); CreateFile(fs, file); System.Console.Out.WriteLine("Created file " + file); } catch (IOException) { hitException = true; } NUnit.Framework.Assert.IsTrue("Was able to exceed file limit", hitException); // delete one file Path file0 = new Path("/filestatus0"); fs.Delete(file0, true); System.Console.Out.WriteLine("Deleted file " + file0); currentNodes -= 2; // wait for number of blocks to decrease WaitForLimit(namesys, currentNodes); // now, we shud be able to create a new file CreateFile(fs, file0); System.Console.Out.WriteLine("Created file " + file0 + " again."); currentNodes += 2; // delete the file again file0 = new Path("/filestatus0"); fs.Delete(file0, true); System.Console.Out.WriteLine("Deleted file " + file0 + " again."); currentNodes -= 2; // wait for number of blocks to decrease WaitForLimit(namesys, currentNodes); // create two directories in place of the file that we deleted Path dir = new Path("/dir0/dir1"); fs.Mkdirs(dir); System.Console.Out.WriteLine("Created directories " + dir); currentNodes += 2; WaitForLimit(namesys, currentNodes); // verify that creating another directory fails hitException = false; try { fs.Mkdirs(new Path("dir.fail")); System.Console.Out.WriteLine("Created directory should not have succeeded."); } catch (IOException) { hitException = true; } NUnit.Framework.Assert.IsTrue("Was able to exceed dir limit", hitException); } finally { fs.Close(); cluster.Shutdown(); } }
public virtual void SetUp() { conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); }
public virtual void TestInjection() { MiniDFSCluster cluster = null; string testFile = "/replication-test-file"; Path testPath = new Path(testFile); byte[] buffer = new byte[1024]; for (int i = 0; i < buffer.Length; i++) { buffer[i] = (byte)('1'); } try { Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes )); conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, checksumSize); SimulatedFSDataset.SetFactory(conf); //first time format cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build(); cluster.WaitActive(); string bpid = cluster.GetNamesystem().GetBlockPoolId(); DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort ()), conf); WriteFile(cluster.GetFileSystem(), testPath, numDataNodes); WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, 20); IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blocksList = cluster.GetAllBlockReports (bpid); cluster.Shutdown(); cluster = null; /* Start the MiniDFSCluster with more datanodes since once a writeBlock * to a datanode node fails, same block can not be written to it * immediately. In our case some replication attempts will fail. */ Log.Info("Restarting minicluster"); conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.0f"); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes * 2).Format( false).Build(); cluster.WaitActive(); ICollection <Block> uniqueBlocks = new HashSet <Block>(); foreach (IDictionary <DatanodeStorage, BlockListAsLongs> map in blocksList) { foreach (BlockListAsLongs blockList in map.Values) { foreach (Block b in blockList) { uniqueBlocks.AddItem(new Block(b)); } } } // Insert all the blocks in the first data node Log.Info("Inserting " + uniqueBlocks.Count + " blocks"); cluster.InjectBlocks(0, uniqueBlocks, null); dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), conf); WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { int numDataNodes = 0; int numRacks = 0; bool inject = false; long startingBlockId = 1; int numBlocksPerDNtoInject = 0; int replication = 1; bool checkDataNodeAddrConfig = false; long simulatedCapacityPerDn = SimulatedFSDataset.DefaultCapacity; string bpid = null; Configuration conf = new HdfsConfiguration(); for (int i = 0; i < args.Length; i++) { // parse command line if (args[i].Equals("-n")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("missing number of nodes"); } numDataNodes = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-racks")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing number of racks"); } numRacks = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-r")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing replication factor"); } replication = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-d")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing datanode dirs parameter"); } dataNodeDirs = args[i]; } else { if (args[i].Equals("-simulated")) { SimulatedFSDataset.SetFactory(conf); if ((i + 1) < args.Length && !args[i + 1].StartsWith("-")) { simulatedCapacityPerDn = long.Parse(args[++i]); } } else { if (args[i].Equals("-bpid")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing blockpoolid parameter"); } bpid = args[i]; } else { if (args[i].Equals("-inject")) { if (!FsDatasetSpi.Factory.GetFactory(conf).IsSimulated()) { System.Console.Out.Write("-inject is valid only for simulated"); PrintUsageExit(); } inject = true; if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing starting block and number of blocks per DN to inject"); } startingBlockId = System.Convert.ToInt32(args[i]); if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing number of blocks to inject"); } numBlocksPerDNtoInject = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-checkDataNodeAddrConfig")) { checkDataNodeAddrConfig = true; } else { PrintUsageExit(); } } } } } } } } } if (numDataNodes <= 0 || replication <= 0) { PrintUsageExit("numDataNodes and replication have to be greater than zero"); } if (replication > numDataNodes) { PrintUsageExit("Replication must be less than or equal to numDataNodes"); } if (bpid == null) { PrintUsageExit("BlockPoolId must be provided"); } string nameNodeAdr = FileSystem.GetDefaultUri(conf).GetAuthority(); if (nameNodeAdr == null) { System.Console.Out.WriteLine("No name node address and port in config"); System.Environment.Exit(-1); } bool simulated = FsDatasetSpi.Factory.GetFactory(conf).IsSimulated(); System.Console.Out.WriteLine("Starting " + numDataNodes + (simulated ? " Simulated " : " ") + " Data Nodes that will connect to Name Node at " + nameNodeAdr); Runtime.SetProperty("test.build.data", dataNodeDirs); long[] simulatedCapacities = new long[numDataNodes]; for (int i_1 = 0; i_1 < numDataNodes; ++i_1) { simulatedCapacities[i_1] = simulatedCapacityPerDn; } MiniDFSCluster mc = new MiniDFSCluster(); try { mc.FormatDataNodeDirs(); } catch (IOException e) { System.Console.Out.WriteLine("Error formating data node dirs:" + e); } string[] rack4DataNode = null; if (numRacks > 0) { System.Console.Out.WriteLine("Using " + numRacks + " racks: "); string rackPrefix = GetUniqueRackPrefix(); rack4DataNode = new string[numDataNodes]; for (int i_2 = 0; i_2 < numDataNodes; ++i_2) { //rack4DataNode[i] = racks[i%numRacks]; rack4DataNode[i_2] = rackPrefix + "-" + i_2 % numRacks; System.Console.Out.WriteLine("Data Node " + i_2 + " using " + rack4DataNode[i_2]); } } try { mc.StartDataNodes(conf, numDataNodes, true, HdfsServerConstants.StartupOption.Regular , rack4DataNode, null, simulatedCapacities, false, checkDataNodeAddrConfig); Sharpen.Thread.Sleep(10 * 1000); // Give the DN some time to connect to NN and init storage directories. if (inject) { long blockSize = 10; System.Console.Out.WriteLine("Injecting " + numBlocksPerDNtoInject + " blocks in each DN starting at blockId " + startingBlockId + " with blocksize of " + blockSize); Block[] blocks = new Block[numBlocksPerDNtoInject]; long blkid = startingBlockId; for (int i_dn = 0; i_dn < numDataNodes; ++i_dn) { for (int i_2 = 0; i_2 < blocks.Length; ++i_2) { blocks[i_2] = new Block(blkid++, blockSize, CreateEditsLog.BlockGenerationStamp); } for (int i_3 = 1; i_3 <= replication; ++i_3) { // inject blocks for dn_i into dn_i and replica in dn_i's neighbors mc.InjectBlocks((i_dn + i_3 - 1) % numDataNodes, Arrays.AsList(blocks), bpid); System.Console.Out.WriteLine("Injecting blocks of dn " + i_dn + " into dn" + ((i_dn + i_3 - 1) % numDataNodes)); } } System.Console.Out.WriteLine("Created blocks from Bids " + startingBlockId + " to " + (blkid - 1)); } } catch (IOException e) { System.Console.Out.WriteLine("Error creating data node:" + e); } }
/// <exception cref="System.IO.IOException"/> public virtual void StartDataNodes(Configuration conf, int numDataNodes, StorageType [][] storageTypes, bool manageDfsDirs, HdfsServerConstants.StartupOption operation , string[] racks, string[] nodeGroups, string[] hosts, long[][] storageCapacities , long[] simulatedCapacities, bool setupHostsFile, bool checkDataNodeAddrConfig, bool checkDataNodeHostConfig) { lock (this) { System.Diagnostics.Debug.Assert(storageCapacities == null || simulatedCapacities == null); System.Diagnostics.Debug.Assert(storageTypes == null || storageTypes.Length == numDataNodes ); System.Diagnostics.Debug.Assert(storageCapacities == null || storageCapacities.Length == numDataNodes); if (operation == HdfsServerConstants.StartupOption.Recover) { return; } if (checkDataNodeHostConfig) { conf.SetIfUnset(DFSConfigKeys.DfsDatanodeHostNameKey, "127.0.0.1"); } else { conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "127.0.0.1"); } conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "127.0.0.1"); int curDatanodesNum = dataNodes.Count; // for mincluster's the default initialDelay for BRs is 0 if (conf.Get(DFSConfigKeys.DfsBlockreportInitialDelayKey) == null) { conf.SetLong(DFSConfigKeys.DfsBlockreportInitialDelayKey, 0); } // If minicluster's name node is null assume that the conf has been // set with the right address:port of the name node. // if (racks != null && numDataNodes > racks.Length) { throw new ArgumentException("The length of racks [" + racks.Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } if (nodeGroups != null && numDataNodes > nodeGroups.Length) { throw new ArgumentException("The length of nodeGroups [" + nodeGroups.Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } if (hosts != null && numDataNodes > hosts.Length) { throw new ArgumentException("The length of hosts [" + hosts.Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } //Generate some hostnames if required if (racks != null && hosts == null) { hosts = new string[numDataNodes]; for (int i = curDatanodesNum; i < curDatanodesNum + numDataNodes; i++) { hosts[i - curDatanodesNum] = "host" + i + ".foo.com"; } } if (simulatedCapacities != null && numDataNodes > simulatedCapacities.Length) { throw new ArgumentException("The length of simulatedCapacities [" + simulatedCapacities .Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } string[] dnArgs = (operation == null || operation != HdfsServerConstants.StartupOption .Rollback) ? null : new string[] { operation.GetName() }; DataNode[] dns = new DataNode[numDataNodes]; for (int i_1 = curDatanodesNum; i_1 < curDatanodesNum + numDataNodes; i_1++) { Configuration dnConf = new HdfsConfiguration(conf); // Set up datanode address SetupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { string dirs = MakeDataNodeDirs(i_1, storageTypes == null ? null : storageTypes[i_1 ]); dnConf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dirs); conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dirs); } if (simulatedCapacities != null) { SimulatedFSDataset.SetFactory(dnConf); dnConf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, simulatedCapacities[i_1 - curDatanodesNum]); } Log.Info("Starting DataNode " + i_1 + " with " + DFSConfigKeys.DfsDatanodeDataDirKey + ": " + dnConf.Get(DFSConfigKeys.DfsDatanodeDataDirKey)); if (hosts != null) { dnConf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, hosts[i_1 - curDatanodesNum]); Log.Info("Starting DataNode " + i_1 + " with hostname set to: " + dnConf.Get(DFSConfigKeys .DfsDatanodeHostNameKey)); } if (racks != null) { string name = hosts[i_1 - curDatanodesNum]; if (nodeGroups == null) { Log.Info("Adding node with hostname : " + name + " to rack " + racks[i_1 - curDatanodesNum ]); StaticMapping.AddNodeToRack(name, racks[i_1 - curDatanodesNum]); } else { Log.Info("Adding node with hostname : " + name + " to serverGroup " + nodeGroups[ i_1 - curDatanodesNum] + " and rack " + racks[i_1 - curDatanodesNum]); StaticMapping.AddNodeToRack(name, racks[i_1 - curDatanodesNum] + nodeGroups[i_1 - curDatanodesNum]); } } Configuration newconf = new HdfsConfiguration(dnConf); // save config if (hosts != null) { NetUtils.AddStaticResolution(hosts[i_1 - curDatanodesNum], "localhost"); } SecureDataNodeStarter.SecureResources secureResources = null; if (UserGroupInformation.IsSecurityEnabled()) { try { secureResources = SecureDataNodeStarter.GetSecureResources(dnConf); } catch (Exception ex) { Sharpen.Runtime.PrintStackTrace(ex); } } DataNode dn = DataNode.InstantiateDataNode(dnArgs, dnConf, secureResources); if (dn == null) { throw new IOException("Cannot start DataNode in " + dnConf.Get(DFSConfigKeys.DfsDatanodeDataDirKey )); } //since the HDFS does things based on IP:port, we need to add the mapping //for IP:port to rackId string ipAddr = dn.GetXferAddress().Address.GetHostAddress(); if (racks != null) { int port = dn.GetXferAddress().Port; if (nodeGroups == null) { Log.Info("Adding node with IP:port : " + ipAddr + ":" + port + " to rack " + racks [i_1 - curDatanodesNum]); StaticMapping.AddNodeToRack(ipAddr + ":" + port, racks[i_1 - curDatanodesNum]); } else { Log.Info("Adding node with IP:port : " + ipAddr + ":" + port + " to nodeGroup " + nodeGroups[i_1 - curDatanodesNum] + " and rack " + racks[i_1 - curDatanodesNum] ); StaticMapping.AddNodeToRack(ipAddr + ":" + port, racks[i_1 - curDatanodesNum] + nodeGroups [i_1 - curDatanodesNum]); } } dn.RunDatanodeDaemon(); dataNodes.AddItem(new MiniDFSCluster.DataNodeProperties(this, dn, newconf, dnArgs , secureResources, dn.GetIpcPort())); dns[i_1 - curDatanodesNum] = dn; } curDatanodesNum += numDataNodes; this.numDataNodes += numDataNodes; WaitActive(); if (storageCapacities != null) { for (int i = curDatanodesNum; i_1 < curDatanodesNum + numDataNodes; ++i_1) { IList <FsVolumeSpi> volumes = dns[i_1].GetFSDataset().GetVolumes(); System.Diagnostics.Debug.Assert(volumes.Count == storagesPerDatanode); for (int j = 0; j < volumes.Count; ++j) { FsVolumeImpl volume = (FsVolumeImpl)volumes[j]; volume.SetCapacityForTesting(storageCapacities[i_1][j]); } } } } }
/// <exception cref="System.IO.IOException"/> internal virtual int AddSomeBlocks(SimulatedFSDataset fsdataset) { return(AddSomeBlocks(fsdataset, 1)); }