public virtual void Setup() { conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); Configuration[] overlays = new Configuration[NumDatanodes]; for (int i = 0; i < overlays.Length; i++) { overlays[i] = new Configuration(); if (i == RoNodeIndex) { overlays[i].SetEnum(SimulatedFSDataset.ConfigPropertyState, i == RoNodeIndex ? DatanodeStorage.State .ReadOnlyShared : DatanodeStorage.State.Normal); } } cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).DataNodeConfOverlays (overlays).Build(); fs = cluster.GetFileSystem(); blockManager = cluster.GetNameNode().GetNamesystem().GetBlockManager(); datanodeManager = blockManager.GetDatanodeManager(); client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), cluster .GetConfiguration(0)); for (int i_1 = 0; i_1 < NumDatanodes; i_1++) { DataNode dataNode = cluster.GetDataNodes()[i_1]; ValidateStorageState(BlockManagerTestUtil.GetStorageReportsForDatanode(datanodeManager .GetDatanode(dataNode.GetDatanodeId())), i_1 == RoNodeIndex ? DatanodeStorage.State .ReadOnlyShared : DatanodeStorage.State.Normal); } // Create a 1 block file DFSTestUtil.CreateFile(fs, Path, BlockSize, BlockSize, BlockSize, (short)1, seed); LocatedBlock locatedBlock = GetLocatedBlock(); extendedBlock = locatedBlock.GetBlock(); block = extendedBlock.GetLocalBlock(); Assert.AssertThat(locatedBlock.GetLocations().Length, CoreMatchers.Is(1)); normalDataNode = locatedBlock.GetLocations()[0]; readOnlyDataNode = datanodeManager.GetDatanode(cluster.GetDataNodes()[RoNodeIndex ].GetDatanodeId()); Assert.AssertThat(normalDataNode, CoreMatchers.Is(CoreMatchers.Not(readOnlyDataNode ))); ValidateNumberReplicas(1); // Inject the block into the datanode with READ_ONLY_SHARED storage cluster.InjectBlocks(0, RoNodeIndex, Collections.Singleton(block)); // There should now be 2 *locations* for the block // Must wait until the NameNode has processed the block report for the injected blocks WaitForLocations(2); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { int numDataNodes = 0; int numRacks = 0; bool inject = false; long startingBlockId = 1; int numBlocksPerDNtoInject = 0; int replication = 1; bool checkDataNodeAddrConfig = false; long simulatedCapacityPerDn = SimulatedFSDataset.DefaultCapacity; string bpid = null; Configuration conf = new HdfsConfiguration(); for (int i = 0; i < args.Length; i++) { // parse command line if (args[i].Equals("-n")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("missing number of nodes"); } numDataNodes = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-racks")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing number of racks"); } numRacks = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-r")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing replication factor"); } replication = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-d")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing datanode dirs parameter"); } dataNodeDirs = args[i]; } else { if (args[i].Equals("-simulated")) { SimulatedFSDataset.SetFactory(conf); if ((i + 1) < args.Length && !args[i + 1].StartsWith("-")) { simulatedCapacityPerDn = long.Parse(args[++i]); } } else { if (args[i].Equals("-bpid")) { if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing blockpoolid parameter"); } bpid = args[i]; } else { if (args[i].Equals("-inject")) { if (!FsDatasetSpi.Factory.GetFactory(conf).IsSimulated()) { System.Console.Out.Write("-inject is valid only for simulated"); PrintUsageExit(); } inject = true; if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing starting block and number of blocks per DN to inject"); } startingBlockId = System.Convert.ToInt32(args[i]); if (++i >= args.Length || args[i].StartsWith("-")) { PrintUsageExit("Missing number of blocks to inject"); } numBlocksPerDNtoInject = System.Convert.ToInt32(args[i]); } else { if (args[i].Equals("-checkDataNodeAddrConfig")) { checkDataNodeAddrConfig = true; } else { PrintUsageExit(); } } } } } } } } } if (numDataNodes <= 0 || replication <= 0) { PrintUsageExit("numDataNodes and replication have to be greater than zero"); } if (replication > numDataNodes) { PrintUsageExit("Replication must be less than or equal to numDataNodes"); } if (bpid == null) { PrintUsageExit("BlockPoolId must be provided"); } string nameNodeAdr = FileSystem.GetDefaultUri(conf).GetAuthority(); if (nameNodeAdr == null) { System.Console.Out.WriteLine("No name node address and port in config"); System.Environment.Exit(-1); } bool simulated = FsDatasetSpi.Factory.GetFactory(conf).IsSimulated(); System.Console.Out.WriteLine("Starting " + numDataNodes + (simulated ? " Simulated " : " ") + " Data Nodes that will connect to Name Node at " + nameNodeAdr); Runtime.SetProperty("test.build.data", dataNodeDirs); long[] simulatedCapacities = new long[numDataNodes]; for (int i_1 = 0; i_1 < numDataNodes; ++i_1) { simulatedCapacities[i_1] = simulatedCapacityPerDn; } MiniDFSCluster mc = new MiniDFSCluster(); try { mc.FormatDataNodeDirs(); } catch (IOException e) { System.Console.Out.WriteLine("Error formating data node dirs:" + e); } string[] rack4DataNode = null; if (numRacks > 0) { System.Console.Out.WriteLine("Using " + numRacks + " racks: "); string rackPrefix = GetUniqueRackPrefix(); rack4DataNode = new string[numDataNodes]; for (int i_2 = 0; i_2 < numDataNodes; ++i_2) { //rack4DataNode[i] = racks[i%numRacks]; rack4DataNode[i_2] = rackPrefix + "-" + i_2 % numRacks; System.Console.Out.WriteLine("Data Node " + i_2 + " using " + rack4DataNode[i_2]); } } try { mc.StartDataNodes(conf, numDataNodes, true, HdfsServerConstants.StartupOption.Regular , rack4DataNode, null, simulatedCapacities, false, checkDataNodeAddrConfig); Sharpen.Thread.Sleep(10 * 1000); // Give the DN some time to connect to NN and init storage directories. if (inject) { long blockSize = 10; System.Console.Out.WriteLine("Injecting " + numBlocksPerDNtoInject + " blocks in each DN starting at blockId " + startingBlockId + " with blocksize of " + blockSize); Block[] blocks = new Block[numBlocksPerDNtoInject]; long blkid = startingBlockId; for (int i_dn = 0; i_dn < numDataNodes; ++i_dn) { for (int i_2 = 0; i_2 < blocks.Length; ++i_2) { blocks[i_2] = new Block(blkid++, blockSize, CreateEditsLog.BlockGenerationStamp); } for (int i_3 = 1; i_3 <= replication; ++i_3) { // inject blocks for dn_i into dn_i and replica in dn_i's neighbors mc.InjectBlocks((i_dn + i_3 - 1) % numDataNodes, Arrays.AsList(blocks), bpid); System.Console.Out.WriteLine("Injecting blocks of dn " + i_dn + " into dn" + ((i_dn + i_3 - 1) % numDataNodes)); } } System.Console.Out.WriteLine("Created blocks from Bids " + startingBlockId + " to " + (blkid - 1)); } } catch (IOException e) { System.Console.Out.WriteLine("Error creating data node:" + e); } }