public virtual void Setup()
{
conf = new HdfsConfiguration();
SimulatedFSDataset.SetFactory(conf);
Configuration[] overlays = new Configuration[NumDatanodes];
for (int i = 0; i < overlays.Length; i++)
{
overlays[i] = new Configuration();
if (i == RoNodeIndex)
{
overlays[i].SetEnum(SimulatedFSDataset.ConfigPropertyState, i == RoNodeIndex ? DatanodeStorage.State
.ReadOnlyShared : DatanodeStorage.State.Normal);
}
}
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).DataNodeConfOverlays
(overlays).Build();
fs = cluster.GetFileSystem();
blockManager = cluster.GetNameNode().GetNamesystem().GetBlockManager();
datanodeManager = blockManager.GetDatanodeManager();
client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), cluster
.GetConfiguration(0));
for (int i_1 = 0; i_1 < NumDatanodes; i_1++)
{
DataNode dataNode = cluster.GetDataNodes()[i_1];
ValidateStorageState(BlockManagerTestUtil.GetStorageReportsForDatanode(datanodeManager
.GetDatanode(dataNode.GetDatanodeId())), i_1 == RoNodeIndex ? DatanodeStorage.State
.ReadOnlyShared : DatanodeStorage.State.Normal);
}
// Create a 1 block file
DFSTestUtil.CreateFile(fs, Path, BlockSize, BlockSize, BlockSize, (short)1, seed);
LocatedBlock locatedBlock = GetLocatedBlock();
extendedBlock = locatedBlock.GetBlock();
block = extendedBlock.GetLocalBlock();
Assert.AssertThat(locatedBlock.GetLocations().Length, CoreMatchers.Is(1));
normalDataNode = locatedBlock.GetLocations()[0];
readOnlyDataNode = datanodeManager.GetDatanode(cluster.GetDataNodes()[RoNodeIndex
].GetDatanodeId());
Assert.AssertThat(normalDataNode, CoreMatchers.Is(CoreMatchers.Not(readOnlyDataNode
)));
ValidateNumberReplicas(1);
// Inject the block into the datanode with READ_ONLY_SHARED storage
cluster.InjectBlocks(0, RoNodeIndex, Collections.Singleton(block));
// There should now be 2 *locations* for the block
// Must wait until the NameNode has processed the block report for the injected blocks
WaitForLocations(2);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
int numDataNodes = 0;
int numRacks = 0;
bool inject = false;
long startingBlockId = 1;
int numBlocksPerDNtoInject = 0;
int replication = 1;
bool checkDataNodeAddrConfig = false;
long simulatedCapacityPerDn = SimulatedFSDataset.DefaultCapacity;
string bpid = null;
Configuration conf = new HdfsConfiguration();
for (int i = 0; i < args.Length; i++)
{
// parse command line
if (args[i].Equals("-n"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("missing number of nodes");
}
numDataNodes = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-racks"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing number of racks");
}
numRacks = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-r"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing replication factor");
}
replication = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-d"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing datanode dirs parameter");
}
dataNodeDirs = args[i];
}
else
{
if (args[i].Equals("-simulated"))
{
SimulatedFSDataset.SetFactory(conf);
if ((i + 1) < args.Length && !args[i + 1].StartsWith("-"))
{
simulatedCapacityPerDn = long.Parse(args[++i]);
}
}
else
{
if (args[i].Equals("-bpid"))
{
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing blockpoolid parameter");
}
bpid = args[i];
}
else
{
if (args[i].Equals("-inject"))
{
if (!FsDatasetSpi.Factory.GetFactory(conf).IsSimulated())
{
System.Console.Out.Write("-inject is valid only for simulated");
PrintUsageExit();
}
inject = true;
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing starting block and number of blocks per DN to inject");
}
startingBlockId = System.Convert.ToInt32(args[i]);
if (++i >= args.Length || args[i].StartsWith("-"))
{
PrintUsageExit("Missing number of blocks to inject");
}
numBlocksPerDNtoInject = System.Convert.ToInt32(args[i]);
}
else
{
if (args[i].Equals("-checkDataNodeAddrConfig"))
{
checkDataNodeAddrConfig = true;
}
else
{
PrintUsageExit();
}
}
}
}
}
}
}
}
}
if (numDataNodes <= 0 || replication <= 0)
{
PrintUsageExit("numDataNodes and replication have to be greater than zero");
}
if (replication > numDataNodes)
{
PrintUsageExit("Replication must be less than or equal to numDataNodes");
}
if (bpid == null)
{
PrintUsageExit("BlockPoolId must be provided");
}
string nameNodeAdr = FileSystem.GetDefaultUri(conf).GetAuthority();
if (nameNodeAdr == null)
{
System.Console.Out.WriteLine("No name node address and port in config");
System.Environment.Exit(-1);
}
bool simulated = FsDatasetSpi.Factory.GetFactory(conf).IsSimulated();
System.Console.Out.WriteLine("Starting " + numDataNodes + (simulated ? " Simulated "
: " ") + " Data Nodes that will connect to Name Node at " + nameNodeAdr);
Runtime.SetProperty("test.build.data", dataNodeDirs);
long[] simulatedCapacities = new long[numDataNodes];
for (int i_1 = 0; i_1 < numDataNodes; ++i_1)
{
simulatedCapacities[i_1] = simulatedCapacityPerDn;
}
MiniDFSCluster mc = new MiniDFSCluster();
try
{
mc.FormatDataNodeDirs();
}
catch (IOException e)
{
System.Console.Out.WriteLine("Error formating data node dirs:" + e);
}
string[] rack4DataNode = null;
if (numRacks > 0)
{
System.Console.Out.WriteLine("Using " + numRacks + " racks: ");
string rackPrefix = GetUniqueRackPrefix();
rack4DataNode = new string[numDataNodes];
for (int i_2 = 0; i_2 < numDataNodes; ++i_2)
{
//rack4DataNode[i] = racks[i%numRacks];
rack4DataNode[i_2] = rackPrefix + "-" + i_2 % numRacks;
System.Console.Out.WriteLine("Data Node " + i_2 + " using " + rack4DataNode[i_2]);
}
}
try
{
mc.StartDataNodes(conf, numDataNodes, true, HdfsServerConstants.StartupOption.Regular
, rack4DataNode, null, simulatedCapacities, false, checkDataNodeAddrConfig);
Sharpen.Thread.Sleep(10 * 1000);
// Give the DN some time to connect to NN and init storage directories.
if (inject)
{
long blockSize = 10;
System.Console.Out.WriteLine("Injecting " + numBlocksPerDNtoInject + " blocks in each DN starting at blockId "
+ startingBlockId + " with blocksize of " + blockSize);
Block[] blocks = new Block[numBlocksPerDNtoInject];
long blkid = startingBlockId;
for (int i_dn = 0; i_dn < numDataNodes; ++i_dn)
{
for (int i_2 = 0; i_2 < blocks.Length; ++i_2)
{
blocks[i_2] = new Block(blkid++, blockSize, CreateEditsLog.BlockGenerationStamp);
}
for (int i_3 = 1; i_3 <= replication; ++i_3)
{
// inject blocks for dn_i into dn_i and replica in dn_i's neighbors
mc.InjectBlocks((i_dn + i_3 - 1) % numDataNodes, Arrays.AsList(blocks), bpid);
System.Console.Out.WriteLine("Injecting blocks of dn " + i_dn + " into dn" + ((i_dn
+ i_3 - 1) % numDataNodes));
}
}
System.Console.Out.WriteLine("Created blocks from Bids " + startingBlockId + " to "
+ (blkid - 1));
}
}
catch (IOException e)
{
System.Console.Out.WriteLine("Error creating data node:" + e);
}
}