public virtual void TestLocalRackPlacement() { string clientMachine = "client.foo.com"; // Map client to RACK2 string clientRack = "/RACK2"; StaticMapping.AddNodeToRack(clientMachine, clientRack); TestPlacement(clientMachine, clientRack); }
public virtual void Setup() { StaticMapping.ResetMap(); Configuration conf = new HdfsConfiguration(); string[] racks = new string[] { "/RACK0", "/RACK0", "/RACK2", "/RACK3", "/RACK2" }; string[] hosts = new string[] { "/host0", "/host1", "/host2", "/host3", "/host4" }; conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize); conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, DefaultBlockSize / 2); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Racks(racks).Hosts(hosts ).Build(); cluster.WaitActive(); nameNodeRpc = cluster.GetNameNodeRpc(); namesystem = cluster.GetNamesystem(); perm = new PermissionStatus("TestDefaultBlockPlacementPolicy", null, FsPermission .GetDefault()); }
public void RedrawStaticTargets(StaticTextureSource source, string variant, Dictionary <TextureCategory, bool> options) { if (source.Categories != null) { // Exclude it if any of its categories are in the options and switched off foreach (TextureCategory category in source.Categories) { if (options.ContainsKey(category) && !options[category]) { return; } } } // For sprite sequence sources, the targets are mapped dynamically. if (source.IsSpriteSequence && (!StaticMapping.ContainsKey(source) || StaticMapping[source].Count == 0)) { GenerateSpriteSequenceTargets(source); } // This can happen if we have a source grouped for this level, // but the source is actually only in place on certain conditions // - an example is the flame in Venice, which is only added if // the Flamethrower has been imported. if (!StaticMapping.ContainsKey(source)) { return; } List <Rectangle> segments = source.VariantMap[variant]; foreach (StaticTextureTarget target in StaticMapping[source]) { if (target.Segment < 0 || target.Segment >= segments.Count) { throw new IndexOutOfRangeException(string.Format("Segment {0} is invalid for texture source {1}.", target.Segment, source.PNGPath)); } GetBitmapGraphics(target.Tile).ImportSegment(source, target, segments[target.Segment]); } if (source.EntityColourMap != null) { foreach (TR2Entities entity in source.EntityColourMap.Keys) { TRMesh[] meshes = TR2LevelUtilities.GetModelMeshes(_level, entity); ISet <int> colourIndices = new HashSet <int>(); foreach (TRMesh mesh in meshes) { foreach (TRFace4 t in mesh.ColouredRectangles) { colourIndices.Add(BitConverter.GetBytes(t.Texture)[1]); } foreach (TRFace3 t in mesh.ColouredTriangles) { colourIndices.Add(BitConverter.GetBytes(t.Texture)[1]); } } Dictionary <int, int> remapIndices = new Dictionary <int, int>(); foreach (Color targetColour in source.EntityColourMap[entity].Keys) { int matchedIndex = -1; foreach (int currentIndex in colourIndices) { TRColour4 currentColour = _level.Palette16[currentIndex]; if (currentColour.Red == targetColour.R && currentColour.Green == targetColour.G && currentColour.Blue == targetColour.B) { matchedIndex = currentIndex; } } if (matchedIndex == -1) { continue; } // Extract the colour from the top-left of the rectangle specified in the source, and import that into the level int sourceRectangle = source.EntityColourMap[entity][targetColour]; int newColourIndex = P16Importer.Import(_level, source.Bitmap.GetPixel(segments[sourceRectangle].X, segments[sourceRectangle].Y)); remapIndices.Add(matchedIndex, newColourIndex); } // Remap the affected mesh textures to the newly inserted colours foreach (TRMesh mesh in meshes) { foreach (TRFace4 t in mesh.ColouredRectangles) { t.Texture = ConvertMeshTexture(t.Texture, remapIndices); } foreach (TRFace3 t in mesh.ColouredTriangles) { t.Texture = ConvertMeshTexture(t.Texture, remapIndices); } } } // Reset the palette tracking P16Importer.ResetPaletteTracking(_level); } }
/// <exception cref="System.IO.IOException"/> public virtual void StartDataNodes(Configuration conf, int numDataNodes, StorageType [][] storageTypes, bool manageDfsDirs, HdfsServerConstants.StartupOption operation , string[] racks, string[] nodeGroups, string[] hosts, long[][] storageCapacities , long[] simulatedCapacities, bool setupHostsFile, bool checkDataNodeAddrConfig, bool checkDataNodeHostConfig) { lock (this) { System.Diagnostics.Debug.Assert(storageCapacities == null || simulatedCapacities == null); System.Diagnostics.Debug.Assert(storageTypes == null || storageTypes.Length == numDataNodes ); System.Diagnostics.Debug.Assert(storageCapacities == null || storageCapacities.Length == numDataNodes); if (operation == HdfsServerConstants.StartupOption.Recover) { return; } if (checkDataNodeHostConfig) { conf.SetIfUnset(DFSConfigKeys.DfsDatanodeHostNameKey, "127.0.0.1"); } else { conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "127.0.0.1"); } conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "127.0.0.1"); int curDatanodesNum = dataNodes.Count; // for mincluster's the default initialDelay for BRs is 0 if (conf.Get(DFSConfigKeys.DfsBlockreportInitialDelayKey) == null) { conf.SetLong(DFSConfigKeys.DfsBlockreportInitialDelayKey, 0); } // If minicluster's name node is null assume that the conf has been // set with the right address:port of the name node. // if (racks != null && numDataNodes > racks.Length) { throw new ArgumentException("The length of racks [" + racks.Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } if (nodeGroups != null && numDataNodes > nodeGroups.Length) { throw new ArgumentException("The length of nodeGroups [" + nodeGroups.Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } if (hosts != null && numDataNodes > hosts.Length) { throw new ArgumentException("The length of hosts [" + hosts.Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } //Generate some hostnames if required if (racks != null && hosts == null) { hosts = new string[numDataNodes]; for (int i = curDatanodesNum; i < curDatanodesNum + numDataNodes; i++) { hosts[i - curDatanodesNum] = "host" + i + ".foo.com"; } } if (simulatedCapacities != null && numDataNodes > simulatedCapacities.Length) { throw new ArgumentException("The length of simulatedCapacities [" + simulatedCapacities .Length + "] is less than the number of datanodes [" + numDataNodes + "]."); } string[] dnArgs = (operation == null || operation != HdfsServerConstants.StartupOption .Rollback) ? null : new string[] { operation.GetName() }; DataNode[] dns = new DataNode[numDataNodes]; for (int i_1 = curDatanodesNum; i_1 < curDatanodesNum + numDataNodes; i_1++) { Configuration dnConf = new HdfsConfiguration(conf); // Set up datanode address SetupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { string dirs = MakeDataNodeDirs(i_1, storageTypes == null ? null : storageTypes[i_1 ]); dnConf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dirs); conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dirs); } if (simulatedCapacities != null) { SimulatedFSDataset.SetFactory(dnConf); dnConf.SetLong(SimulatedFSDataset.ConfigPropertyCapacity, simulatedCapacities[i_1 - curDatanodesNum]); } Log.Info("Starting DataNode " + i_1 + " with " + DFSConfigKeys.DfsDatanodeDataDirKey + ": " + dnConf.Get(DFSConfigKeys.DfsDatanodeDataDirKey)); if (hosts != null) { dnConf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, hosts[i_1 - curDatanodesNum]); Log.Info("Starting DataNode " + i_1 + " with hostname set to: " + dnConf.Get(DFSConfigKeys .DfsDatanodeHostNameKey)); } if (racks != null) { string name = hosts[i_1 - curDatanodesNum]; if (nodeGroups == null) { Log.Info("Adding node with hostname : " + name + " to rack " + racks[i_1 - curDatanodesNum ]); StaticMapping.AddNodeToRack(name, racks[i_1 - curDatanodesNum]); } else { Log.Info("Adding node with hostname : " + name + " to serverGroup " + nodeGroups[ i_1 - curDatanodesNum] + " and rack " + racks[i_1 - curDatanodesNum]); StaticMapping.AddNodeToRack(name, racks[i_1 - curDatanodesNum] + nodeGroups[i_1 - curDatanodesNum]); } } Configuration newconf = new HdfsConfiguration(dnConf); // save config if (hosts != null) { NetUtils.AddStaticResolution(hosts[i_1 - curDatanodesNum], "localhost"); } SecureDataNodeStarter.SecureResources secureResources = null; if (UserGroupInformation.IsSecurityEnabled()) { try { secureResources = SecureDataNodeStarter.GetSecureResources(dnConf); } catch (Exception ex) { Sharpen.Runtime.PrintStackTrace(ex); } } DataNode dn = DataNode.InstantiateDataNode(dnArgs, dnConf, secureResources); if (dn == null) { throw new IOException("Cannot start DataNode in " + dnConf.Get(DFSConfigKeys.DfsDatanodeDataDirKey )); } //since the HDFS does things based on IP:port, we need to add the mapping //for IP:port to rackId string ipAddr = dn.GetXferAddress().Address.GetHostAddress(); if (racks != null) { int port = dn.GetXferAddress().Port; if (nodeGroups == null) { Log.Info("Adding node with IP:port : " + ipAddr + ":" + port + " to rack " + racks [i_1 - curDatanodesNum]); StaticMapping.AddNodeToRack(ipAddr + ":" + port, racks[i_1 - curDatanodesNum]); } else { Log.Info("Adding node with IP:port : " + ipAddr + ":" + port + " to nodeGroup " + nodeGroups[i_1 - curDatanodesNum] + " and rack " + racks[i_1 - curDatanodesNum] ); StaticMapping.AddNodeToRack(ipAddr + ":" + port, racks[i_1 - curDatanodesNum] + nodeGroups [i_1 - curDatanodesNum]); } } dn.RunDatanodeDaemon(); dataNodes.AddItem(new MiniDFSCluster.DataNodeProperties(this, dn, newconf, dnArgs , secureResources, dn.GetIpcPort())); dns[i_1 - curDatanodesNum] = dn; } curDatanodesNum += numDataNodes; this.numDataNodes += numDataNodes; WaitActive(); if (storageCapacities != null) { for (int i = curDatanodesNum; i_1 < curDatanodesNum + numDataNodes; ++i_1) { IList <FsVolumeSpi> volumes = dns[i_1].GetFSDataset().GetVolumes(); System.Diagnostics.Debug.Assert(volumes.Count == storagesPerDatanode); for (int j = 0; j < volumes.Count; ++j) { FsVolumeImpl volume = (FsVolumeImpl)volumes[j]; volume.SetCapacityForTesting(storageCapacities[i_1][j]); } } } } }