public static void Setup() { Configuration conf = new Configuration(); Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp")); testdir = new Path(TestRootDir, "TestMiniMRClientCluster"); inDir = new Path(testdir, "in"); outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.GetLocal(conf); if (fs.Exists(testdir) && !fs.Delete(testdir, true)) { throw new IOException("Could not delete " + testdir); } if (!fs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir); } for (int i = 0; i < inFiles.Length; i++) { inFiles[i] = new Path(inDir, "part_" + i); CreateFile(inFiles[i], conf); } // create the mini cluster to be used for the tests mrCluster = MiniMRClientClusterFactory.Create(typeof(TestMiniMRClientCluster.InternalClass ), 1, new Configuration()); }
// Number of input files is same as the number of mappers. // Number of reducers. // Number of lines per input file. // Where MR job's input will reside. // Where output goes. /// <exception cref="System.Exception"/> public virtual void TestMerge() { MiniDFSCluster dfsCluster = null; MiniMRClientCluster mrCluster = null; FileSystem fileSystem = null; try { Configuration conf = new Configuration(); // Start the mini-MR and mini-DFS clusters dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumHadoopDataNodes).Build (); fileSystem = dfsCluster.GetFileSystem(); mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), NumHadoopDataNodes, conf); // Generate input. CreateInput(fileSystem); // Run the test. RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem); } finally { if (dfsCluster != null) { dfsCluster.Shutdown(); } if (mrCluster != null) { mrCluster.Stop(); } } }
/// <exception cref="System.Exception"/> public virtual void DoEncryptionTest(int numMappers, int numReducers, int numNodes , int numLines, bool isUber) { MiniDFSCluster dfsCluster = null; MiniMRClientCluster mrCluster = null; FileSystem fileSystem = null; try { Configuration conf = new Configuration(); // Start the mini-MR and mini-DFS clusters dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numNodes).Build(); fileSystem = dfsCluster.GetFileSystem(); mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), numNodes, conf); // Generate input. CreateInput(fileSystem, numMappers, numLines); // Run the test. RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem, numMappers, numReducers , numLines, isUber); } finally { if (dfsCluster != null) { dfsCluster.Shutdown(); } if (mrCluster != null) { mrCluster.Stop(); } } }
/// <exception cref="System.IO.IOException"/> private MiniMRClientCluster CreateMiniClusterWithCapacityScheduler() { Configuration conf = new Configuration(); // Expected queue names depending on Capacity Scheduler queue naming conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(CapacityScheduler )); return(MiniMRClientClusterFactory.Create(this.GetType(), 2, conf)); }
/// <exception cref="System.IO.IOException"/> public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, int numTaskTrackers , string namenode, int numDir, string[] racks, string[] hosts, UserGroupInformation ugi, JobConf conf, int numTrackerToExclude, Clock clock) { if (conf == null) { conf = new JobConf(); } FileSystem.SetDefaultUri(conf, namenode); string identifier = this.GetType().Name + "_" + Sharpen.Extensions.ToString(new Random ().Next(int.MaxValue)); mrClientCluster = MiniMRClientClusterFactory.Create(this.GetType(), identifier, numTaskTrackers , conf); }