Пример #1
0
 /// <exception cref="System.Exception"/>
 protected override void TearDown()
 {
     if (mrCluster != null)
     {
         mrCluster.Shutdown();
     }
     if (dfsCluster != null)
     {
         dfsCluster.Shutdown();
     }
     base.TearDown();
 }
Пример #2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestPipes()
        {
            if (Runtime.GetProperty("compile.c++") == null)
            {
                Log.Info("compile.c++ is not defined, so skipping TestPipes");
                return;
            }
            MiniDFSCluster dfs        = null;
            MiniMRCluster  mr         = null;
            Path           inputPath  = new Path("testing/in");
            Path           outputPath = new Path("testing/out");

            try
            {
                int           numSlaves = 2;
                Configuration conf      = new Configuration();
                dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(numSlaves).Build();
                mr  = new MiniMRCluster(numSlaves, dfs.GetFileSystem().GetUri().ToString(), 1);
                WriteInputFile(dfs.GetFileSystem(), inputPath);
                RunProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 2, twoSplitOutput,
                           null);
                Cleanup(dfs.GetFileSystem(), outputPath);
                RunProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 0, noSortOutput, null
                           );
                Cleanup(dfs.GetFileSystem(), outputPath);
                RunProgram(mr, dfs, wordCountPart, inputPath, outputPath, 3, 2, fixedPartitionOutput
                           , null);
                RunNonPipedProgram(mr, dfs, wordCountNoPipes, null);
                mr.WaitUntilIdle();
            }
            finally
            {
                mr.Shutdown();
                dfs.Shutdown();
            }
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestLazyOutput()
        {
            MiniDFSCluster dfs     = null;
            MiniMRCluster  mr      = null;
            FileSystem     fileSys = null;

            try
            {
                Configuration conf = new Configuration();
                // Start the mini-MR and mini-DFS clusters
                dfs     = new MiniDFSCluster.Builder(conf).NumDataNodes(NumHadoopSlaves).Build();
                fileSys = dfs.GetFileSystem();
                mr      = new MiniMRCluster(NumHadoopSlaves, fileSys.GetUri().ToString(), 1);
                int numReducers = 2;
                int numMappers  = NumHadoopSlaves * NumMapsPerNode;
                CreateInput(fileSys, numMappers);
                Path output1 = new Path("/testlazy/output1");
                // Test 1.
                RunTestLazyOutput(mr.CreateJobConf(), output1, numReducers, true);
                Path[] fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output1, new Utils.OutputFileUtils.OutputFilesFilter
                                                                             ()));
                for (int i = 0; i < fileList.Length; ++i)
                {
                    System.Console.Out.WriteLine("Test1 File list[" + i + "]" + ": " + fileList[i]);
                }
                NUnit.Framework.Assert.IsTrue(fileList.Length == (numReducers - 1));
                // Test 2. 0 Reducers, maps directly write to the output files
                Path output2 = new Path("/testlazy/output2");
                RunTestLazyOutput(mr.CreateJobConf(), output2, 0, true);
                fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output2, new Utils.OutputFileUtils.OutputFilesFilter
                                                                      ()));
                for (int i_1 = 0; i_1 < fileList.Length; ++i_1)
                {
                    System.Console.Out.WriteLine("Test2 File list[" + i_1 + "]" + ": " + fileList[i_1
                                                 ]);
                }
                NUnit.Framework.Assert.IsTrue(fileList.Length == numMappers - 1);
                // Test 3. 0 Reducers, but flag is turned off
                Path output3 = new Path("/testlazy/output3");
                RunTestLazyOutput(mr.CreateJobConf(), output3, 0, false);
                fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output3, new Utils.OutputFileUtils.OutputFilesFilter
                                                                      ()));
                for (int i_2 = 0; i_2 < fileList.Length; ++i_2)
                {
                    System.Console.Out.WriteLine("Test3 File list[" + i_2 + "]" + ": " + fileList[i_2
                                                 ]);
                }
                NUnit.Framework.Assert.IsTrue(fileList.Length == numMappers);
            }
            finally
            {
                if (dfs != null)
                {
                    dfs.Shutdown();
                }
                if (mr != null)
                {
                    mr.Shutdown();
                }
            }
        }