Пример #1
0
        public static void RunBroadcastReduceTest(IConfiguration tcpPortConfig, bool runOnYarn, int numNodes, string[] args, params string[] runtimeDir)
        {
            int chunkSize        = 2;
            int dims             = 10;
            int iterations       = 10;
            int mapperMemory     = 512;
            int updateTaskMemory = 512;

            if (args.Length > 0)
            {
                dims = Convert.ToInt32(args[0]);
            }

            if (args.Length > 1)
            {
                chunkSize = Convert.ToInt32(args[1]);
            }

            if (args.Length > 2)
            {
                mapperMemory = Convert.ToInt32(args[2]);
            }

            if (args.Length > 3)
            {
                updateTaskMemory = Convert.ToInt32(args[3]);
            }

            if (args.Length > 4)
            {
                iterations = Convert.ToInt32(args[4]);
            }

            IInjector injector;

            if (!runOnYarn)
            {
                injector =
                    TangFactory.GetTang()
                    .NewInjector(OnREEFIMRURunTimeConfiguration <int[], int[], int[]> .GetLocalIMRUConfiguration(numNodes, runtimeDir), tcpPortConfig);
            }
            else
            {
                injector = TangFactory.GetTang()
                           .NewInjector(OnREEFIMRURunTimeConfiguration <int[], int[], int[]> .GetYarnIMRUConfiguration(), tcpPortConfig);
            }
            var broadcastReduceExample = injector.GetInstance <PipelinedBroadcastAndReduce>();

            broadcastReduceExample.Run(numNodes - 1, chunkSize, iterations, dims, mapperMemory, updateTaskMemory);
        }
Пример #2
0
        public static void RunMapperTest(IConfiguration tcpPortConfig, bool runOnYarn, int numNodes, string filename, params string[] runtimeDir)
        {
            IInjector      injector;
            IConfiguration fileSystemConfig;

            if (!runOnYarn)
            {
                injector =
                    TangFactory.GetTang()
                    .NewInjector(OnREEFIMRURunTimeConfiguration <int, int, int> .GetLocalIMRUConfiguration(numNodes, runtimeDir), tcpPortConfig);
                fileSystemConfig = LocalFileSystemConfiguration.ConfigurationModule.Build();
            }
            else
            {
                injector = TangFactory.GetTang()
                           .NewInjector(OnREEFIMRURunTimeConfiguration <int, int, int> .GetYarnIMRUConfiguration(), tcpPortConfig);
                fileSystemConfig = HDFSConfigurationWithoutDriverBinding.ConfigurationModule.Build();
            }

            var mapperCountExample = injector.GetInstance <MapperCount.MapperCount>();

            mapperCountExample.Run(numNodes - 1, filename, fileSystemConfig);
        }
Пример #3
0
        public static void RunBroadcastReduceTest(IConfiguration tcpPortConfig, bool runOnYarn, int numNodes, bool faultTolerant, string[] args, params string[] runtimeDir)
        {
            int chunkSize                   = 2;
            int dims                        = 10;
            int iterations                  = 100;
            int mapperMemory                = 512;
            int updateTaskMemory            = 512;
            int maxRetryNumberInRecovery    = 10;
            int totalNumberOfForcedFailures = 2;

            if (args.Length > 0)
            {
                dims = Convert.ToInt32(args[0]);
            }

            if (args.Length > 1)
            {
                chunkSize = Convert.ToInt32(args[1]);
            }

            if (args.Length > 2)
            {
                mapperMemory = Convert.ToInt32(args[2]);
            }

            if (args.Length > 3)
            {
                updateTaskMemory = Convert.ToInt32(args[3]);
            }

            if (args.Length > 4)
            {
                iterations = Convert.ToInt32(args[4]);
            }

            if (args.Length > 5)
            {
                maxRetryNumberInRecovery = Convert.ToInt32(args[5]);
            }

            if (args.Length > 6)
            {
                totalNumberOfForcedFailures = Convert.ToInt32(args[6]);
            }

            IInjector injector;

            if (!runOnYarn)
            {
                injector =
                    TangFactory.GetTang()
                    .NewInjector(OnREEFIMRURunTimeConfiguration <int[], int[], int[]> .GetLocalIMRUConfiguration(numNodes, runtimeDir), tcpPortConfig);
            }
            else
            {
                injector = TangFactory.GetTang()
                           .NewInjector(OnREEFIMRURunTimeConfiguration <int[], int[], int[]> .GetYarnIMRUConfiguration(), tcpPortConfig);
            }

            if (faultTolerant)
            {
                var broadcastReduceFtExample = injector.GetInstance <PipelinedBroadcastAndReduceWithFaultTolerant>();
                broadcastReduceFtExample.Run(numNodes - 1, chunkSize, iterations, dims, mapperMemory, updateTaskMemory, maxRetryNumberInRecovery, totalNumberOfForcedFailures);
            }
            else
            {
                var broadcastReduceExample = injector.GetInstance <PipelinedBroadcastAndReduce>();
                broadcastReduceExample.Run(numNodes - 1, chunkSize, iterations, dims, mapperMemory, updateTaskMemory);
            }
        }