Beispiel #1
0
        public static void Main(string[] args)
        {
            assemblyHandler = new SparkCLRAssemblyHandler();
            AppDomain.CurrentDomain.AssemblyResolve += assemblyHandler.Handle;

            // can't initialize logger early because in MultiThreadWorker mode, JVM will read C#'s stdout via
            // pipe. When initialize logger, some unwanted info will be flushed to stdout. But we can still
            // use stderr
            Console.Error.WriteLine("CSharpWorker [{0}]: Input args [{1}] SocketWrapper [{2}]",
                                    Process.GetCurrentProcess().Id, string.Join(" ", args), SocketFactory.SocketWrapperType);

            if (args.Length != 2)
            {
                Console.Error.WriteLine("Wrong number of args: {0}, will exit", args.Count());
                Environment.Exit(-1);
            }

            if ("pyspark.daemon".Equals(args[1]))
            {
                if (SocketFactory.SocketWrapperType == SocketWrapperType.Rio)
                {
                    // In daemon mode, the socket will be used as server.
                    // Use ThreadPool to retrieve RIO socket results has good performance
                    // than a single thread.
                    RioNative.SetUseThreadPool(true);
                }

                var multiThreadWorker = new MultiThreadWorker();
                multiThreadWorker.Run();
            }
            else
            {
                RunSimpleWorker();
            }
        }
Beispiel #2
0
        public void TestUseThreadPoolForRioNative()
        {
            if (!SocketFactory.IsRioSockSupported())
            {
                Assert.Ignore("Omitting due to missing Riosock.dll. It might caused by no VC++ build tool or running on an OS that not supports Windows RIO socket.");
            }

            // Verify default value for no specific executor cores in environment variable.
            RioNative.SetUseThreadPool(true);
            RioNative.EnsureRioLoaded();
            Assert.AreEqual(2, RioNative.GetWorkThreadNumber());
            RioNative.UnloadRio();
            RioNative.SetUseThreadPool(false);

            // Verify the executor cores is less than 2.
            Environment.SetEnvironmentVariable(ConfigurationService.ExecutorCoresEnvName, "1");
            RioNative.SetUseThreadPool(true);
            RioNative.EnsureRioLoaded();
            Assert.AreEqual(2, RioNative.GetWorkThreadNumber());
            RioNative.UnloadRio();
            RioNative.SetUseThreadPool(false);

            // Verify the executor cores is more than 2.
            Environment.SetEnvironmentVariable(ConfigurationService.ExecutorCoresEnvName, "5");
            RioNative.SetUseThreadPool(true);
            RioNative.EnsureRioLoaded();
            Assert.AreEqual(3, RioNative.GetWorkThreadNumber());
            RioNative.UnloadRio();
            RioNative.SetUseThreadPool(false);
        }
Beispiel #3
0
        public void CleanUpSocketWrapper()
        {
            if (socketWrapperType.Equals("Rio") && SocketFactory.IsRioSockSupported())
            {
                RioNative.UnloadRio();
            }

            // Reset Socket wrapper to default
            Environment.SetEnvironmentVariable(ConfigurationService.CSharpSocketTypeEnvName, "Normal");
            SocketFactory.SocketWrapperType = SocketWrapperType.None;
        }
Beispiel #4
0
        public void TestUseSingleThreadForRioNative()
        {
            if (!SocketFactory.IsRioSockSupported())
            {
                Assert.Ignore("Omitting due to missing Riosock.dll. It might caused by no VC++ build tool or running on an OS that not supports Windows RIO socket.");
            }

            RioNative.SetUseThreadPool(false);
            RioNative.EnsureRioLoaded();
            Assert.AreEqual(1, RioNative.GetWorkThreadNumber());
            RioNative.UnloadRio();
        }
Beispiel #5
0
        public void TestRioSocket()
        {
            if (!SocketFactory.IsRioSockSupported())
            {
                Assert.Ignore("Omitting due to missing Riosock.dll. It might caused by no VC++ build tool or running on an OS that not supports Windows RIO socket.");
            }

            // Set Socket wrapper to Rio
            Environment.SetEnvironmentVariable(ConfigurationService.CSharpSocketTypeEnvName, "Rio");
            SocketFactory.SocketWrapperType    = SocketWrapperType.None;
            RioSocketWrapper.rioRqGrowthFactor = 1;
            var serverSocket = SocketFactory.CreateSocket();

            Assert.IsTrue(serverSocket is RioSocketWrapper);
            SocketTest(serverSocket);

            // Reset socket wrapper type
            Environment.SetEnvironmentVariable(ConfigurationService.CSharpSocketTypeEnvName, string.Empty);
            SocketFactory.SocketWrapperType = SocketWrapperType.None;
            RioNative.UnloadRio();
        }