public static void Main(string[] args)
{
assemblyHandler = new SparkCLRAssemblyHandler();
AppDomain.CurrentDomain.AssemblyResolve += assemblyHandler.Handle;
// can't initialize logger early because in MultiThreadWorker mode, JVM will read C#'s stdout via
// pipe. When initialize logger, some unwanted info will be flushed to stdout. But we can still
// use stderr
Console.Error.WriteLine("CSharpWorker [{0}]: Input args [{1}] SocketWrapper [{2}]",
Process.GetCurrentProcess().Id, string.Join(" ", args), SocketFactory.SocketWrapperType);
if (args.Length != 2)
{
Console.Error.WriteLine("Wrong number of args: {0}, will exit", args.Count());
Environment.Exit(-1);
}
if ("pyspark.daemon".Equals(args[1]))
{
if (SocketFactory.SocketWrapperType == SocketWrapperType.Rio)
{
// In daemon mode, the socket will be used as server.
// Use ThreadPool to retrieve RIO socket results has good performance
// than a single thread.
RioNative.SetUseThreadPool(true);
}
var multiThreadWorker = new MultiThreadWorker();
multiThreadWorker.Run();
}
else
{
RunSimpleWorker();
}
}
public void TestUseThreadPoolForRioNative()
{
if (!SocketFactory.IsRioSockSupported())
{
Assert.Ignore("Omitting due to missing Riosock.dll. It might caused by no VC++ build tool or running on an OS that not supports Windows RIO socket.");
}
// Verify default value for no specific executor cores in environment variable.
RioNative.SetUseThreadPool(true);
RioNative.EnsureRioLoaded();
Assert.AreEqual(2, RioNative.GetWorkThreadNumber());
RioNative.UnloadRio();
RioNative.SetUseThreadPool(false);
// Verify the executor cores is less than 2.
Environment.SetEnvironmentVariable(ConfigurationService.ExecutorCoresEnvName, "1");
RioNative.SetUseThreadPool(true);
RioNative.EnsureRioLoaded();
Assert.AreEqual(2, RioNative.GetWorkThreadNumber());
RioNative.UnloadRio();
RioNative.SetUseThreadPool(false);
// Verify the executor cores is more than 2.
Environment.SetEnvironmentVariable(ConfigurationService.ExecutorCoresEnvName, "5");
RioNative.SetUseThreadPool(true);
RioNative.EnsureRioLoaded();
Assert.AreEqual(3, RioNative.GetWorkThreadNumber());
RioNative.UnloadRio();
RioNative.SetUseThreadPool(false);
}
public void CleanUpSocketWrapper()
{
if (socketWrapperType.Equals("Rio") && SocketFactory.IsRioSockSupported())
{
RioNative.UnloadRio();
}
// Reset Socket wrapper to default
Environment.SetEnvironmentVariable(ConfigurationService.CSharpSocketTypeEnvName, "Normal");
SocketFactory.SocketWrapperType = SocketWrapperType.None;
}
public void TestUseSingleThreadForRioNative()
{
if (!SocketFactory.IsRioSockSupported())
{
Assert.Ignore("Omitting due to missing Riosock.dll. It might caused by no VC++ build tool or running on an OS that not supports Windows RIO socket.");
}
RioNative.SetUseThreadPool(false);
RioNative.EnsureRioLoaded();
Assert.AreEqual(1, RioNative.GetWorkThreadNumber());
RioNative.UnloadRio();
}
public void TestRioSocket()
{
if (!SocketFactory.IsRioSockSupported())
{
Assert.Ignore("Omitting due to missing Riosock.dll. It might caused by no VC++ build tool or running on an OS that not supports Windows RIO socket.");
}
// Set Socket wrapper to Rio
Environment.SetEnvironmentVariable(ConfigurationService.CSharpSocketTypeEnvName, "Rio");
SocketFactory.SocketWrapperType = SocketWrapperType.None;
RioSocketWrapper.rioRqGrowthFactor = 1;
var serverSocket = SocketFactory.CreateSocket();
Assert.IsTrue(serverSocket is RioSocketWrapper);
SocketTest(serverSocket);
// Reset socket wrapper type
Environment.SetEnvironmentVariable(ConfigurationService.CSharpSocketTypeEnvName, string.Empty);
SocketFactory.SocketWrapperType = SocketWrapperType.None;
RioNative.UnloadRio();
}