/// <summary>
/// Starts listening to any connection from JVM.
/// </summary>
private void StartServer(ISocketWrapper listener)
{
try
{
bool reuseWorker =
"1".Equals(Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"));
string secret = Utils.SettingUtils.GetWorkerFactorySecret();
int taskRunnerId = 1;
int numWorkerThreads = 0;
while (true)
{
ISocketWrapper socket = listener.Accept();
s_logger.LogInfo($"New connection accepted for TaskRunner [{taskRunnerId}]");
bool authStatus = true;
if (!string.IsNullOrWhiteSpace(secret))
{
// The Spark side expects the PID from a forked process.
// In .NET implementation, a task runner id is used instead.
SerDe.Write(socket.OutputStream, taskRunnerId);
socket.OutputStream.Flush();
if (ConfigurationService.IsDatabricks)
{
SerDe.ReadString(socket.InputStream);
}
authStatus = Authenticator.AuthenticateAsServer(socket, secret);
}
if (authStatus)
{
var taskRunner = new TaskRunner(
taskRunnerId,
socket,
reuseWorker,
_version);
_waitingTaskRunners.Add(taskRunner);
_taskRunners[taskRunnerId] = taskRunner;
++taskRunnerId;
// When reuseWorker is set to true, numTaskRunners will be always one
// greater than numWorkerThreads since TaskRunner.Run() does not return
// so that the task runner object is not removed from _taskRunners.
int numTaskRunners = CurrentNumTaskRunners;
while (numWorkerThreads < numTaskRunners)
{
// Note that in the current implementation of RunWorkerThread() does
// not return. If more graceful exit is required, RunWorkerThread() can
// be updated to return upon receiving a signal from this main thread.
new Thread(RunWorkerThread).Start();
++numWorkerThreads;
}
s_logger.LogInfo(
$"Pool snapshot: [NumThreads:{numWorkerThreads}], [NumTaskRunners:{numTaskRunners}]");
}
else
{
// Use SerDe.ReadBytes() to detect Java side has closed socket
// properly. ReadBytes() will block until the socket is closed.
s_logger.LogError(
"Authentication failed. Waiting for JVM side to close socket.");
SerDe.ReadBytes(socket.InputStream);
socket.Dispose();
}
}
}
catch (Exception e)
{
s_logger.LogError($"StartServer() exits with exception: {e}");
Environment.Exit(-1);
}
}
