/// <summary> /// Starts listening to any connection from JVM. /// </summary> private void StartServer(ISocketWrapper listener) { try { bool reuseWorker = "1".Equals(Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER")); string secret = Utils.SettingUtils.GetWorkerFactorySecret(); int taskRunnerId = 1; int numWorkerThreads = 0; while (true) { ISocketWrapper socket = listener.Accept(); s_logger.LogInfo($"New connection accepted for TaskRunner [{taskRunnerId}]"); bool authStatus = true; if (!string.IsNullOrWhiteSpace(secret)) { // The Spark side expects the PID from a forked process. // In .NET implementation, a task runner id is used instead. SerDe.Write(socket.OutputStream, taskRunnerId); socket.OutputStream.Flush(); if (ConfigurationService.IsDatabricks) { SerDe.ReadString(socket.InputStream); } authStatus = Authenticator.AuthenticateAsServer(socket, secret); } if (authStatus) { var taskRunner = new TaskRunner( taskRunnerId, socket, reuseWorker, _version); _waitingTaskRunners.Add(taskRunner); _taskRunners[taskRunnerId] = taskRunner; ++taskRunnerId; // When reuseWorker is set to true, numTaskRunners will be always one // greater than numWorkerThreads since TaskRunner.Run() does not return // so that the task runner object is not removed from _taskRunners. int numTaskRunners = CurrentNumTaskRunners; while (numWorkerThreads < numTaskRunners) { // Note that in the current implementation of RunWorkerThread() does // not return. If more graceful exit is required, RunWorkerThread() can // be updated to return upon receiving a signal from this main thread. new Thread(RunWorkerThread).Start(); ++numWorkerThreads; } s_logger.LogInfo( $"Pool snapshot: [NumThreads:{numWorkerThreads}], [NumTaskRunners:{numTaskRunners}]"); } else { // Use SerDe.ReadBytes() to detect Java side has closed socket // properly. ReadBytes() will block until the socket is closed. s_logger.LogError( "Authentication failed. Waiting for JVM side to close socket."); SerDe.ReadBytes(socket.InputStream); socket.Dispose(); } } } catch (Exception e) { s_logger.LogError($"StartServer() exits with exception: {e}"); Environment.Exit(-1); } }