internal static void TestTaskRunnerReadWrite(
ISocketWrapper serverListener,
PayloadWriter payloadWriter)
{
using (ISocketWrapper serverSocket = serverListener.Accept())
{
System.IO.Stream inputStream = serverSocket.InputStream;
System.IO.Stream outputStream = serverSocket.OutputStream;
payloadWriter.WriteTestData(outputStream);
// Now process the bytes flowing in from the client.
List <object[]> rowsReceived = PayloadReader.Read(inputStream);
// Validate rows received.
Assert.Equal(10, rowsReceived.Count);
for (int i = 0; i < 10; ++i)
{
// Two UDFs registered, thus expecting two columns.
// Refer to TestData.GetDefaultCommandPayload().
object[] row = rowsReceived[i];
Assert.Equal(2, rowsReceived[i].Length);
Assert.Equal($"udf2 udf1 {i}", row[0]);
Assert.Equal(i + i, row[1]);
}
}
}
public void TestsSimpleWorkerTaskRunners(string version)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
serverListener.Listen();
var typedVersion = new Version(version);
var simpleWorker = new SimpleWorker(typedVersion);
Task clientTask = Task.Run(() => simpleWorker.Run(ipEndpoint.Port));
PayloadWriter payloadWriter = new PayloadWriterFactory().Create(typedVersion);
using (ISocketWrapper serverSocket = serverListener.Accept())
{
if ((typedVersion.Major == 3 && typedVersion.Minor >= 2) || typedVersion.Major > 3)
{
int pid = SerDe.ReadInt32(serverSocket.InputStream);
}
TaskRunnerTests.TestTaskRunnerReadWrite(serverSocket, payloadWriter);
}
Assert.True(clientTask.Wait(5000));
}
/// <summary>
/// Listen to the server socket and accept new TCP connection from JVM side. Then create new TaskRunner instance and
/// add it to waitingTaskRunners queue.
/// </summary>
private void StartDaemonServer(ISocketWrapper listener)
{
logger.LogInfo("StartDaemonServer ...");
bool sparkReuseWorker = false;
string envVar = Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"); // this envVar is set in JVM side
var secret = Environment.GetEnvironmentVariable("PYTHON_WORKER_FACTORY_SECRET");
if ((envVar != null) && envVar.Equals("1"))
{
sparkReuseWorker = true;
}
try
{
int trId = 1;
int workThreadNum = 0;
while (true)
{
var socket = listener.Accept();
logger.LogInfo("Connection accepted for taskRunnerId: {0}", trId);
using (var s = socket.GetStream())
{
SerDe.Write(s, trId); // write taskRunnerId to JVM side
s.Flush();
}
TaskRunner taskRunner = new TaskRunner(trId, socket, sparkReuseWorker, secret);
waitingTaskRunners.Add(taskRunner);
taskRunnerRegistry[trId] = taskRunner;
trId++;
int taskRunnerNum = taskRunnerRegistry.Count();
while (workThreadNum < taskRunnerNum) // launch new work thread as appropriate
{
// start threads that do the actual work of running tasks, there are several options here:
// Option 1. TPL - Task Parallel Library
// Option 2. ThreadPool
// Option 3. Self managed threads group
// Option 3 is selected after testing in real cluster because it can get the best performance.
// When using option 1 or 2, it is observered that the boot time may be as large as 50 ~ 60s.
// But it is always less than 1s for option 3. Perhaps this is because TPL and ThreadPool are not
// suitable for long running threads.
new Thread(FetchAndRun).Start();
workThreadNum++;
}
}
}
catch (Exception e)
{
logger.LogError("StartDaemonServer exception, will exit");
logger.LogException(e);
Environment.Exit(-1);
}
}
/// <summary>
/// Starts listening to any connection from JVM.
/// </summary>
/// <param name="listener"></param>
private void StartServer(ISocketWrapper listener)
{
try
{
long connectionId = 1;
int numWorkerThreads = 0;
while (_isRunning)
{
ISocketWrapper socket = listener.Accept();
var connection =
new CallbackConnection(connectionId, socket, _callbackHandlers);
_waitingConnections.Add(connection);
_connections[connectionId] = connection;
++connectionId;
int numConnections = CurrentNumConnections;
// Start worker thread until there are at least as many worker threads
// as there are CallbackConnections. CallbackConnections are expected
// to stay open and reuse the socket to service repeated callback
// requests. However, if there is an issue with a connection, then
// CallbackConnection.Run will return, freeing up extra worker threads
// to service any _waitingConnections.
//
// For example,
// Assume there were 5 worker threads, each servicing a CallbackConnection
// (5 total healthy connections). If 2 CallbackConnection sockets closed
// unexpectedly, then there would be 5 worker threads and 3 healthy
// connections. If a new connection request arrived, then the
// CallbackConnection would be added to the _waitingConnections collection
// and no new worker threads would be started (2 worker threads are already
// waiting to take CallbackConnections from _waitingConnections).
while (numWorkerThreads < numConnections)
{
new Thread(RunWorkerThread)
{
IsBackground = true
}.Start();
++numWorkerThreads;
}
s_logger.LogInfo(
$"Pool snapshot: [NumThreads:{numWorkerThreads}], " +
$"[NumConnections:{numConnections}]");
}
}
catch (Exception e)
{
s_logger.LogError($"StartServer() exits with exception: {e}");
Shutdown();
}
}
internal int StartUpdateServer()
{
innerSocket.Listen();
Task.Run(() =>
{
try
{
IFormatter formatter = new BinaryFormatter();
using (var s = innerSocket.Accept())
using (var ns = s.GetStream())
{
while (!serverShutdown)
{
int numUpdates = SerDe.ReadInt(ns);
for (int i = 0; i < numUpdates; i++)
{
var ms = new MemoryStream(SerDe.ReadBytes(ns));
var update = (Tuple <int, dynamic>)formatter.Deserialize(ms);
if (Accumulator.accumulatorRegistry.ContainsKey(update.Item1))
{
Accumulator accumulator = Accumulator.accumulatorRegistry[update.Item1];
accumulator.GetType().GetMethod("Add").Invoke(accumulator, new object[] { update.Item2 });
}
else
{
Console.Error.WriteLine("WARN: cann't find update.Key: {0} for accumulator, will create a new one", update.Item1);
var genericAccumulatorType = typeof(Accumulator <>);
var specificAccumulatorType = genericAccumulatorType.MakeGenericType(update.Item2.GetType());
Activator.CreateInstance(specificAccumulatorType, new object[] { update.Item1, update.Item2 });
}
}
ns.WriteByte((byte)1); // acknowledge byte other than -1
ns.Flush();
}
}
}
catch (SocketException e)
{
if (e.ErrorCode != 10004) // A blocking operation was interrupted by a call to WSACancelBlockingCall - ISocketWrapper.Close canceled Accep() as expected
{
throw e;
}
}
catch (Exception e)
{
logger.LogError(e.ToString());
throw;
}
});
return((innerSocket.LocalEndPoint as IPEndPoint).Port);
}
private void TestCallbackConnection(
ConcurrentDictionary <int, ICallbackHandler> callbackHandlersDict,
ITestCallbackHandler callbackHandler,
int inputToHandler,
CancellationToken token)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
serverListener.Listen();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
using ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port);
// Don't use "using" here. The CallbackConnection will dispose the socket.
ISocketWrapper serverSocket = serverListener.Accept();
var callbackConnection = new CallbackConnection(0, serverSocket, callbackHandlersDict);
Task task = Task.Run(() => callbackConnection.Run(token));
if (token.IsCancellationRequested)
{
task.Wait();
Assert.False(callbackConnection.IsRunning);
}
else
{
WriteAndReadTestData(clientSocket, callbackHandler, inputToHandler);
if (callbackHandler.Throws)
{
task.Wait();
Assert.False(callbackConnection.IsRunning);
}
else
{
Assert.True(callbackConnection.IsRunning);
// Clean up CallbackConnection
Stream outputStream = clientSocket.OutputStream;
SerDe.Write(outputStream, (int)CallbackConnection.ConnectionStatus.REQUEST_CLOSE);
outputStream.Flush();
task.Wait();
Assert.False(callbackConnection.IsRunning);
}
}
}
private void TestCallbackConnection(
ConcurrentDictionary <int, ICallbackHandler> callbackHandlersDict,
ITestCallbackHandler callbackHandler,
int inputToHandler,
CancellationToken token)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
serverListener.Listen();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port);
var callbackConnection = new CallbackConnection(0, clientSocket, callbackHandlersDict);
Task.Run(() => callbackConnection.Run(token));
using ISocketWrapper serverSocket = serverListener.Accept();
WriteAndReadTestData(serverSocket, callbackHandler, inputToHandler, token);
}
/// <summary>
/// Starts listening to any connection from JVM.
/// </summary>
private void StartServer(ISocketWrapper listener)
{
try
{
bool reuseWorker =
"1".Equals(Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"));
string secret = Utils.SettingUtils.GetWorkerFactorySecret();
int taskRunnerId = 1;
int numWorkerThreads = 0;
while (true)
{
ISocketWrapper socket = listener.Accept();
s_logger.LogInfo($"New connection accepted for TaskRunner [{taskRunnerId}]");
bool authStatus = true;
if (!string.IsNullOrWhiteSpace(secret))
{
// The Spark side expects the PID from a forked process.
// In .NET implementation, a task runner id is used instead.
SerDe.Write(socket.OutputStream, taskRunnerId);
socket.OutputStream.Flush();
if (ConfigurationService.IsDatabricks)
{
SerDe.ReadString(socket.InputStream);
}
authStatus = Authenticator.AuthenticateAsServer(socket, secret);
}
if (authStatus)
{
var taskRunner = new TaskRunner(
taskRunnerId,
socket,
reuseWorker,
_version);
_waitingTaskRunners.Add(taskRunner);
_taskRunners[taskRunnerId] = taskRunner;
++taskRunnerId;
// When reuseWorker is set to true, numTaskRunners will be always one
// greater than numWorkerThreads since TaskRunner.Run() does not return
// so that the task runner object is not removed from _taskRunners.
int numTaskRunners = CurrentNumTaskRunners;
while (numWorkerThreads < numTaskRunners)
{
// Note that in the current implementation of RunWorkerThread() does
// not return. If more graceful exit is required, RunWorkerThread() can
// be updated to return upon receiving a signal from this main thread.
new Thread(RunWorkerThread).Start();
++numWorkerThreads;
}
s_logger.LogInfo(
$"Pool snapshot: [NumThreads:{numWorkerThreads}], [NumTaskRunners:{numTaskRunners}]");
}
else
{
// Use SerDe.ReadBytes() to detect Java side has closed socket
// properly. ReadBytes() will block until the socket is closed.
s_logger.LogError(
"Authentication failed. Waiting for JVM side to close socket.");
SerDe.ReadBytes(socket.InputStream);
socket.Dispose();
}
}
}
catch (Exception e)
{
s_logger.LogError($"StartServer() exits with exception: {e}");
Environment.Exit(-1);
}
}
/// <summary>
/// Listen to the server socket and accept new TCP connection from JVM side. Then create new TaskRunner instance and
/// add it to waitingTaskRunners queue.
/// </summary>
private void StartDaemonServer(ISocketWrapper listener)
{
logger.LogInfo("StartDaemonServer ...");
bool sparkReuseWorker = false;
string envVar = Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"); // this envVar is set in JVM side
if ((envVar != null) && envVar.Equals("1"))
{
sparkReuseWorker = true;
}
try
{
int trId = 1;
int workThreadNum = 0;
while (true)
{
var socket = listener.Accept();
logger.LogInfo("Connection accepted for taskRunnerId: {0}", trId);
using (var s = socket.GetStream())
{
SerDe.Write(s, trId); // write taskRunnerId to JVM side
s.Flush();
}
TaskRunner taskRunner = new TaskRunner(trId, socket, sparkReuseWorker);
waitingTaskRunners.Add(taskRunner);
taskRunnerRegistry[trId] = taskRunner;
trId++;
int taskRunnerNum = taskRunnerRegistry.Count();
while (workThreadNum < taskRunnerNum) // launch new work thread as appropriate
{
// start threads that do the actual work of running tasks, there are several options here:
// Option 1. TPL - Task Parallel Library
// Option 2. ThreadPool
// Option 3. Self managed threads group
// Option 3 is selected after testing in real cluster because it can get the best performance.
// When using option 1 or 2, it is observered that the boot time may be as large as 50 ~ 60s.
// But it is always less than 1s for option 3. Perhaps this is because TPL and ThreadPool are not
// suitable for long running threads.
new Thread(FetchAndRun).Start();
workThreadNum++;
}
}
}
catch (Exception e)
{
logger.LogError("StartDaemonServer exception, will exit");
logger.LogException(e);
Environment.Exit(-1);
}
}
private void SocketTest(ISocketWrapper serverSocket)
{
serverSocket.Listen();
if (serverSocket is RioSocketWrapper)
{
// Do nothing for second listen operation.
Assert.DoesNotThrow(() => serverSocket.Listen(int.MaxValue));
}
var port = ((IPEndPoint)serverSocket.LocalEndPoint).Port;
var clientMsg = "Hello Message from client";
var clientMsgBytes = Encoding.UTF8.GetBytes(clientMsg);
Task.Run(() =>
{
var bytes = new byte[1024];
using (var socket = serverSocket.Accept())
{
using (var s = socket.GetStream())
{
// Receive data
var bytesRec = s.Read(bytes, 0, bytes.Length);
// send echo message.
s.Write(bytes, 0, bytesRec);
s.Flush();
// Receive one byte
var oneByte = s.ReadByte();
// Send echo one byte
byte[] oneBytes = { (byte)oneByte };
s.Write(oneBytes, 0, oneBytes.Length);
Thread.SpinWait(0);
// Keep sending to ensure no memory leak
var longBytes = Encoding.UTF8.GetBytes(new string('x', 8192));
for (int i = 0; i < 1000; i++)
{
s.Write(longBytes, 0, longBytes.Length);
}
byte[] msg = Encoding.ASCII.GetBytes("This is a test<EOF>");
s.Write(msg, 0, msg.Length);
// Receive echo byte.
s.ReadByte();
}
}
});
var clientSock = SocketFactory.CreateSocket();
// Valid invalid operation
Assert.Throws<InvalidOperationException>(() => clientSock.GetStream());
Assert.Throws<InvalidOperationException>(() => clientSock.Receive());
Assert.Throws<InvalidOperationException>(() => clientSock.Send(null));
Assert.Throws<SocketException>(() => clientSock.Connect(IPAddress.Any, 1024));
clientSock.Connect(IPAddress.Loopback, port);
// Valid invalid operation
var byteBuf = ByteBufPool.Default.Allocate();
Assert.Throws<ArgumentException>(() => clientSock.Send(byteBuf));
byteBuf.Release();
Assert.Throws<SocketException>(() => clientSock.Listen());
if (clientSock is RioSocketWrapper)
{
Assert.Throws<InvalidOperationException>(() => clientSock.Accept());
}
using (var s = clientSock.GetStream())
{
// Send message
s.Write(clientMsgBytes, 0, clientMsgBytes.Length);
// Receive echo message
var bytes = new byte[1024];
var bytesRec = s.Read(bytes, 0, bytes.Length);
Assert.AreEqual(clientMsgBytes.Length, bytesRec);
var recvStr = Encoding.UTF8.GetString(bytes, 0, bytesRec);
Assert.AreEqual(clientMsg, recvStr);
// Send one byte
byte[] oneBytes = { 1 };
s.Write(oneBytes, 0, oneBytes.Length);
// Receive echo message
var oneByte = s.ReadByte();
Assert.AreEqual((byte)1, oneByte);
// Keep receiving to ensure no memory leak.
while (true)
{
bytesRec = s.Read(bytes, 0, bytes.Length);
recvStr = Encoding.UTF8.GetString(bytes, 0, bytesRec);
if (recvStr.IndexOf("<EOF>", StringComparison.OrdinalIgnoreCase) > -1)
{
break;
}
}
// send echo bytes
s.Write(oneBytes, 0, oneBytes.Length);
}
clientSock.Close();
// Verify invalid operation
Assert.Throws<ObjectDisposedException>(() => clientSock.Receive());
serverSocket.Close();
}
private void SocketTest(ISocketWrapper serverSocket)
{
serverSocket.Listen();
if (serverSocket is RioSocketWrapper)
{
// Do nothing for second listen operation.
Assert.DoesNotThrow(() => serverSocket.Listen(int.MaxValue));
}
var port = ((IPEndPoint)serverSocket.LocalEndPoint).Port;
var clientMsg = "Hello Message from client";
var clientMsgBytes = Encoding.UTF8.GetBytes(clientMsg);
Task.Run(() =>
{
var bytes = new byte[1024];
using (var socket = serverSocket.Accept())
{
using (var s = socket.GetStream())
{
// Receive data
var bytesRec = s.Read(bytes, 0, bytes.Length);
// send echo message.
s.Write(bytes, 0, bytesRec);
s.Flush();
// Receive one byte
var oneByte = s.ReadByte();
// Send echo one byte
byte[] oneBytes = { (byte)oneByte };
s.Write(oneBytes, 0, oneBytes.Length);
Thread.SpinWait(0);
// Keep sending to ensure no memory leak
var longBytes = Encoding.UTF8.GetBytes(new string('x', 8192));
for (int i = 0; i < 1000; i++)
{
s.Write(longBytes, 0, longBytes.Length);
}
byte[] msg = Encoding.ASCII.GetBytes("This is a test<EOF>");
s.Write(msg, 0, msg.Length);
// Receive echo byte.
s.ReadByte();
}
}
});
var clientSock = SocketFactory.CreateSocket();
// Valid invalid operation
Assert.Throws <InvalidOperationException>(() => clientSock.GetStream());
Assert.Throws <InvalidOperationException>(() => clientSock.Receive());
Assert.Throws <InvalidOperationException>(() => clientSock.Send(null));
Assert.Throws <SocketException>(() => clientSock.Connect(IPAddress.Any, 1024));
clientSock.Connect(IPAddress.Loopback, port);
// Valid invalid operation
var byteBuf = ByteBufPool.Default.Allocate();
Assert.Throws <ArgumentException>(() => clientSock.Send(byteBuf));
byteBuf.Release();
Assert.Throws <SocketException>(() => clientSock.Listen());
if (clientSock is RioSocketWrapper)
{
Assert.Throws <InvalidOperationException>(() => clientSock.Accept());
}
using (var s = clientSock.GetStream())
{
// Send message
s.Write(clientMsgBytes, 0, clientMsgBytes.Length);
// Receive echo message
var bytes = new byte[1024];
var bytesRec = s.Read(bytes, 0, bytes.Length);
Assert.AreEqual(clientMsgBytes.Length, bytesRec);
var recvStr = Encoding.UTF8.GetString(bytes, 0, bytesRec);
Assert.AreEqual(clientMsg, recvStr);
// Send one byte
byte[] oneBytes = { 1 };
s.Write(oneBytes, 0, oneBytes.Length);
// Receive echo message
var oneByte = s.ReadByte();
Assert.AreEqual((byte)1, oneByte);
// Keep receiving to ensure no memory leak.
while (true)
{
bytesRec = s.Read(bytes, 0, bytes.Length);
recvStr = Encoding.UTF8.GetString(bytes, 0, bytesRec);
if (recvStr.IndexOf("<EOF>", StringComparison.OrdinalIgnoreCase) > -1)
{
break;
}
}
// send echo bytes
s.Write(oneBytes, 0, oneBytes.Length);
}
clientSock.Close();
// Verify invalid operation
Assert.Throws <ObjectDisposedException>(() => clientSock.Receive());
serverSocket.Close();
}