public RequestHandler CreateRequestHandler(ConnectionManager manager, ISocketWrapper socket)
{
RequestHandler.HttpHeader header;
string gdl = string.Empty; // new string(contentInput);
using (var input = new StreamReader(socket.GetStream()))
{
header = ReadHeader(input);
var buffer = new char[header.ContentLength];
int bytesRemaining = header.ContentLength;
while (bytesRemaining > 0)
{
int read = input.Read(buffer, 0, header.ContentLength);
gdl += new string(buffer).Substring(0, read);
bytesRemaining -= read;
}
}
GdlList content = GameContainer.Parser.Parse(gdl);
if (content == null || !content.Any())
{
Console.WriteLine("There was no content parsed");
return null;
}
RequestHandler handler = CreateFromList(socket, header, content);
handler.SetManager(manager);
return handler;
}
public TaskRunner(int taskId, ISocketWrapper socket, bool reuseSocket, Version version)
{
TaskId = taskId;
_socket = socket;
_reuseSocket = reuseSocket;
_version = version;
}
public void TestCallbackServer()
{
var callbackServer = new CallbackServer(_mockJvm.Object, false);
var callbackHandler = new TestCallbackHandler();
callbackHandler.Id = callbackServer.RegisterCallback(callbackHandler);
Assert.Equal(1, callbackHandler.Id);
using ISocketWrapper callbackSocket = SocketFactory.CreateSocket();
callbackServer.Run(callbackSocket);
int connectionNumber = 2;
var clientSockets = new ISocketWrapper[connectionNumber];
for (int i = 0; i < connectionNumber; ++i)
{
var ipEndpoint = (IPEndPoint)callbackSocket.LocalEndPoint;
ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSockets[i] = clientSocket;
clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port);
WriteAndReadTestData(clientSocket, callbackHandler, i);
}
Assert.Equal(connectionNumber, callbackServer.CurrentNumConnections);
IOrderedEnumerable <int> actualValues = callbackHandler.Inputs.OrderBy(i => i);
IEnumerable <int> expectedValues = Enumerable
.Range(0, connectionNumber)
.Select(i => callbackHandler.Apply(i))
.OrderBy(i => i);
Assert.True(expectedValues.SequenceEqual(actualValues));
}
public void ProcessCall(ISocketWrapper socketWrapper)
{
var result = socketWrapper.PeekToString(9);
Exception caughtException = null;
// TODO: Add exception handlers for custom exceptions and pass
// the exceptions to the respective return processor
// Ex: BadRequestException -> HttpBadRequestReturnController
try
{
IHttpCallProcessor callProcessor = VerifyAndExtractProcessor(result);
callProcessor.ProcessCall(socketWrapper);
}
catch (Exception ex)
{
if (_exceptionHandler.HasResponse(ex.GetType().Name) == false)
{
throw;
}
caughtException = ex;
}
IHttpResponse response = null;
if (caughtException != null)
{
response = _exceptionHandler.GetResponseFromException(caughtException);
}
}
public void ReadWhenClosedAfterWrite()
{
// Read when closed (no data pending etc), after state refreshed by Write.
using (ITestSocketPair pair = m_spFactory.CreateSocketPair()) {
ISocketWrapper sock = pair.SocketA;
NetworkStream strm = sock.GetStream();
Assert.IsTrue(sock.Connected, "10a");
PeerCloses(pair);
if (!m_connectedMayBeFalseEarlier)
{
Assert.IsTrue(sock.Connected, "10b");
WriteSomeBytes(strm);
}
int count = 0;
while (true)
{
bool didFail = WriteSomeBytesExpectError(strm);
++count;
if (didFail)
{
break;
}
Assert.IsTrue(sock.Connected, "11c_" + count);
}
Assert.IsFalse(sock.Connected, "10d");
ReadSomeBytesExpectError(strm);
Assert.IsFalse(sock.Connected, "10e");
}
}
internal void Run(ISocketWrapper listener)
{
try
{
listener.Listen();
// Communicate the server port back to the Spark using standard output.
Stream outputStream = Console.OpenStandardOutput();
var serverPort = ((IPEndPoint)listener.LocalEndPoint).Port;
SerDe.Write(outputStream, serverPort);
// Now the logger can be initialized after standard output's usage is done.
s_logger = LoggerServiceFactory.GetLogger(typeof(DaemonWorker));
s_logger.LogInfo($"Started .NET DaemonServer with port {serverPort}.");
// Start accepting connections from JVM.
new Thread(() => { StartServer(listener); }).Start();
WaitForSignal();
}
catch (Exception e)
{
s_logger.LogError($".NET DaemonWorker is exiting with exception: {e}.");
Environment.Exit(-1);
}
finally
{
_waitingTaskRunners.Dispose();
}
}
private void WriteAndReadTestData(
ISocketWrapper socket,
ITestCallbackHandler callbackHandler,
int inputToHandler)
{
Stream inputStream = socket.InputStream;
Stream outputStream = socket.OutputStream;
SerDe.Write(outputStream, (int)CallbackFlags.CALLBACK);
SerDe.Write(outputStream, callbackHandler.Id);
SerDe.Write(outputStream, sizeof(int));
SerDe.Write(outputStream, inputToHandler);
SerDe.Write(outputStream, (int)CallbackFlags.END_OF_STREAM);
outputStream.Flush();
int callbackFlag = SerDe.ReadInt32(inputStream);
if (callbackFlag == (int)CallbackFlags.DOTNET_EXCEPTION_THROWN)
{
string exceptionMessage = SerDe.ReadString(inputStream);
Assert.False(string.IsNullOrEmpty(exceptionMessage));
Assert.Contains(callbackHandler.ExceptionMessage, exceptionMessage);
}
else
{
Assert.Equal((int)CallbackFlags.END_OF_STREAM, callbackFlag);
}
}
public TaskRunner(int trId, ISocketWrapper socket, bool socketReuse, string secret)
{
TaskId = trId;
this.socket = socket;
this.socketReuse = socketReuse;
this.secret = secret;
}
public void WriteWhenClosedAfterRead()
{
// Write when closed.
using (ITestSocketPair pair = m_spFactory.CreateSocketPair()) {
ISocketWrapper sock = pair.SocketA;
NetworkStream strm = sock.GetStream();
Assert.IsTrue(sock.Connected, "11a");
PeerSendsData(pair);
PeerCloses(pair);
Assert.IsTrue(sock.Connected, "11b");
ReadSomeBytes(strm, 5);
Assert.IsTrue(sock.Connected, "11c");
if (!m_connectedMayBeFalseEarlier)
{
WriteSomeBytes(strm);
Assert.IsTrue(sock.Connected, "11d");
}
int count = 0;
while (true)
{
bool didFail = WriteSomeBytesExpectError(strm);
++count;
if (didFail)
{
break;
}
Assert.IsTrue(sock.Connected, "11e_" + count);
}
Assert.IsFalse(sock.Connected, "11f");
}
}
private static void CreateAndVerifyConnection(ISocketWrapper daemonSocket)
{
var ipEndpoint = (IPEndPoint)daemonSocket.LocalEndPoint;
int port = ipEndpoint.Port;
ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSocket.Connect(ipEndpoint.Address, port);
// Now process the bytes flowing in from the client.
PayloadWriter payloadWriter = new PayloadWriterFactory().Create();
payloadWriter.WriteTestData(clientSocket.OutputStream);
List <object[]> rowsReceived = PayloadReader.Read(clientSocket.InputStream);
// Validate rows received.
Assert.Equal(10, rowsReceived.Count);
for (int i = 0; i < 10; ++i)
{
// Two UDFs registered, thus expecting two columns.
// Refer to TestData.GetDefaultCommandPayload().
object[] row = rowsReceived[i];
Assert.Equal(2, rowsReceived[i].Length);
Assert.Equal($"udf2 udf1 {i}", row[0]);
Assert.Equal(i + i, row[1]);
}
}
internal DefaultKayakSocket(ISocketWrapper socket, IScheduler scheduler)
{
this.ID = Interlocked.Increment(ref s_NextId);
this.m_Socket = socket;
this.m_Scheduler = scheduler;
m_State = new KayakSocketState(false);
}
internal static void TestTaskRunnerReadWrite(
ISocketWrapper serverListener,
PayloadWriter payloadWriter)
{
using (ISocketWrapper serverSocket = serverListener.Accept())
{
System.IO.Stream inputStream = serverSocket.InputStream;
System.IO.Stream outputStream = serverSocket.OutputStream;
payloadWriter.WriteTestData(outputStream);
// Now process the bytes flowing in from the client.
List <object[]> rowsReceived = PayloadReader.Read(inputStream);
// Validate rows received.
Assert.Equal(10, rowsReceived.Count);
for (int i = 0; i < 10; ++i)
{
// Two UDFs registered, thus expecting two columns.
// Refer to TestData.GetDefaultCommandPayload().
object[] row = rowsReceived[i];
Assert.Equal(2, rowsReceived[i].Length);
Assert.Equal($"udf2 udf1 {i}", row[0]);
Assert.Equal(i + i, row[1]);
}
}
}
internal DefaultKayakSocket(ISocketWrapper socket, IScheduler scheduler)
{
this.id = nextId++;
this.socket = socket;
this.scheduler = scheduler;
state = new KayakSocketState(false);
}
/// <summary>
/// Mark request as 'finished' i.e. handled. This closes the socket.
/// </summary>
public void Finish()
{
_socket.Close();
_socket = null;
_writer = null;
}
internal DefaultKayakSocket(ISocketWrapper socket, IScheduler scheduler)
{
this.id = nextId++;
this.socket = socket;
this.scheduler = scheduler;
state = new KayakSocketState(false);
}
public void TestsSimpleWorkerTaskRunners(string version)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
serverListener.Listen();
var typedVersion = new Version(version);
var simpleWorker = new SimpleWorker(typedVersion);
Task clientTask = Task.Run(() => simpleWorker.Run(ipEndpoint.Port));
PayloadWriter payloadWriter = new PayloadWriterFactory().Create(typedVersion);
using (ISocketWrapper serverSocket = serverListener.Accept())
{
if ((typedVersion.Major == 3 && typedVersion.Minor >= 2) || typedVersion.Major > 3)
{
int pid = SerDe.ReadInt32(serverSocket.InputStream);
}
TaskRunnerTests.TestTaskRunnerReadWrite(serverSocket, payloadWriter);
}
Assert.True(clientTask.Wait(5000));
}
internal void Run(int port)
{
try
{
string secret = Utils.SettingUtils.GetWorkerFactorySecret();
s_logger.LogInfo($"RunSimpleWorker() is starting with port = {port}.");
ISocketWrapper socket = SocketFactory.CreateSocket();
socket.Connect(IPAddress.Loopback, port, secret);
if ((_version.Major == 3 && _version.Minor >= 2) || _version.Major > 3)
{
int pid = Process.GetCurrentProcess().Id;
SerDe.Write(socket.OutputStream, pid);
socket.OutputStream.Flush();
}
new TaskRunner(0, socket, false, _version).Run();
}
catch (Exception e)
{
s_logger.LogError("RunSimpleWorker() failed with exception:");
s_logger.LogException(e);
Environment.Exit(-1);
}
s_logger.LogInfo("RunSimpleWorker() finished successfully");
}
/// <summary>
/// Reads the given stream to construct a BroadcastVariables object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>BroadcastVariables object</returns>
internal BroadcastVariables Process(Stream stream)
{
var broadcastVars = new BroadcastVariables();
ISocketWrapper socket = null;
broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream);
broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0);
if (broadcastVars.DecryptionServerNeeded)
{
broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream);
broadcastVars.Secret = SerDe.ReadString(stream);
if (broadcastVars.Count > 0)
{
socket = SocketFactory.CreateSocket();
socket.Connect(
IPAddress.Loopback,
broadcastVars.DecryptionServerPort,
broadcastVars.Secret);
}
}
var formatter = new BinaryFormatter();
for (int i = 0; i < broadcastVars.Count; ++i)
{
long bid = SerDe.ReadInt64(stream);
if (bid >= 0)
{
if (broadcastVars.DecryptionServerNeeded)
{
long readBid = SerDe.ReadInt64(socket.InputStream);
if (bid != readBid)
{
throw new Exception("The Broadcast Id received from the encryption " +
$"server {readBid} is different from the Broadcast Id received " +
$"from the payload {bid}.");
}
object value = formatter.Deserialize(socket.InputStream);
BroadcastRegistry.Add(bid, value);
}
else
{
string path = SerDe.ReadString(stream);
using FileStream fStream =
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
object value = formatter.Deserialize(fStream);
BroadcastRegistry.Add(bid, value);
}
}
else
{
bid = -bid - 1;
BroadcastRegistry.Remove(bid);
}
}
socket?.Dispose();
return(broadcastVars);
}
public ConnectionHandler(Socket client, IServerRouteConfig serverRouteConfig)
{
CommonValidator.ThrowIfNull(client, nameof(client));
CommonValidator.ThrowIfNull(serverRouteConfig, nameof(serverRouteConfig));
this.client = new SocketWrapper(client);
this.serverRouteConfig = serverRouteConfig;
}
/// <summary>
/// Initializes a SocketStream with a RioSocketWrapper object.
/// </summary>
/// <param name="socket">a RioSocketWrapper object</param>
public SocketStream(RioSocketWrapper socket)
{
if (socket == null)
{
throw new ArgumentNullException("socket");
}
streamSocket = socket;
bufPool = ByteBufPool.UnsafeDefault;
}
/// <summary>
/// Initializes a SocketStream with a RioSocketWrapper object.
/// </summary>
/// <param name="socket">a RioSocketWrapper object</param>
public SocketStream(RioSocketWrapper socket)
{
if (socket == null)
{
throw new ArgumentNullException("socket");
}
streamSocket = socket;
bufPool = ByteBufPool.UnsafeDefault;
}
private static void AsyncReadSomeBytes(Stream strm, ISocketWrapper sock, int count, bool expectedConnectedValue)
{
if (true != expectedConnectedValue)
{
throw new ArgumentException("expectedConnectedValue NotImplemented"); // FOR
}
int readLen = AsyncReadSomeBytes_(strm, count, sock, expectedConnectedValue);
Assert.AreEqual(count, readLen);
}
public void TestJvmCallbackClientProperty()
{
var server = new CallbackServer(_mockJvm.Object, run: false);
Assert.Throws <InvalidOperationException>(() => server.JvmCallbackClient);
using ISocketWrapper callbackSocket = SocketFactory.CreateSocket();
server.Run(callbackSocket);
Assert.NotNull(server.JvmCallbackClient);
}
/// <summary>
/// Listen to the server socket and accept new TCP connection from JVM side. Then create new TaskRunner instance and
/// add it to waitingTaskRunners queue.
/// </summary>
private void StartDaemonServer(ISocketWrapper listener)
{
logger.LogInfo("StartDaemonServer ...");
bool sparkReuseWorker = false;
string envVar = Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"); // this envVar is set in JVM side
var secret = Environment.GetEnvironmentVariable("PYTHON_WORKER_FACTORY_SECRET");
if ((envVar != null) && envVar.Equals("1"))
{
sparkReuseWorker = true;
}
try
{
int trId = 1;
int workThreadNum = 0;
while (true)
{
var socket = listener.Accept();
logger.LogInfo("Connection accepted for taskRunnerId: {0}", trId);
using (var s = socket.GetStream())
{
SerDe.Write(s, trId); // write taskRunnerId to JVM side
s.Flush();
}
TaskRunner taskRunner = new TaskRunner(trId, socket, sparkReuseWorker, secret);
waitingTaskRunners.Add(taskRunner);
taskRunnerRegistry[trId] = taskRunner;
trId++;
int taskRunnerNum = taskRunnerRegistry.Count();
while (workThreadNum < taskRunnerNum) // launch new work thread as appropriate
{
// start threads that do the actual work of running tasks, there are several options here:
// Option 1. TPL - Task Parallel Library
// Option 2. ThreadPool
// Option 3. Self managed threads group
// Option 3 is selected after testing in real cluster because it can get the best performance.
// When using option 1 or 2, it is observered that the boot time may be as large as 50 ~ 60s.
// But it is always less than 1s for option 3. Perhaps this is because TPL and ThreadPool are not
// suitable for long running threads.
new Thread(FetchAndRun).Start();
workThreadNum++;
}
}
}
catch (Exception e)
{
logger.LogError("StartDaemonServer exception, will exit");
logger.LogException(e);
Environment.Exit(-1);
}
}
public void LocalCloseOnSocket()
{
// Local close (Socket).
using (ITestSocketPair pair = m_spFactory.CreateSocketPair()) {
ISocketWrapper sock = pair.SocketA;
NetworkStream strm = sock.GetStream();
Assert.IsTrue(sock.Connected, "2a");
sock.Close();
Assert.IsFalse(sock.Connected, "2b");
}
}
public void TestInitialize()
{
sc = new SparkContext(null);
sc.StartAccumulatorServer();
// get accumulator server port and connect to accumuator server
int serverPort = (sc.SparkContextProxy as MockSparkContextProxy).AccumulatorServerPort;
sock = SocketFactory.CreateSocket();
sock.Connect(IPAddress.Loopback, serverPort, null);
}
/// <summary>
/// Starts listening to any connection from JVM.
/// </summary>
/// <param name="listener"></param>
private void StartServer(ISocketWrapper listener)
{
try
{
long connectionId = 1;
int numWorkerThreads = 0;
while (_isRunning)
{
ISocketWrapper socket = listener.Accept();
var connection =
new CallbackConnection(connectionId, socket, _callbackHandlers);
_waitingConnections.Add(connection);
_connections[connectionId] = connection;
++connectionId;
int numConnections = CurrentNumConnections;
// Start worker thread until there are at least as many worker threads
// as there are CallbackConnections. CallbackConnections are expected
// to stay open and reuse the socket to service repeated callback
// requests. However, if there is an issue with a connection, then
// CallbackConnection.Run will return, freeing up extra worker threads
// to service any _waitingConnections.
//
// For example,
// Assume there were 5 worker threads, each servicing a CallbackConnection
// (5 total healthy connections). If 2 CallbackConnection sockets closed
// unexpectedly, then there would be 5 worker threads and 3 healthy
// connections. If a new connection request arrived, then the
// CallbackConnection would be added to the _waitingConnections collection
// and no new worker threads would be started (2 worker threads are already
// waiting to take CallbackConnections from _waitingConnections).
while (numWorkerThreads < numConnections)
{
new Thread(RunWorkerThread)
{
IsBackground = true
}.Start();
++numWorkerThreads;
}
s_logger.LogInfo(
$"Pool snapshot: [NumThreads:{numWorkerThreads}], " +
$"[NumConnections:{numConnections}]");
}
}
catch (Exception e)
{
s_logger.LogError($"StartServer() exits with exception: {e}");
Shutdown();
}
}
/**
* Constructor.
*
* @param socket The socket connection used for this handler.
* @param header The HTTP header sent with this request.
* @param gameId The game this connection is operating on.
*
* @throws IOException If something goes wrong with the socket.
*/
protected RequestHandler(ISocketWrapper socket, HttpHeader header, string gameId)
{
_header = header;
_socket = socket;
//writer_ = new StreamWriter(socket_.getOutputStream());
//writer_ = new StreamWriter(socket_.GetStream());
_writer = _socket.GetWriter();
GameId = gameId;
}
//======================================================================
//----
private void AsyncWriteSomeBytes(NetworkStream strm, ISocketWrapper sock)
{
byte[] data = Encoding.ASCII.GetBytes("ABCDEFHIJK");
IAsyncResult ar = strm.BeginWrite(data, 0, data.Length, null, null);
Assert.IsTrue(sock.Connected, "after Begin expected to be Connected");
strm.EndWrite(ar);
if (!m_connectedMayBeFalseEarlier)
{
Assert.IsTrue(sock.Connected, "after End expected to be Connected");
}
}
public void LocalCloseOnSocketAndWrite()
{
// Local close (Socket).
using (ITestSocketPair pair = m_spFactory.CreateSocketPair()) {
ISocketWrapper sock = pair.SocketA;
NetworkStream strm = sock.GetStream();
Assert.IsTrue(sock.Connected, "2a");
sock.Close();
Assert.IsFalse(sock.Connected, "2b");
WriteSomeBytesExpectError(strm, typeof(ObjectDisposedException));
}
}
/// <summary>
/// Runs the DaemonWorker server.
/// </summary>
internal void Run()
{
// TODO: Note that daemon server is stopped from Spark, it is done with
// Process.destroy(). It should send SIGTERM and the signal should be handled
// by the following:
// AppDomain.CurrentDomain.ProcessExit += (s, e) => {};,
// but the above handler is not invoked. This can be investigated if more
// graceful exit is required.
ISocketWrapper listener = SocketFactory.CreateSocket();
Run(listener);
}
internal CallbackConnection(
long connectionId,
ISocketWrapper socket,
ConcurrentDictionary <int, ICallbackHandler> callbackHandlers)
{
ConnectionId = connectionId;
_socket = socket;
_callbackHandlers = callbackHandlers;
s_logger.LogInfo(
$"[{ConnectionId}] Connected with RemoteEndPoint: {socket.RemoteEndPoint}");
}
/// <summary>
/// Returns row objects based on the function (either "toPythonIterator" or
/// "collectToPython").
/// </summary>
/// <param name="funcName"></param>
/// <returns></returns>
private IEnumerable <Row> GetRows(string funcName)
{
(int port, string secret) = GetConnectionInfo(funcName);
using (ISocketWrapper socket = SocketFactory.CreateSocket())
{
socket.Connect(IPAddress.Loopback, port, secret);
foreach (Row row in new RowCollector().Collect(socket))
{
yield return(row);
}
}
}
private static int AsyncReadSomeBytes_(Stream strm, int count, ISocketWrapper sock, bool expectedConnectedValue)
{
byte[] buf = new byte[7];
IAsyncResult ar = strm.BeginRead(buf, 0, count, null, null);
Assert.AreEqual(expectedConnectedValue, sock.Connected, "after Begin");
int readLen = strm.EndRead(ar);
bool cs = ar.CompletedSynchronously;
Assert.AreEqual(expectedConnectedValue, sock.Connected, "after End");
return(readLen);
}
public BluetoothClientTestSocketPair()
{
TestRfCommIf rfcommIf;
TestRfcommPort port;
BluetoothClient cli;
Stream strm2;
WidcommBluetoothClientCommsTest.Create_ConnectedBluetoothClient(
out rfcommIf, out port, out cli, out strm2);
m_cliA = new BluetoothClientWrapper(cli);
m_port = port;
m_rfcommIf = rfcommIf;
}
private void Create(AddressFamily af)
{
using (Socket lstnr = new Socket(af, SocketType.Stream, ProtocolType.Unspecified)) {
lstnr.Bind(new IPEndPoint(
af == AddressFamily.InterNetworkV6 ? IPAddress.IPv6Loopback : IPAddress.Loopback, 0));
lstnr.Listen(1);
EndPoint svrEp = lstnr.LocalEndPoint;
Socket cli = new Socket(svrEp.AddressFamily, lstnr.SocketType, lstnr.ProtocolType);
cli.Connect(svrEp);
m_cli = new SocketWrapper(cli);
m_svr = lstnr.Accept();
}
}
public void Connect(IPEndPoint ep)
{
state.SetConnecting();
Debug.WriteLine("KayakSocket: connecting to " + ep);
this.socket = new SocketWrapper(ep.Address.AddressFamily);
socket.BeginConnect(ep, iasr =>
{
Debug.WriteLine("KayakSocket: connected to " + ep);
Exception error = null;
try
{
socket.EndConnect(iasr);
}
catch (Exception e)
{
error = e;
}
scheduler.Post(() =>
{
if (error is ObjectDisposedException)
return;
if (error != null)
{
state.SetError();
Debug.WriteLine("KayakSocket: error while connecting to " + ep);
RaiseError(error);
}
else
{
state.SetConnected();
Debug.WriteLine("KayakSocket: connected to " + ep);
del.OnConnected(this);
BeginRead();
}
});
});
}
internal StopRequestHandler(ISocketWrapper socket, HttpHeader header, GdlList content, string matchId)
: base(socket, header, matchId)
{
_content = content;
}
public TaskRunner(int trId, ISocketWrapper socket, bool socketReuse)
{
this.trId = trId;
this.socket = socket;
this.socketReuse = socketReuse;
}
public void TestMultiThreadWorkerMultiTaskRunners()
{
Process worker;
int serverPort = CreateServer(out worker, true);
Console.WriteLine("serverPort: {0}", serverPort);
int num = 2;
var sockets = new ISocketWrapper[2];
var taskRunnerIds = new int[2];
for (int index = 0; index < num; index++)
{
sockets[index] = CreateSocket(serverPort);
}
for (int index = 0; index < num; index++)
{
using (var s = sockets[index].GetStream())
{
taskRunnerIds[index] = SerDe.ReadInt(s);
WritePayloadHeaderToWorker(s);
SerDe.Write(s, command.Length);
SerDe.Write(s, command);
for (int i = 0; i < 100; i++)
SerDe.Write(s, i.ToString());
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
int count = 0;
foreach (var bytes in ReadWorker(s))
{
Assert.AreEqual(count++.ToString(), Encoding.UTF8.GetString(bytes));
}
Assert.AreEqual(100, count);
}
}
using (Stream s = worker.StandardInput.BaseStream)
{
SerDe.Write(s, taskRunnerIds[0]);
s.Flush();
Thread.Sleep(200);
sockets[0].Close();
Thread.Sleep(200);
SerDe.Write(s, taskRunnerIds[1]);
s.Flush();
Thread.Sleep(200);
sockets[1].Close();
SerDe.Write(s, (byte)1); // use -1 to stop the server
s.Flush();
}
AssertWorker(worker, -1, "try to stop TaskRunner");
}
protected abstract RequestHandler CreateFromList(ISocketWrapper socket, RequestHandler.HttpHeader header, GdlList list);
/// <summary>
/// Listen to the server socket and accept new TCP connection from JVM side. Then create new TaskRunner instance and
/// add it to waitingTaskRunners queue.
/// </summary>
private void StartDaemonServer(ISocketWrapper listener)
{
logger.LogInfo("StartDaemonServer ...");
bool sparkReuseWorker = false;
string envVar = Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"); // this envVar is set in JVM side
if ((envVar != null) && envVar.Equals("1"))
{
sparkReuseWorker = true;
}
try
{
int trId = 1;
int workThreadNum = 0;
while (true)
{
var socket = listener.Accept();
logger.LogInfo("Connection accepted for taskRunnerId: {0}", trId);
using (var s = socket.GetStream())
{
SerDe.Write(s, trId); // write taskRunnerId to JVM side
s.Flush();
}
TaskRunner taskRunner = new TaskRunner(trId, socket, sparkReuseWorker);
waitingTaskRunners.Add(taskRunner);
taskRunnerRegistry[trId] = taskRunner;
trId++;
int taskRunnerNum = taskRunnerRegistry.Count();
while (workThreadNum < taskRunnerNum) // launch new work thread as appropriate
{
// start threads that do the actual work of running tasks, there are several options here:
// Option 1. TPL - Task Parallel Library
// Option 2. ThreadPool
// Option 3. Self managed threads group
// Option 3 is selected after testing in real cluster because it can get the best performance.
// When using option 1 or 2, it is observered that the boot time may be as large as 50 ~ 60s.
// But it is always less than 1s for option 3. Perhaps this is because TPL and ThreadPool are not
// suitable for long running threads.
new Thread(FetchAndRun).Start();
workThreadNum++;
}
}
}
catch (Exception e)
{
logger.LogError("StartDaemonServer exception, will exit");
logger.LogException(e);
Environment.Exit(-1);
}
}
public void TestInitialize()
{
sc = new SparkContext(null);
sc.StartAccumulatorServer();
// get accumulator server port and connect to accumuator server
int serverPort = (sc.SparkContextProxy as MockSparkContextProxy).AccumulatorServerPort;
sock = SocketFactory.CreateSocket();
sock.Connect(IPAddress.Loopback, serverPort);
}
private void SocketTest(ISocketWrapper serverSocket)
{
serverSocket.Listen();
if (serverSocket is RioSocketWrapper)
{
// Do nothing for second listen operation.
Assert.DoesNotThrow(() => serverSocket.Listen(int.MaxValue));
}
var port = ((IPEndPoint)serverSocket.LocalEndPoint).Port;
var clientMsg = "Hello Message from client";
var clientMsgBytes = Encoding.UTF8.GetBytes(clientMsg);
Task.Run(() =>
{
var bytes = new byte[1024];
using (var socket = serverSocket.Accept())
{
using (var s = socket.GetStream())
{
// Receive data
var bytesRec = s.Read(bytes, 0, bytes.Length);
// send echo message.
s.Write(bytes, 0, bytesRec);
s.Flush();
// Receive one byte
var oneByte = s.ReadByte();
// Send echo one byte
byte[] oneBytes = { (byte)oneByte };
s.Write(oneBytes, 0, oneBytes.Length);
Thread.SpinWait(0);
// Keep sending to ensure no memory leak
var longBytes = Encoding.UTF8.GetBytes(new string('x', 8192));
for (int i = 0; i < 1000; i++)
{
s.Write(longBytes, 0, longBytes.Length);
}
byte[] msg = Encoding.ASCII.GetBytes("This is a test<EOF>");
s.Write(msg, 0, msg.Length);
// Receive echo byte.
s.ReadByte();
}
}
});
var clientSock = SocketFactory.CreateSocket();
// Valid invalid operation
Assert.Throws<InvalidOperationException>(() => clientSock.GetStream());
Assert.Throws<InvalidOperationException>(() => clientSock.Receive());
Assert.Throws<InvalidOperationException>(() => clientSock.Send(null));
Assert.Throws<SocketException>(() => clientSock.Connect(IPAddress.Any, 1024));
clientSock.Connect(IPAddress.Loopback, port);
// Valid invalid operation
var byteBuf = ByteBufPool.Default.Allocate();
Assert.Throws<ArgumentException>(() => clientSock.Send(byteBuf));
byteBuf.Release();
Assert.Throws<SocketException>(() => clientSock.Listen());
if (clientSock is RioSocketWrapper)
{
Assert.Throws<InvalidOperationException>(() => clientSock.Accept());
}
using (var s = clientSock.GetStream())
{
// Send message
s.Write(clientMsgBytes, 0, clientMsgBytes.Length);
// Receive echo message
var bytes = new byte[1024];
var bytesRec = s.Read(bytes, 0, bytes.Length);
Assert.AreEqual(clientMsgBytes.Length, bytesRec);
var recvStr = Encoding.UTF8.GetString(bytes, 0, bytesRec);
Assert.AreEqual(clientMsg, recvStr);
// Send one byte
byte[] oneBytes = { 1 };
s.Write(oneBytes, 0, oneBytes.Length);
// Receive echo message
var oneByte = s.ReadByte();
Assert.AreEqual((byte)1, oneByte);
// Keep receiving to ensure no memory leak.
while (true)
{
bytesRec = s.Read(bytes, 0, bytes.Length);
recvStr = Encoding.UTF8.GetString(bytes, 0, bytesRec);
if (recvStr.IndexOf("<EOF>", StringComparison.OrdinalIgnoreCase) > -1)
{
break;
}
}
// send echo bytes
s.Write(oneBytes, 0, oneBytes.Length);
}
clientSock.Close();
// Verify invalid operation
Assert.Throws<ObjectDisposedException>(() => clientSock.Receive());
serverSocket.Close();
}
public InfoRequestHandler(ISocketWrapper socket, HttpHeader header, string gameId)
: base(socket, header, gameId)
{
}