private static void CreateAndVerifyConnection(ISocketWrapper daemonSocket)
{
var ipEndpoint = (IPEndPoint)daemonSocket.LocalEndPoint;
int port = ipEndpoint.Port;
ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSocket.Connect(ipEndpoint.Address, port);
// Now process the bytes flowing in from the client.
PayloadWriter payloadWriter = new PayloadWriterFactory().Create();
payloadWriter.WriteTestData(clientSocket.OutputStream);
List <object[]> rowsReceived = PayloadReader.Read(clientSocket.InputStream);
// Validate rows received.
Assert.Equal(10, rowsReceived.Count);
for (int i = 0; i < 10; ++i)
{
// Two UDFs registered, thus expecting two columns.
// Refer to TestData.GetDefaultCommandPayload().
object[] row = rowsReceived[i];
Assert.Equal(2, rowsReceived[i].Length);
Assert.Equal($"udf2 udf1 {i}", row[0]);
Assert.Equal(i + i, row[1]);
}
}
public void TestCallbackServer()
{
var callbackServer = new CallbackServer(_mockJvm.Object, false);
var callbackHandler = new TestCallbackHandler();
callbackHandler.Id = callbackServer.RegisterCallback(callbackHandler);
Assert.Equal(1, callbackHandler.Id);
using ISocketWrapper callbackSocket = SocketFactory.CreateSocket();
callbackServer.Run(callbackSocket);
int connectionNumber = 2;
var clientSockets = new ISocketWrapper[connectionNumber];
for (int i = 0; i < connectionNumber; ++i)
{
var ipEndpoint = (IPEndPoint)callbackSocket.LocalEndPoint;
ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSockets[i] = clientSocket;
clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port);
WriteAndReadTestData(clientSocket, callbackHandler, i);
}
Assert.Equal(connectionNumber, callbackServer.CurrentNumConnections);
IOrderedEnumerable <int> actualValues = callbackHandler.Inputs.OrderBy(i => i);
IEnumerable <int> expectedValues = Enumerable
.Range(0, connectionNumber)
.Select(i => callbackHandler.Apply(i))
.OrderBy(i => i);
Assert.True(expectedValues.SequenceEqual(actualValues));
}
internal void Run(int port)
{
try
{
string secret = Utils.SettingUtils.GetWorkerFactorySecret();
s_logger.LogInfo($"RunSimpleWorker() is starting with port = {port}.");
ISocketWrapper socket = SocketFactory.CreateSocket();
socket.Connect(IPAddress.Loopback, port, secret);
if ((_version.Major == 3 && _version.Minor >= 2) || _version.Major > 3)
{
int pid = Process.GetCurrentProcess().Id;
SerDe.Write(socket.OutputStream, pid);
socket.OutputStream.Flush();
}
new TaskRunner(0, socket, false, _version).Run();
}
catch (Exception e)
{
s_logger.LogError("RunSimpleWorker() failed with exception:");
s_logger.LogException(e);
Environment.Exit(-1);
}
s_logger.LogInfo("RunSimpleWorker() finished successfully");
}
/// <summary>
/// Reads the given stream to construct a BroadcastVariables object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>BroadcastVariables object</returns>
internal BroadcastVariables Process(Stream stream)
{
var broadcastVars = new BroadcastVariables();
ISocketWrapper socket = null;
broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream);
broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0);
if (broadcastVars.DecryptionServerNeeded)
{
broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream);
broadcastVars.Secret = SerDe.ReadString(stream);
if (broadcastVars.Count > 0)
{
socket = SocketFactory.CreateSocket();
socket.Connect(
IPAddress.Loopback,
broadcastVars.DecryptionServerPort,
broadcastVars.Secret);
}
}
var formatter = new BinaryFormatter();
for (int i = 0; i < broadcastVars.Count; ++i)
{
long bid = SerDe.ReadInt64(stream);
if (bid >= 0)
{
if (broadcastVars.DecryptionServerNeeded)
{
long readBid = SerDe.ReadInt64(socket.InputStream);
if (bid != readBid)
{
throw new Exception("The Broadcast Id received from the encryption " +
$"server {readBid} is different from the Broadcast Id received " +
$"from the payload {bid}.");
}
object value = formatter.Deserialize(socket.InputStream);
BroadcastRegistry.Add(bid, value);
}
else
{
string path = SerDe.ReadString(stream);
using FileStream fStream =
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
object value = formatter.Deserialize(fStream);
BroadcastRegistry.Add(bid, value);
}
}
else
{
bid = -bid - 1;
BroadcastRegistry.Remove(bid);
}
}
socket?.Dispose();
return(broadcastVars);
}
public void TestInitialize()
{
sc = new SparkContext(null);
sc.StartAccumulatorServer();
// get accumulator server port and connect to accumuator server
int serverPort = (sc.SparkContextProxy as MockSparkContextProxy).AccumulatorServerPort;
sock = SocketFactory.CreateSocket();
sock.Connect(IPAddress.Loopback, serverPort, null);
}
/// <summary>
/// Returns row objects based on the function (either "toPythonIterator" or
/// "collectToPython").
/// </summary>
/// <param name="funcName"></param>
/// <returns></returns>
private IEnumerable <Row> GetRows(string funcName)
{
(int port, string secret) = GetConnectionInfo(funcName);
using (ISocketWrapper socket = SocketFactory.CreateSocket())
{
socket.Connect(IPAddress.Loopback, port, secret);
foreach (Row row in new RowCollector().Collect(socket))
{
yield return(row);
}
}
}
/// <summary>
/// Return an enumerable collection that contains all of the elements in this RDD.
/// </summary>
/// <remarks>
/// This method should only be used if the resulting array is expected to be small,
/// as all the data is loaded into the driver's memory.
/// </remarks>
/// <returns>An enumerable collection of all the elements.</returns>
public IEnumerable <T> Collect()
{
(int port, string secret) = CollectAndServe();
using ISocketWrapper socket = SocketFactory.CreateSocket();
socket.Connect(IPAddress.Loopback, port, secret);
var collector = new RDD.Collector();
System.IO.Stream stream = socket.InputStream;
foreach (T element in collector.Collect(stream, _serializedMode).Cast <T>())
{
yield return(element);
}
}
private void TestCallbackConnection(
ConcurrentDictionary <int, ICallbackHandler> callbackHandlersDict,
ITestCallbackHandler callbackHandler,
int inputToHandler,
CancellationToken token)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
serverListener.Listen();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
using ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port);
// Don't use "using" here. The CallbackConnection will dispose the socket.
ISocketWrapper serverSocket = serverListener.Accept();
var callbackConnection = new CallbackConnection(0, serverSocket, callbackHandlersDict);
Task task = Task.Run(() => callbackConnection.Run(token));
if (token.IsCancellationRequested)
{
task.Wait();
Assert.False(callbackConnection.IsRunning);
}
else
{
WriteAndReadTestData(clientSocket, callbackHandler, inputToHandler);
if (callbackHandler.Throws)
{
task.Wait();
Assert.False(callbackConnection.IsRunning);
}
else
{
Assert.True(callbackConnection.IsRunning);
// Clean up CallbackConnection
Stream outputStream = clientSocket.OutputStream;
SerDe.Write(outputStream, (int)CallbackConnection.ConnectionStatus.REQUEST_CLOSE);
outputStream.Flush();
task.Wait();
Assert.False(callbackConnection.IsRunning);
}
}
}
private void TestCallbackConnection(
ConcurrentDictionary <int, ICallbackHandler> callbackHandlersDict,
ITestCallbackHandler callbackHandler,
int inputToHandler,
CancellationToken token)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
serverListener.Listen();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
ISocketWrapper clientSocket = SocketFactory.CreateSocket();
clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port);
var callbackConnection = new CallbackConnection(0, clientSocket, callbackHandlersDict);
Task.Run(() => callbackConnection.Run(token));
using ISocketWrapper serverSocket = serverListener.Accept();
WriteAndReadTestData(serverSocket, callbackHandler, inputToHandler, token);
}
internal void Run(int port)
{
try
{
string secret = Utils.SettingUtils.GetWorkerFactorySecret();
s_logger.LogInfo($"RunSimpleWorker() is starting with port = {port}.");
ISocketWrapper socket = SocketFactory.CreateSocket();
socket.Connect(IPAddress.Loopback, port, secret);
new TaskRunner(0, socket, false, _version).Run();
}
catch (Exception e)
{
s_logger.LogError("RunSimpleWorker() failed with exception:");
s_logger.LogException(e);
Environment.Exit(-1);
}
s_logger.LogInfo("RunSimpleWorker() finished successfully");
}
public void TestInitialize()
{
sc = new SparkContext(null);
sc.StartAccumulatorServer();
// get accumulator server port and connect to accumuator server
int serverPort = (sc.SparkContextProxy as MockSparkContextProxy).AccumulatorServerPort;
sock = SocketFactory.CreateSocket();
sock.Connect(IPAddress.Loopback, serverPort);
}
