private static void CreateAndVerifyConnection(ISocketWrapper daemonSocket) { var ipEndpoint = (IPEndPoint)daemonSocket.LocalEndPoint; int port = ipEndpoint.Port; ISocketWrapper clientSocket = SocketFactory.CreateSocket(); clientSocket.Connect(ipEndpoint.Address, port); // Now process the bytes flowing in from the client. PayloadWriter payloadWriter = new PayloadWriterFactory().Create(); payloadWriter.WriteTestData(clientSocket.OutputStream); List <object[]> rowsReceived = PayloadReader.Read(clientSocket.InputStream); // Validate rows received. Assert.Equal(10, rowsReceived.Count); for (int i = 0; i < 10; ++i) { // Two UDFs registered, thus expecting two columns. // Refer to TestData.GetDefaultCommandPayload(). object[] row = rowsReceived[i]; Assert.Equal(2, rowsReceived[i].Length); Assert.Equal($"udf2 udf1 {i}", row[0]); Assert.Equal(i + i, row[1]); } }
public void TestCallbackServer() { var callbackServer = new CallbackServer(_mockJvm.Object, false); var callbackHandler = new TestCallbackHandler(); callbackHandler.Id = callbackServer.RegisterCallback(callbackHandler); Assert.Equal(1, callbackHandler.Id); using ISocketWrapper callbackSocket = SocketFactory.CreateSocket(); callbackServer.Run(callbackSocket); int connectionNumber = 2; var clientSockets = new ISocketWrapper[connectionNumber]; for (int i = 0; i < connectionNumber; ++i) { var ipEndpoint = (IPEndPoint)callbackSocket.LocalEndPoint; ISocketWrapper clientSocket = SocketFactory.CreateSocket(); clientSockets[i] = clientSocket; clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port); WriteAndReadTestData(clientSocket, callbackHandler, i); } Assert.Equal(connectionNumber, callbackServer.CurrentNumConnections); IOrderedEnumerable <int> actualValues = callbackHandler.Inputs.OrderBy(i => i); IEnumerable <int> expectedValues = Enumerable .Range(0, connectionNumber) .Select(i => callbackHandler.Apply(i)) .OrderBy(i => i); Assert.True(expectedValues.SequenceEqual(actualValues)); }
internal void Run(int port) { try { string secret = Utils.SettingUtils.GetWorkerFactorySecret(); s_logger.LogInfo($"RunSimpleWorker() is starting with port = {port}."); ISocketWrapper socket = SocketFactory.CreateSocket(); socket.Connect(IPAddress.Loopback, port, secret); if ((_version.Major == 3 && _version.Minor >= 2) || _version.Major > 3) { int pid = Process.GetCurrentProcess().Id; SerDe.Write(socket.OutputStream, pid); socket.OutputStream.Flush(); } new TaskRunner(0, socket, false, _version).Run(); } catch (Exception e) { s_logger.LogError("RunSimpleWorker() failed with exception:"); s_logger.LogException(e); Environment.Exit(-1); } s_logger.LogInfo("RunSimpleWorker() finished successfully"); }
/// <summary> /// Reads the given stream to construct a BroadcastVariables object. /// </summary> /// <param name="stream">The stream to read from</param> /// <returns>BroadcastVariables object</returns> internal BroadcastVariables Process(Stream stream) { var broadcastVars = new BroadcastVariables(); ISocketWrapper socket = null; broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream); broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0); if (broadcastVars.DecryptionServerNeeded) { broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream); broadcastVars.Secret = SerDe.ReadString(stream); if (broadcastVars.Count > 0) { socket = SocketFactory.CreateSocket(); socket.Connect( IPAddress.Loopback, broadcastVars.DecryptionServerPort, broadcastVars.Secret); } } var formatter = new BinaryFormatter(); for (int i = 0; i < broadcastVars.Count; ++i) { long bid = SerDe.ReadInt64(stream); if (bid >= 0) { if (broadcastVars.DecryptionServerNeeded) { long readBid = SerDe.ReadInt64(socket.InputStream); if (bid != readBid) { throw new Exception("The Broadcast Id received from the encryption " + $"server {readBid} is different from the Broadcast Id received " + $"from the payload {bid}."); } object value = formatter.Deserialize(socket.InputStream); BroadcastRegistry.Add(bid, value); } else { string path = SerDe.ReadString(stream); using FileStream fStream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); object value = formatter.Deserialize(fStream); BroadcastRegistry.Add(bid, value); } } else { bid = -bid - 1; BroadcastRegistry.Remove(bid); } } socket?.Dispose(); return(broadcastVars); }
public void TestInitialize() { sc = new SparkContext(null); sc.StartAccumulatorServer(); // get accumulator server port and connect to accumuator server int serverPort = (sc.SparkContextProxy as MockSparkContextProxy).AccumulatorServerPort; sock = SocketFactory.CreateSocket(); sock.Connect(IPAddress.Loopback, serverPort, null); }
/// <summary> /// Returns row objects based on the function (either "toPythonIterator" or /// "collectToPython"). /// </summary> /// <param name="funcName"></param> /// <returns></returns> private IEnumerable <Row> GetRows(string funcName) { (int port, string secret) = GetConnectionInfo(funcName); using (ISocketWrapper socket = SocketFactory.CreateSocket()) { socket.Connect(IPAddress.Loopback, port, secret); foreach (Row row in new RowCollector().Collect(socket)) { yield return(row); } } }
/// <summary> /// Return an enumerable collection that contains all of the elements in this RDD. /// </summary> /// <remarks> /// This method should only be used if the resulting array is expected to be small, /// as all the data is loaded into the driver's memory. /// </remarks> /// <returns>An enumerable collection of all the elements.</returns> public IEnumerable <T> Collect() { (int port, string secret) = CollectAndServe(); using ISocketWrapper socket = SocketFactory.CreateSocket(); socket.Connect(IPAddress.Loopback, port, secret); var collector = new RDD.Collector(); System.IO.Stream stream = socket.InputStream; foreach (T element in collector.Collect(stream, _serializedMode).Cast <T>()) { yield return(element); } }
private void TestCallbackConnection( ConcurrentDictionary <int, ICallbackHandler> callbackHandlersDict, ITestCallbackHandler callbackHandler, int inputToHandler, CancellationToken token) { using ISocketWrapper serverListener = SocketFactory.CreateSocket(); serverListener.Listen(); var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint; using ISocketWrapper clientSocket = SocketFactory.CreateSocket(); clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port); // Don't use "using" here. The CallbackConnection will dispose the socket. ISocketWrapper serverSocket = serverListener.Accept(); var callbackConnection = new CallbackConnection(0, serverSocket, callbackHandlersDict); Task task = Task.Run(() => callbackConnection.Run(token)); if (token.IsCancellationRequested) { task.Wait(); Assert.False(callbackConnection.IsRunning); } else { WriteAndReadTestData(clientSocket, callbackHandler, inputToHandler); if (callbackHandler.Throws) { task.Wait(); Assert.False(callbackConnection.IsRunning); } else { Assert.True(callbackConnection.IsRunning); // Clean up CallbackConnection Stream outputStream = clientSocket.OutputStream; SerDe.Write(outputStream, (int)CallbackConnection.ConnectionStatus.REQUEST_CLOSE); outputStream.Flush(); task.Wait(); Assert.False(callbackConnection.IsRunning); } } }
private void TestCallbackConnection( ConcurrentDictionary <int, ICallbackHandler> callbackHandlersDict, ITestCallbackHandler callbackHandler, int inputToHandler, CancellationToken token) { using ISocketWrapper serverListener = SocketFactory.CreateSocket(); serverListener.Listen(); var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint; ISocketWrapper clientSocket = SocketFactory.CreateSocket(); clientSocket.Connect(ipEndpoint.Address, ipEndpoint.Port); var callbackConnection = new CallbackConnection(0, clientSocket, callbackHandlersDict); Task.Run(() => callbackConnection.Run(token)); using ISocketWrapper serverSocket = serverListener.Accept(); WriteAndReadTestData(serverSocket, callbackHandler, inputToHandler, token); }
internal void Run(int port) { try { string secret = Utils.SettingUtils.GetWorkerFactorySecret(); s_logger.LogInfo($"RunSimpleWorker() is starting with port = {port}."); ISocketWrapper socket = SocketFactory.CreateSocket(); socket.Connect(IPAddress.Loopback, port, secret); new TaskRunner(0, socket, false, _version).Run(); } catch (Exception e) { s_logger.LogError("RunSimpleWorker() failed with exception:"); s_logger.LogException(e); Environment.Exit(-1); } s_logger.LogInfo("RunSimpleWorker() finished successfully"); }
public void TestInitialize() { sc = new SparkContext(null); sc.StartAccumulatorServer(); // get accumulator server port and connect to accumuator server int serverPort = (sc.SparkContextProxy as MockSparkContextProxy).AccumulatorServerPort; sock = SocketFactory.CreateSocket(); sock.Connect(IPAddress.Loopback, serverPort); }