internal static object ReadTypedObject(Stream s, char type)
{
switch (type)
{
case 'n':
return(null);
case 'i':
return(SerDe.ReadInt(s));
case 'g':
return(SerDe.ReadLong(s));
case 'd':
return(SerDe.ReadDouble(s));
case 'b':
return(ReadBoolean(s));
case 'c':
return(SerDe.ReadString(s));
case 'e':
return(ReadDictionary(s));
case 'r':
return(SerDe.ReadBytes(s));
case 'l':
return(ReadArray(s));
case 'D':
return(ReadDate(s));
case 't':
return(ReadTime(s));
case 'j':
return(new JvmObjectReference(SerDe.ReadString(s)));
default:
throw new ArgumentException("Invalid type " + type);
}
}
internal int StartUpdateServer()
{
base.Start();
Task.Run(() =>
{
try
{
IFormatter formatter = new BinaryFormatter();
using (Socket s = AcceptSocket())
using (var ns = new NetworkStream(s))
{
while (!serverShutdown)
{
int numUpdates = SerDe.ReadInt(ns);
for (int i = 0; i < numUpdates; i++)
{
var ms = new MemoryStream(SerDe.ReadBytes(ns));
KeyValuePair <int, dynamic> update = (KeyValuePair <int, dynamic>)formatter.Deserialize(ms);
Accumulator accumulator = Accumulator.accumulatorRegistry[update.Key];
accumulator.GetType().GetMethod("Add").Invoke(accumulator, new object[] { update.Value });
}
ns.WriteByte((byte)1); // acknowledge byte other than -1
ns.Flush();
Thread.Sleep(1000);
}
}
}
catch (SocketException e)
{
if (e.ErrorCode != 10004) // A blocking operation was interrupted by a call to WSACancelBlockingCall - TcpListener.Stop cancelled AccepSocket as expected
{
throw e;
}
}
catch (Exception e)
{
logger.LogError(e.ToString());
throw;
}
});
return((base.LocalEndpoint as IPEndPoint).Port);
}
public void TestSerDeWithPythonSerDe()
{
const int expectedCount = 5;
using (var ms = new MemoryStream())
{
new StructTypePickler().Register();
new RowPickler().Register();
var pickler = new Pickler();
for (int i = 0; i < expectedCount; i++)
{
var pickleBytes = pickler.dumps(new[] { RowHelper.BuildRowForBasicSchema(i) });
SerDe.Write(ms, pickleBytes.Length);
SerDe.Write(ms, pickleBytes);
}
SerDe.Write(ms, (int)SpecialLengths.END_OF_STREAM);
ms.Flush();
ms.Position = 0;
int count = 0;
while (true)
{
byte[] outBuffer = null;
int length = SerDe.ReadInt(ms);
if (length > 0)
{
outBuffer = SerDe.ReadBytes(ms, length);
}
else if (length == (int)SpecialLengths.END_OF_STREAM)
{
break;
}
var unpickledObjs = PythonSerDe.GetUnpickledObjects(outBuffer);
var rows = unpickledObjs.Select(item => (item as RowConstructor).GetRow()).ToList();
Assert.AreEqual(1, rows.Count);
Assert.AreEqual(count++, rows[0].Get("age"));
}
Assert.AreEqual(expectedCount, count);
}
}
/// <summary>
/// Processes the given stream to construct a Payload object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>
/// Returns a valid payload object if the stream contains all the necessary data.
/// Returns null if the stream is already closed at the beginning of the read.
/// </returns>
internal Payload Process(Stream stream)
{
var payload = new Payload();
byte[] splitIndexBytes = null;
try
{
splitIndexBytes = SerDe.ReadBytes(stream, sizeof(int));
// For socket stream, read on the stream returns 0, which
// SerDe.ReadBytes() returns as null to denote the stream is closed.
if (splitIndexBytes == null)
{
return(null);
}
}
catch (ObjectDisposedException)
{
// For stream implementation such as MemoryStream will throw
// ObjectDisposedException if the stream is already closed.
return(null);
}
payload.SplitIndex = BinaryPrimitives.ReadInt32BigEndian(splitIndexBytes);
payload.Version = SerDe.ReadString(stream);
payload.TaskContext = new TaskContextProcessor(_version).Process(stream);
payload.SparkFilesDir = SerDe.ReadString(stream);
if (Utils.SettingUtils.IsDatabricks)
{
SerDe.ReadString(stream);
SerDe.ReadString(stream);
}
payload.IncludeItems = ReadIncludeItems(stream);
payload.BroadcastVariables = new BroadcastVariableProcessor(_version).Process(stream);
// TODO: Accumulate registration should be done here.
payload.Command = new CommandProcessor(_version).Process(stream);
return(payload);
}
private static void ReadCommand(Stream networkStream, IFormatter formatter, out int stageId,
out string deserializerMode,
out string serializerMode, out CSharpWorkerFunc workerFunc)
{
stageId = ReadDiagnosticsInfo(networkStream);
deserializerMode = SerDe.ReadString(networkStream);
logger.LogDebug("Deserializer mode: " + deserializerMode);
serializerMode = SerDe.ReadString(networkStream);
logger.LogDebug("Serializer mode: " + serializerMode);
string runMode = SerDe.ReadString(networkStream);
if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase))
{
var compilationDumpDir = SerDe.ReadString(networkStream);
if (Directory.Exists(compilationDumpDir))
{
assemblyHandler.LoadAssemblies(Directory.GetFiles(compilationDumpDir, "ReplCompilation.*",
SearchOption.TopDirectoryOnly));
}
else
{
logger.LogError("Directory " + compilationDumpDir + " dose not exist.");
}
}
byte[] command = SerDe.ReadBytes(networkStream);
logger.LogDebug("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
workerFunc = (CSharpWorkerFunc)formatter.Deserialize(stream);
logger.LogDebug(
"------------------------ Printing stack trace of workerFunc for ** debugging ** ------------------------------");
logger.LogDebug(workerFunc.StackTrace);
logger.LogDebug(
"--------------------------------------------------------------------------------------------------------------");
}
public IEnumerable <dynamic> Collect(int port, SerializedMode serializedMode, Type type)
{
IFormatter formatter = new BinaryFormatter();
var sock = SocketFactory.CreateSocket();
sock.Connect(IPAddress.Loopback, port);
using (var s = sock.GetStream())
{
byte[] buffer;
while ((buffer = SerDe.ReadBytes(s)) != null && buffer.Length > 0)
{
if (serializedMode == SerializedMode.Byte)
{
MemoryStream ms = new MemoryStream(buffer);
yield return(formatter.Deserialize(ms));
}
else if (serializedMode == SerializedMode.String)
{
yield return(Encoding.UTF8.GetString(buffer));
}
else if (serializedMode == SerializedMode.Pair)
{
MemoryStream ms = new MemoryStream(buffer);
MemoryStream ms2 = new MemoryStream(SerDe.ReadBytes(s));
ConstructorInfo ci = type.GetConstructors()[0];
yield return(ci.Invoke(new object[] { formatter.Deserialize(ms), formatter.Deserialize(ms2) }));
}
else if (serializedMode == SerializedMode.Row)
{
var unpickledObjects = PythonSerDe.GetUnpickledObjects(buffer);
foreach (var item in unpickledObjects)
{
yield return((item as RowConstructor).GetRow());
}
}
}
}
}
/// <summary>
/// read data from worker
/// </summary>
/// <param name="s"></param>
/// <param name="expectedCount"></param>
/// <returns></returns>
private IEnumerable <byte[]> ReadWorker(Stream s, int expectedCount = 0)
{
int count = 0;
while (true)
{
int length = SerDe.ReadInt(s);
if (length > 0)
{
yield return(SerDe.ReadBytes(s, length));
if (expectedCount > 0 && ++count >= expectedCount)
{
break;
}
}
else if (length == (int)SpecialLengths.TIMING_DATA)
{
var bootTime = SerDe.ReadLong(s);
var initTime = SerDe.ReadLong(s);
var finishTime = SerDe.ReadLong(s);
var memoryBytesSpilled = SerDe.ReadLong(s);
var diskBytesSpilled = SerDe.ReadLong(s);
}
else if (length == (int)SpecialLengths.DOTNET_EXCEPTION_THROWN)
{
SerDe.ReadString(s);
break;
}
else if (length == (int)SpecialLengths.END_OF_DATA_SECTION)
{
var numAccumulatorUpdates = SerDe.ReadInt(s);
SerDe.ReadInt(s);
break;
}
}
}
/// <summary>
/// read accumulator
/// </summary>
private IEnumerable <KeyValuePair <int, dynamic> > ReadAccumulator(Stream s, int expectedCount = 0)
{
int count = 0;
var formatter = new BinaryFormatter();
while (true)
{
int length = SerDe.ReadInt(s);
if (length > 0)
{
var ms = new MemoryStream(SerDe.ReadBytes(s, length));
yield return((KeyValuePair <int, dynamic>)formatter.Deserialize(ms));
if (expectedCount > 0 && ++count >= expectedCount)
{
break;
}
}
else if (length == (int)SpecialLengths.END_OF_STREAM)
{
break;
}
}
}
public void Run()
{
s_logger.LogInfo($"[{TaskId}] Starting with ReuseSocket[{_reuseSocket}].");
if (EnvironmentUtils.GetEnvironmentVariableAsBool("DOTNET_WORKER_DEBUG"))
{
Debugger.Launch();
}
_isRunning = true;
Stream inputStream = _socket.InputStream;
Stream outputStream = _socket.OutputStream;
try
{
while (_isRunning)
{
Payload payload = ProcessStream(
inputStream,
outputStream,
_version,
out bool readComplete);
if (payload != null)
{
outputStream.Flush();
++_numTasksRun;
// If the socket is not read through completely, then it cannot be reused.
if (!readComplete)
{
_isRunning = false;
// Wait for server to complete to avoid 'connection reset' exception.
s_logger.LogInfo($"[{TaskId}] Sleep 500 millisecond to close socket.");
Thread.Sleep(500);
}
else if (!_reuseSocket)
{
_isRunning = false;
// Use SerDe.ReadBytes() to detect Java side has closed socket
// properly. SerDe.ReadBytes() will block until the socket is closed.
s_logger.LogInfo($"[{TaskId}] Waiting for JVM side to close socket.");
SerDe.ReadBytes(inputStream);
s_logger.LogInfo($"[{TaskId}] JVM side has closed socket.");
}
}
else
{
_isRunning = false;
s_logger.LogWarn(
$"[{TaskId}] Read null payload. Socket is closed by JVM.");
}
}
}
catch (Exception e)
{
_isRunning = false;
s_logger.LogError($"[{TaskId}] Exiting with exception: {e}");
}
finally
{
try
{
_socket.Dispose();
}
catch (Exception ex)
{
s_logger.LogWarn($"[{TaskId}] Exception while closing socket: {ex}");
}
s_logger.LogInfo($"[{TaskId}] Finished running {_numTasksRun} task(s).");
}
}
internal static char ReadObjectType(Stream s)
{
return(SerDe.ToChar(SerDe.ReadBytes(s, 1)[0]));
}
private void ProcessCallbackRequest(object socket)
{
logger.LogInfo("new thread created to process callback request");
try
{
using (Socket sock = (Socket)socket)
using (var s = new NetworkStream(sock))
{
while (true)
{
try
{
string cmd = SerDe.ReadString(s);
if (cmd == "close")
{
logger.LogInfo("receive close cmd from Scala side");
break;
}
else if (cmd == "callback")
{
int numRDDs = SerDe.ReadInt(s);
var jrdds = new List <JvmObjectReference>();
for (int i = 0; i < numRDDs; i++)
{
jrdds.Add(new JvmObjectReference(SerDe.ReadObjectId(s)));
}
double time = SerDe.ReadDouble(s);
IFormatter formatter = new BinaryFormatter();
object func = formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
string deserializer = SerDe.ReadString(s);
RDD <dynamic> rdd = null;
if (jrdds[0].Id != null)
{
rdd = new RDD <dynamic>(new RDDIpcProxy(jrdds[0]), sparkContext, (SerializedMode)Enum.Parse(typeof(SerializedMode), deserializer));
}
if (func is Func <double, RDD <dynamic>, RDD <dynamic> > )
{
JvmObjectReference jrdd = (((Func <double, RDD <dynamic>, RDD <dynamic> >)func)(time, rdd).RddProxy as RDDIpcProxy).JvmRddReference;
SerDe.Write(s, (byte)'j');
SerDe.Write(s, jrdd.Id);
}
else if (func is Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > )
{
string deserializer2 = SerDe.ReadString(s);
RDD <dynamic> rdd2 = new RDD <dynamic>(new RDDIpcProxy(jrdds[1]), sparkContext, (SerializedMode)Enum.Parse(typeof(SerializedMode), deserializer2));
JvmObjectReference jrdd = (((Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)func)(time, rdd, rdd2).RddProxy as RDDIpcProxy).JvmRddReference;
SerDe.Write(s, (byte)'j');
SerDe.Write(s, jrdd.Id);
}
else
{
((Action <double, RDD <dynamic> >)func)(time, rdd);
SerDe.Write(s, (byte)'n');
}
}
}
catch (Exception e)
{
//log exception only when callback socket is not shutdown explicitly
if (!callbackSocketShutdown)
{
logger.LogException(e);
}
}
}
}
}
catch (Exception e)
{
logger.LogException(e);
}
logger.LogInfo("thread to process callback request exit");
}
private object[] GetNext(int messageLength)
{
object[] result = null;
switch ((SerializedMode)Enum.Parse(typeof(SerializedMode), deserializedMode))
{
case SerializedMode.String:
{
result = new object[1];
if (messageLength > 0)
{
byte[] buffer = SerDe.ReadBytes(inputStream, messageLength);
result[0] = SerDe.ToString(buffer);
}
else
{
result[0] = null;
}
break;
}
case SerializedMode.Row:
{
Debug.Assert(messageLength > 0);
byte[] buffer = SerDe.ReadBytes(inputStream, messageLength);
var unpickledObjects = PythonSerDe.GetUnpickledObjects(buffer);
var rows = unpickledObjects.Select(item => (item as RowConstructor).GetRow()).ToList();
result = rows.Cast <object>().ToArray();
break;
}
case SerializedMode.Pair:
{
byte[] pairKey = (messageLength > 0) ? SerDe.ReadBytes(inputStream, messageLength) : null;
byte[] pairValue = null;
int valueLength = SerDe.ReadInt(inputStream);
if (valueLength > 0)
{
pairValue = SerDe.ReadBytes(inputStream, valueLength);
}
else if (valueLength == (int)SpecialLengths.NULL)
{
pairValue = null;
}
else
{
throw new Exception(string.Format("unexpected valueLength: {0}", valueLength));
}
result = new object[1];
result[0] = new KeyValuePair <byte[], byte[]>(pairKey, pairValue);
break;
}
case SerializedMode.None: //just read raw bytes
{
result = new object[1];
if (messageLength > 0)
{
result[0] = SerDe.ReadBytes(inputStream, messageLength);
}
else
{
result[0] = null;
}
break;
}
case SerializedMode.Byte:
default:
{
result = new object[1];
if (messageLength > 0)
{
byte[] buffer = SerDe.ReadBytes(inputStream, messageLength);
var ms = new MemoryStream(buffer);
result[0] = formatter.Deserialize(ms);
}
else
{
result[0] = null;
}
break;
}
}
return(result);
}
public static int Run(Socket sock = null)
{
// if there exists exe.config file, then use log4net
if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile))
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance);
}
logger = LoggerServiceFactory.GetLogger(typeof(Worker));
if (sock == null)
{
try
{
PrintFiles();
int javaPort = int.Parse(Console.ReadLine());
logger.LogDebug("java_port: " + javaPort);
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(IPAddress.Loopback, javaPort);
}
catch (Exception e)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(e);
Environment.Exit(-1);
}
}
using (NetworkStream s = new NetworkStream(sock))
{
try
{
DateTime bootTime = DateTime.UtcNow;
int splitIndex = SerDe.ReadInt(s);
logger.LogDebug("split_index: " + splitIndex);
if (splitIndex == -1)
{
Environment.Exit(-1);
}
string ver = SerDe.ReadString(s);
logger.LogDebug("ver: " + ver);
//// initialize global state
//shuffle.MemoryBytesSpilled = 0
//shuffle.DiskBytesSpilled = 0
// fetch name of workdir
string sparkFilesDir = SerDe.ReadString(s);
logger.LogDebug("spark_files_dir: " + sparkFilesDir);
//SparkFiles._root_directory = sparkFilesDir
//SparkFiles._is_running_on_worker = True
// fetch names of includes - not used //TODO - complete the impl
int numberOfIncludesItems = SerDe.ReadInt(s);
logger.LogDebug("num_includes: " + numberOfIncludesItems);
if (numberOfIncludesItems > 0)
{
for (int i = 0; i < numberOfIncludesItems; i++)
{
string filename = SerDe.ReadString(s);
}
}
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(s);
logger.LogDebug("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(s);
if (bid >= 0)
{
string path = SerDe.ReadString(s);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast.broadcastRegistry.Remove(bid);
}
}
}
Accumulator.accumulatorRegistry.Clear();
int lengthOfCommandByteArray = SerDe.ReadInt(s);
logger.LogDebug("command length: " + lengthOfCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOfCommandByteArray > 0)
{
Stopwatch commandProcessWatch = new Stopwatch();
Stopwatch funcProcessWatch = new Stopwatch();
commandProcessWatch.Start();
int rddId = SerDe.ReadInt(s);
int stageId = SerDe.ReadInt(s);
int partitionId = SerDe.ReadInt(s);
logger.LogInfo(string.Format("rddInfo: rddId {0}, stageId {1}, partitionId {2}", rddId, stageId, partitionId));
string deserializerMode = SerDe.ReadString(s);
logger.LogDebug("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(s);
logger.LogDebug("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(s);
logger.LogDebug("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var workerFunc = (CSharpWorkerFunc)formatter.Deserialize(stream);
var func = workerFunc.Func;
//logger.LogDebug("------------------------ Printing stack trace of workerFunc for ** debugging ** ------------------------------");
//logger.LogDebug(workerFunc.StackTrace);
//logger.LogDebug("--------------------------------------------------------------------------------------------------------------");
DateTime initTime = DateTime.UtcNow;
int count = 0;
// here we use low level API because we need to get perf metrics
WorkerInputEnumerator inputEnumerator = new WorkerInputEnumerator(s, deserializerMode);
IEnumerable <dynamic> inputEnumerable = Enumerable.Cast <dynamic>(inputEnumerator);
funcProcessWatch.Start();
IEnumerable <dynamic> outputEnumerable = func(splitIndex, inputEnumerable);
var outputEnumerator = outputEnumerable.GetEnumerator();
funcProcessWatch.Stop();
while (true)
{
funcProcessWatch.Start();
bool hasNext = outputEnumerator.MoveNext();
funcProcessWatch.Stop();
if (!hasNext)
{
break;
}
funcProcessWatch.Start();
var message = outputEnumerator.Current;
funcProcessWatch.Stop();
if (object.ReferenceEquals(null, message))
{
continue;
}
byte[] buffer;
switch ((SerializedMode)Enum.Parse(typeof(SerializedMode), serializerMode))
{
case SerializedMode.None:
buffer = message as byte[];
break;
case SerializedMode.String:
buffer = SerDe.ToBytes(message as string);
break;
case SerializedMode.Row:
Pickler pickler = new Pickler();
buffer = pickler.dumps(new ArrayList {
message
});
break;
default:
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
logger.LogError("Exception serializing output");
logger.LogError("{0} : {1}", message.GetType().Name, message.GetType().FullName);
throw;
}
break;
}
count++;
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
//TODO - complete the impl
logger.LogDebug("Output entries count: " + count);
//if profiler:
// profiler.profile(process)
//else:
// process()
DateTime finish_time = DateTime.UtcNow;
const string format = "MM/dd/yyyy hh:mm:ss.fff tt";
logger.LogDebug(string.Format("bootTime: {0}, initTime: {1}, finish_time: {2}",
bootTime.ToString(format), initTime.ToString(format), finish_time.ToString(format)));
SerDe.Write(s, (int)SpecialLengths.TIMING_DATA);
SerDe.Write(s, ToUnixTime(bootTime));
SerDe.Write(s, ToUnixTime(initTime));
SerDe.Write(s, ToUnixTime(finish_time));
SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled
SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled
commandProcessWatch.Stop();
// log statistics
inputEnumerator.LogStatistic();
logger.LogInfo(string.Format("func process time: {0}", funcProcessWatch.ElapsedMilliseconds));
logger.LogInfo(string.Format("command process time: {0}", commandProcessWatch.ElapsedMilliseconds));
}
else
{
logger.LogWarn("lengthOfCommandByteArray = 0. Nothing to execute :-(");
}
// Mark the beginning of the accumulators section of the output
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, Accumulator.accumulatorRegistry.Count);
foreach (var item in Accumulator.accumulatorRegistry)
{
var ms = new MemoryStream();
var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value);
logger.LogDebug(string.Format("({0}, {1})", item.Key, value));
formatter.Serialize(ms, new KeyValuePair <int, dynamic>(item.Key, value));
byte[] buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
int end = SerDe.ReadInt(s);
// check end of stream
if (end == (int)SpecialLengths.END_OF_STREAM)
{
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
logger.LogDebug("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM);
}
else
{
// write a different value to tell JVM to not reuse this worker
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
Environment.Exit(-1);
}
s.Flush();
// log bytes read and write
logger.LogDebug(string.Format("total read bytes: {0}", SerDe.totalReadNum));
logger.LogDebug(string.Format("total write bytes: {0}", SerDe.totalWriteNum));
}
catch (Exception e)
{
logger.LogError(e.ToString());
try
{
SerDe.Write(s, e.ToString());
}
catch (IOException)
{
// JVM close the socket
}
catch (Exception ex)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(ex);
}
Environment.Exit(-1);
return(-1);
}
return(0);
}
}
/// <summary>
/// Starts listening to any connection from JVM.
/// </summary>
private void StartServer(ISocketWrapper listener)
{
try
{
bool reuseWorker =
"1".Equals(Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"));
string secret = Utils.SettingUtils.GetWorkerFactorySecret();
int taskRunnerId = 1;
int numWorkerThreads = 0;
while (true)
{
ISocketWrapper socket = listener.Accept();
s_logger.LogInfo($"New connection accepted for TaskRunner [{taskRunnerId}]");
bool authStatus = true;
if (!string.IsNullOrWhiteSpace(secret))
{
// The Spark side expects the PID from a forked process.
// In .NET implementation, a task runner id is used instead.
SerDe.Write(socket.OutputStream, taskRunnerId);
socket.OutputStream.Flush();
if (ConfigurationService.IsDatabricks)
{
SerDe.ReadString(socket.InputStream);
}
authStatus = Authenticator.AuthenticateAsServer(socket, secret);
}
if (authStatus)
{
var taskRunner = new TaskRunner(
taskRunnerId,
socket,
reuseWorker,
_version);
_waitingTaskRunners.Add(taskRunner);
_taskRunners[taskRunnerId] = taskRunner;
++taskRunnerId;
// When reuseWorker is set to true, numTaskRunners will be always one
// greater than numWorkerThreads since TaskRunner.Run() does not return
// so that the task runner object is not removed from _taskRunners.
int numTaskRunners = CurrentNumTaskRunners;
while (numWorkerThreads < numTaskRunners)
{
// Note that in the current implementation of RunWorkerThread() does
// not return. If more graceful exit is required, RunWorkerThread() can
// be updated to return upon receiving a signal from this main thread.
new Thread(RunWorkerThread).Start();
++numWorkerThreads;
}
s_logger.LogInfo(
$"Pool snapshot: [NumThreads:{numWorkerThreads}], [NumTaskRunners:{numTaskRunners}]");
}
else
{
// Use SerDe.ReadBytes() to detect Java side has closed socket
// properly. ReadBytes() will block until the socket is closed.
s_logger.LogError(
"Authentication failed. Waiting for JVM side to close socket.");
SerDe.ReadBytes(socket.InputStream);
socket.Dispose();
}
}
}
catch (Exception e)
{
s_logger.LogError($"StartServer() exits with exception: {e}");
Environment.Exit(-1);
}
}
public void TestTaskRunner()
{
using var serverListener = new DefaultSocketWrapper();
serverListener.Listen();
var port = (serverListener.LocalEndPoint as IPEndPoint).Port;
var clientSocket = new DefaultSocketWrapper();
clientSocket.Connect(IPAddress.Loopback, port, null);
PayloadWriter payloadWriter = new PayloadWriterFactory().Create();
var taskRunner = new TaskRunner(0, clientSocket, false, payloadWriter.Version);
var clientTask = Task.Run(() => taskRunner.Run());
using (ISocketWrapper serverSocket = serverListener.Accept())
{
System.IO.Stream inputStream = serverSocket.InputStream;
System.IO.Stream outputStream = serverSocket.OutputStream;
Payload payload = TestData.GetDefaultPayload();
CommandPayload commandPayload = TestData.GetDefaultCommandPayload();
payloadWriter.Write(outputStream, payload, commandPayload);
// Write 10 rows to the output stream.
var pickler = new Pickler();
for (int i = 0; i < 10; ++i)
{
var pickled = pickler.dumps(
new[] { new object[] { i.ToString(), i, i } });
SerDe.Write(outputStream, pickled.Length);
SerDe.Write(outputStream, pickled);
}
// Signal the end of data and stream.
SerDe.Write(outputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(outputStream, (int)SpecialLengths.END_OF_STREAM);
outputStream.Flush();
// Now process the bytes flowing in from the client.
var timingDataReceived = false;
var exceptionThrown = false;
var rowsReceived = new List <object[]>();
while (true)
{
var length = SerDe.ReadInt32(inputStream);
if (length > 0)
{
var pickledBytes = SerDe.ReadBytes(inputStream, length);
using var unpickler = new Unpickler();
var rows = unpickler.loads(pickledBytes) as ArrayList;
foreach (object row in rows)
{
rowsReceived.Add((object[])row);
}
}
else if (length == (int)SpecialLengths.TIMING_DATA)
{
var bootTime = SerDe.ReadInt64(inputStream);
var initTime = SerDe.ReadInt64(inputStream);
var finishTime = SerDe.ReadInt64(inputStream);
var memoryBytesSpilled = SerDe.ReadInt64(inputStream);
var diskBytesSpilled = SerDe.ReadInt64(inputStream);
timingDataReceived = true;
}
else if (length == (int)SpecialLengths.PYTHON_EXCEPTION_THROWN)
{
SerDe.ReadString(inputStream);
exceptionThrown = true;
break;
}
else if (length == (int)SpecialLengths.END_OF_DATA_SECTION)
{
var numAccumulatorUpdates = SerDe.ReadInt32(inputStream);
SerDe.ReadInt32(inputStream);
break;
}
}
Assert.True(timingDataReceived);
Assert.False(exceptionThrown);
// Validate rows received.
Assert.Equal(10, rowsReceived.Count);
for (int i = 0; i < 10; ++i)
{
// Two UDFs registered, thus expecting two columns.
// Refer to TestData.GetDefaultCommandPayload().
var row = rowsReceived[i];
Assert.Equal(2, rowsReceived[i].Length);
Assert.Equal($"udf2 udf1 {i}", row[0]);
Assert.Equal(i + i, row[1]);
}
}
Assert.True(clientTask.Wait(5000));
}
private static IEnumerable <dynamic> GetIterator(Stream inputStream, string serializedMode, int isFuncSqlUdf)
{
logger.LogInfo("Serialized mode in GetIterator: " + serializedMode);
IFormatter formatter = new BinaryFormatter();
var mode = (SerializedMode)Enum.Parse(typeof(SerializedMode), serializedMode);
int messageLength;
Stopwatch watch = Stopwatch.StartNew();
while ((messageLength = SerDe.ReadInt(inputStream)) != (int)SpecialLengths.END_OF_DATA_SECTION)
{
watch.Stop();
if (messageLength > 0 || messageLength == (int)SpecialLengths.NULL)
{
watch.Start();
byte[] buffer = messageLength > 0 ? SerDe.ReadBytes(inputStream, messageLength) : null;
watch.Stop();
switch (mode)
{
case SerializedMode.String:
{
if (messageLength > 0)
{
if (buffer == null)
{
logger.LogDebug("Buffer is null. Message length is {0}", messageLength);
}
yield return(SerDe.ToString(buffer));
}
else
{
yield return(null);
}
break;
}
case SerializedMode.Row:
{
Debug.Assert(messageLength > 0);
var unpickledObjects = PythonSerDe.GetUnpickledObjects(buffer);
if (isFuncSqlUdf == 0)
{
foreach (var row in unpickledObjects.Select(item => (item as RowConstructor).GetRow()))
{
yield return(row);
}
}
else
{
foreach (var row in unpickledObjects)
{
yield return(row);
}
}
break;
}
case SerializedMode.Pair:
{
byte[] pairKey = buffer;
byte[] pairValue = null;
watch.Start();
int valueLength = SerDe.ReadInt(inputStream);
if (valueLength > 0)
{
pairValue = SerDe.ReadBytes(inputStream, valueLength);
}
else if (valueLength == (int)SpecialLengths.NULL)
{
pairValue = null;
}
else
{
throw new Exception(string.Format("unexpected valueLength: {0}", valueLength));
}
watch.Stop();
yield return(new KeyValuePair <byte[], byte[]>(pairKey, pairValue));
break;
}
case SerializedMode.None: //just return raw bytes
{
yield return(buffer);
break;
}
case SerializedMode.Byte:
default:
{
if (buffer != null)
{
var ms = new MemoryStream(buffer);
yield return(formatter.Deserialize(ms));
}
else
{
yield return(null);
}
break;
}
}
}
watch.Start();
}
logger.LogInfo(string.Format("total receive time: {0}", watch.ElapsedMilliseconds));
}
static void Main(string[] args)
{
// if there exists exe.config file, then use log4net
if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile))
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance);
}
logger = LoggerServiceFactory.GetLogger(typeof(Worker));
Socket sock = null;
try
{
PrintFiles();
int javaPort = int.Parse(Console.ReadLine());
logger.LogInfo("java_port: " + javaPort);
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(IPAddress.Parse("127.0.0.1"), javaPort);
}
catch (Exception e)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(e);
Environment.Exit(-1);
}
using (NetworkStream s = new NetworkStream(sock))
{
try
{
DateTime bootTime = DateTime.UtcNow;
int splitIndex = SerDe.ReadInt(s);
logger.LogInfo("split_index: " + splitIndex);
if (splitIndex == -1)
{
Environment.Exit(-1);
}
string ver = SerDe.ReadString(s);
logger.LogInfo("ver: " + ver);
//// initialize global state
//shuffle.MemoryBytesSpilled = 0
//shuffle.DiskBytesSpilled = 0
// fetch name of workdir
string sparkFilesDir = SerDe.ReadString(s);
logger.LogInfo("spark_files_dir: " + sparkFilesDir);
//SparkFiles._root_directory = sparkFilesDir
//SparkFiles._is_running_on_worker = True
// fetch names of includes - not used //TODO - complete the impl
int numberOfIncludesItems = SerDe.ReadInt(s);
logger.LogInfo("num_includes: " + numberOfIncludesItems);
if (numberOfIncludesItems > 0)
{
for (int i = 0; i < numberOfIncludesItems; i++)
{
string filename = SerDe.ReadString(s);
}
}
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(s);
logger.LogInfo("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(s);
if (bid >= 0)
{
string path = SerDe.ReadString(s);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast.broadcastRegistry.Remove(bid);
}
}
}
Accumulator.accumulatorRegistry.Clear();
int lengthOCommandByteArray = SerDe.ReadInt(s);
logger.LogInfo("command_len: " + lengthOCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOCommandByteArray > 0)
{
string deserializerMode = SerDe.ReadString(s);
logger.LogInfo("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(s);
logger.LogInfo("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(s);
logger.LogInfo("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var func = (Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> >)formatter.Deserialize(stream);
DateTime initTime = DateTime.UtcNow;
int count = 0;
foreach (var message in func(splitIndex, GetIterator(s, deserializerMode)))
{
byte[] buffer;
if (serializerMode == "None")
{
buffer = message as byte[];
}
else if (serializerMode == "String")
{
buffer = SerDe.ToBytes(message as string);
}
else if (serializerMode == "Row")
{
Pickler pickler = new Pickler();
buffer = pickler.dumps(new ArrayList {
message
});
}
else
{
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
logger.LogError(string.Format("{0} : {1}", message.GetType().Name, message.GetType().FullName));
throw;
}
}
count++;
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
//TODO - complete the impl
logger.LogInfo("Count: " + count);
//if profiler:
// profiler.profile(process)
//else:
// process()
DateTime finish_time = DateTime.UtcNow;
const string format = "MM/dd/yyyy hh:mm:ss.fff tt";
logger.LogInfo(string.Format("bootTime: {0}, initTime: {1}, finish_time: {2}",
bootTime.ToString(format), initTime.ToString(format), finish_time.ToString(format)));
SerDe.Write(s, (int)SpecialLengths.TIMING_DATA);
SerDe.Write(s, ToUnixTime(bootTime));
SerDe.Write(s, ToUnixTime(initTime));
SerDe.Write(s, ToUnixTime(finish_time));
SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled
SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled
}
else
{
logger.LogWarn("Nothing to execute :-(");
}
// Mark the beginning of the accumulators section of the output
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, Accumulator.accumulatorRegistry.Count);
foreach (var item in Accumulator.accumulatorRegistry)
{
var ms = new MemoryStream();
var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value);
logger.LogInfo(string.Format("({0}, {1})", item.Key, value));
formatter.Serialize(ms, new KeyValuePair <int, dynamic>(item.Key, value));
byte[] buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
int end = SerDe.ReadInt(s);
// check end of stream
if (end == (int)SpecialLengths.END_OF_DATA_SECTION || end == (int)SpecialLengths.END_OF_STREAM)
{
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
logger.LogInfo("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM);
}
else
{
// write a different value to tell JVM to not reuse this worker
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
Environment.Exit(-1);
}
s.Flush();
// wait for server to complete, otherwise server gets 'connection reset' exception
// TODO: need to detect java side has closed socket properly
System.Threading.Thread.Sleep(1000);
}
catch (Exception e)
{
logger.LogError(e.ToString());
try
{
SerDe.Write(s, e.ToString());
}
catch (IOException)
{
// JVM close the socket
}
catch (Exception ex)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(ex);
}
Environment.Exit(-1);
}
}
sock.Close();
}
private static IFormatter ProcessCommand(Stream networkStream, int splitIndex, DateTime bootTime)
{
int lengthOfCommandByteArray = SerDe.ReadInt(networkStream);
logger.LogDebug("command length: " + lengthOfCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOfCommandByteArray > 0)
{
var commandProcessWatch = new Stopwatch();
var funcProcessWatch = new Stopwatch();
commandProcessWatch.Start();
int stageId = ReadDiagnosticsInfo(networkStream);
string deserializerMode = SerDe.ReadString(networkStream);
logger.LogDebug("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(networkStream);
logger.LogDebug("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(networkStream);
logger.LogDebug("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var workerFunc = (CSharpWorkerFunc)formatter.Deserialize(stream);
var func = workerFunc.Func;
logger.LogDebug(
"------------------------ Printing stack trace of workerFunc for ** debugging ** ------------------------------");
logger.LogDebug(workerFunc.StackTrace);
logger.LogDebug(
"--------------------------------------------------------------------------------------------------------------");
DateTime initTime = DateTime.UtcNow;
// here we use low level API because we need to get perf metrics
var inputEnumerator = new WorkerInputEnumerator(networkStream, deserializerMode);
IEnumerable <dynamic> inputEnumerable = inputEnumerator.Cast <dynamic>();
funcProcessWatch.Start();
IEnumerable <dynamic> outputEnumerable = func(splitIndex, inputEnumerable);
var outputEnumerator = outputEnumerable.GetEnumerator();
funcProcessWatch.Stop();
int count = 0;
int nullMessageCount = 0;
while (true)
{
funcProcessWatch.Start();
bool hasNext = outputEnumerator.MoveNext();
funcProcessWatch.Stop();
if (!hasNext)
{
break;
}
funcProcessWatch.Start();
var message = outputEnumerator.Current;
funcProcessWatch.Stop();
if (object.ReferenceEquals(null, message))
{
nullMessageCount++;
continue;
}
WriteOutput(networkStream, serializerMode, message, formatter);
count++;
}
logger.LogDebug("Output entries count: " + count);
logger.LogDebug("Null messages count: " + nullMessageCount);
//if profiler:
// profiler.profile(process)
//else:
// process()
WriteDiagnosticsInfo(networkStream, bootTime, initTime);
commandProcessWatch.Stop();
// log statistics
inputEnumerator.LogStatistic();
logger.LogInfo(string.Format("func process time: {0}", funcProcessWatch.ElapsedMilliseconds));
logger.LogInfo(string.Format("stage {0}, command process time: {1}", stageId, commandProcessWatch.ElapsedMilliseconds));
}
else
{
logger.LogWarn("lengthOfCommandByteArray = 0. Nothing to execute :-(");
}
return(formatter);
}
public void TestInitialize()
{
result = null;
// Create Mock object to mock implementation of T by new Mock<T>();
_mockSparkCLRProxy = new Mock <ISparkCLRProxy>();
_mockSparkContextProxy = new Mock <ISparkContextProxy>();
_mockStreamingContextProxy = new Mock <IStreamingContextProxy>();
_mockRddProxy = new Mock <IRDDProxy>();
SparkCLREnvironment.SparkCLRProxy = _mockSparkCLRProxy.Object;
// Mock method of T by Mock<T>.Setup(). For method with parameters, you can mock different method implementation for different method parameters.
// e.g., if you want to mock a method regardless of what values the method parameters are, you can use It.IsAny<T>() for each parameter; if you want
// to mock the method for certain criteria, use It.Is<T>(Func<T, bool>) can. You can mock the same method multiple times for different criteria of
// method parameters.
// If the method to mock has return value and you want to mock the return value only, Use Returns(TReturnValue); if you want to add logics and return,
// use Returns<T1, T2, ...>(Func<T1, T2, ..., TReturnValue>). If method is void, use CallBack<T1, T2, ...>(Action<T1, T2, ...>)
_mockSparkCLRProxy.Setup(m => m.CreateSparkConf(It.IsAny <bool>())).Returns(new MockSparkConfProxy()); // some of mocks which rarely change can be kept
_mockSparkCLRProxy.Setup(m => m.CreateSparkContext(It.IsAny <ISparkConfProxy>())).Returns(_mockSparkContextProxy.Object);
_mockSparkContextProxy.Setup(m => m.CreateStreamingContext(It.IsAny <SparkContext>(), It.IsAny <long>())).Returns(_mockStreamingContextProxy.Object);
_mockRddProxy.Setup(m => m.CollectAndServe()).Returns(() =>
{
TcpListener listener = new TcpListener(IPAddress.Parse("127.0.0.1"), 0);
listener.Start();
Task.Run(() =>
{
using (Socket socket = listener.AcceptSocket())
using (Stream ns = new NetworkStream(socket))
{
foreach (var item in result)
{
var ms = new MemoryStream();
new BinaryFormatter().Serialize(ms, item);
byte[] buffer = ms.ToArray();
SerDe.Write(ns, buffer.Length);
SerDe.Write(ns, buffer);
}
}
});
return((listener.LocalEndpoint as IPEndPoint).Port);
});
_mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(),
It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >()))
.Returns <IRDDProxy, byte[], Dictionary <string, string>, List <string>, bool, List <Broadcast>, List <byte[]> >(
(prefvJavaRddReference, command, environmentVariables, cSharpIncludes, preservePartitioning, broadcastVariables, accumulator) =>
{
IEnumerable <dynamic> input = result ?? (new[] {
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The dog lazy"
}).AsEnumerable();
var formatter = new BinaryFormatter();
using (MemoryStream s = new MemoryStream(command))
{
SerDe.ReadString(s);
SerDe.ReadString(s);
var func = (Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> >)formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
result = func(default(int), input);
}
if (result.FirstOrDefault() is byte[] && (result.First() as byte[]).Length == 8)
{
result = result.Where(e => (e as byte[]).Length != 8).Select(e => formatter.Deserialize(new MemoryStream(e as byte[])));
}
return(_mockRddProxy.Object);
});
_streamingContext = new StreamingContext(new SparkContext("", ""), 1000);
}
public void TestPicklingSqlCommandExecutorWithSingleCommand()
{
var udfWrapper = new Sql.PicklingUdfWrapper <string, string>(
(str) => "udf: " + ((str is null) ? "NULL" : str));
var command = new SqlCommand()
{
ArgOffsets = new[] { 0 },
NumChainedFunctions = 1,
WorkerFunction = new Sql.PicklingWorkerFunction(udfWrapper.Execute),
SerializerMode = CommandSerDe.SerializedMode.Row,
DeserializerMode = CommandSerDe.SerializedMode.Row
};
var commandPayload = new Worker.CommandPayload()
{
EvalType = UdfUtils.PythonEvalType.SQL_BATCHED_UDF,
Commands = new[] { command }
};
using (var inputStream = new MemoryStream())
using (var outputStream = new MemoryStream())
{
int numRows = 10;
// Write test data to the input stream.
var pickler = new Pickler();
for (int i = 0; i < numRows; ++i)
{
var pickled = pickler.dumps(
new[] { new object[] { (i % 2 == 0) ? null : i.ToString() } });
SerDe.Write(inputStream, pickled.Length);
SerDe.Write(inputStream, pickled);
}
SerDe.Write(inputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
inputStream.Seek(0, SeekOrigin.Begin);
CommandExecutorStat stat = new CommandExecutor().Execute(
inputStream,
outputStream,
0,
commandPayload);
// Validate that all the data on the stream is read.
Assert.Equal(inputStream.Length, inputStream.Position);
Assert.Equal(10, stat.NumEntriesProcessed);
// Validate the output stream.
outputStream.Seek(0, SeekOrigin.Begin);
var unpickler = new Unpickler();
// One row was written as a batch above, thus need to read 'numRows' batches.
List <object> rows = new List <object>();
for (int i = 0; i < numRows; ++i)
{
int length = SerDe.ReadInt32(outputStream);
byte[] pickledBytes = SerDe.ReadBytes(outputStream, length);
rows.Add((unpickler.loads(pickledBytes) as ArrayList)[0] as object);
}
Assert.Equal(numRows, rows.Count);
// Validate the single command.
for (int i = 0; i < numRows; ++i)
{
Assert.Equal(
"udf: " + ((i % 2 == 0) ? "NULL" : i.ToString()),
(string)rows[i]);
}
// Validate all the data on the stream is read.
Assert.Equal(outputStream.Length, outputStream.Position);
}
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("the length of args should be 2");
Environment.Exit(-1);
return;
}
string firstParamter = args[0].Trim();
if (!firstParamter.Equals("-port"))
{
Console.WriteLine("the first parameter of args should be -port");
Environment.Exit(-1);
return;
}
string strPort = args[1].Trim();
int portNumber = 0;
if (!int.TryParse(strPort, out portNumber))
{
Console.WriteLine("the second parameter of args should be an integer");
Environment.Exit(-1);
return;
}
Socket listenSocket = null;
Socket clientSocket = null;
SocketInformation socketInfo;
try
{
listenSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
listenSocket.Bind(new IPEndPoint(IPAddress.Loopback, portNumber));
listenSocket.Listen(5);
clientSocket = listenSocket.Accept();
// receive protocolinfo of the duplicated socket
byte[] recvBytes = new byte[10000];
int count = 0;
using (NetworkStream s = new NetworkStream(clientSocket))
{
count = s.Read(recvBytes, 0, 10000);
}
socketInfo = new SocketInformation();
byte[] protocolInfo = new byte[count];
Array.Copy(recvBytes, protocolInfo, count);
socketInfo.ProtocolInformation = protocolInfo;
socketInfo.Options = SocketInformationOptions.Connected;
}
finally
{
if (clientSocket != null)
{
clientSocket.Close();
}
if (listenSocket != null)
{
listenSocket.Close();
}
}
string envVar = Environment.GetEnvironmentVariable("SPARK_REUSE_WORKER"); // this envVar is set in JVM side
bool sparkReuseWorker = false;
if ((envVar != null) && envVar.Equals("1"))
{
sparkReuseWorker = true;
}
Socket socket = new Socket(socketInfo);
// Acknowledge that the fork was successful
using (NetworkStream s = new NetworkStream(socket))
{
SerDe.Write(s, Process.GetCurrentProcess().Id);
}
while (true)
{
int exitcode = Run(socket);
if (!sparkReuseWorker || exitcode == -1)
{
break;
}
}
// wait for server to complete, otherwise server gets 'connection reset' exception
// Use SerDe.ReadBytes() to detect java side has closed socket properly
// ReadBytes() will block until the socket is closed
using (NetworkStream s = new NetworkStream(socket))
{
logger.LogInfo("ReadBytes begin");
SerDe.ReadBytes(s);
logger.LogInfo("ReadBytes end");
}
if (socket != null)
{
socket.Close();
}
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prefvJavaRddReference, byte[] command, Dictionary <string, string> environmentVariables, List <string> cSharpIncludes, bool preservePartitioning, List <Broadcast> broadcastVariables, List <byte[]> accumulator)
{
IEnumerable <dynamic> input = (prefvJavaRddReference as MockRddProxy).result ??
(new string[] {
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The dog lazy"
}).AsEnumerable().Cast <dynamic>();
using (MemoryStream s = new MemoryStream(command))
{
int rddId = SerDe.ReadInt(s);
int stageId = SerDe.ReadInt(s);
int partitionId = SerDe.ReadInt(s);
string deserializerMode = SerDe.ReadString(s);
string serializerMode = SerDe.ReadString(s);
CSharpWorkerFunc workerFunc = (CSharpWorkerFunc)formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
var func = workerFunc.Func;
IEnumerable <dynamic> output = func(default(int), input);
// number 8 indicates shuffling scenario's leading 8-byte hash code of each data row which should be filtered
if (output.FirstOrDefault() is byte[] && (output.First() as byte[]).Length == 8)
{
output = output.Where(e => (e as byte[]).Length != 8).Select(e => formatter.Deserialize(new MemoryStream(e as byte[])));
}
return(new MockRddProxy(output));
}
}
public void TestPicklingSqlCommandExecutorWithMultiCommands()
{
var udfWrapper1 = new Sql.PicklingUdfWrapper <string, string>((str) => $"udf: {str}");
var udfWrapper2 = new Sql.PicklingUdfWrapper <int, int, int>(
(arg1, arg2) => arg1 * arg2);
var command1 = new SqlCommand()
{
ArgOffsets = new[] { 0 },
NumChainedFunctions = 1,
WorkerFunction = new Sql.PicklingWorkerFunction(udfWrapper1.Execute),
SerializerMode = CommandSerDe.SerializedMode.Row,
DeserializerMode = CommandSerDe.SerializedMode.Row
};
var command2 = new SqlCommand()
{
ArgOffsets = new[] { 1, 2 },
NumChainedFunctions = 1,
WorkerFunction = new Sql.PicklingWorkerFunction(udfWrapper2.Execute),
SerializerMode = CommandSerDe.SerializedMode.Row,
DeserializerMode = CommandSerDe.SerializedMode.Row
};
var commandPayload = new Worker.CommandPayload()
{
EvalType = UdfUtils.PythonEvalType.SQL_BATCHED_UDF,
Commands = new[] { command1, command2 }
};
using (var inputStream = new MemoryStream())
using (var outputStream = new MemoryStream())
{
int numRows = 10;
// Write test data to the input stream.
var pickler = new Pickler();
for (int i = 0; i < numRows; ++i)
{
byte[] pickled = pickler.dumps(
new[] { new object[] { i.ToString(), i, i } });
SerDe.Write(inputStream, pickled.Length);
SerDe.Write(inputStream, pickled);
}
SerDe.Write(inputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
inputStream.Seek(0, SeekOrigin.Begin);
CommandExecutorStat stat = new CommandExecutor().Execute(
inputStream,
outputStream,
0,
commandPayload);
// Validate all the data on the stream is read.
Assert.Equal(inputStream.Length, inputStream.Position);
Assert.Equal(10, stat.NumEntriesProcessed);
// Validate the output stream.
outputStream.Seek(0, SeekOrigin.Begin);
var unpickler = new Unpickler();
// One row was written as a batch above, thus need to read 'numRows' batches.
List <object[]> rows = new List <object[]>();
for (int i = 0; i < numRows; ++i)
{
int length = SerDe.ReadInt32(outputStream);
byte[] pickledBytes = SerDe.ReadBytes(outputStream, length);
rows.Add((unpickler.loads(pickledBytes) as ArrayList)[0] as object[]);
}
Assert.Equal(numRows, rows.Count);
for (int i = 0; i < numRows; ++i)
{
// There were two UDFs each of which produces one column.
object[] columns = rows[i];
Assert.Equal($"udf: {i}", (string)columns[0]);
Assert.Equal(i * i, (int)columns[1]);
}
// Validate all the data on the stream is read.
Assert.Equal(outputStream.Length, outputStream.Position);
}
}
public int StartCallback()
{
TcpListener callbackServer = new TcpListener(IPAddress.Parse("127.0.0.1"), 0);
callbackServer.Start();
Task.Run(() =>
{
try
{
using (Socket sock = callbackServer.AcceptSocket())
using (var s = new NetworkStream(sock))
{
while (true)
{
try
{
string cmd = SerDe.ReadString(s);
if (cmd == "close")
{
logger.LogInfo("receive close cmd from Scala side");
break;
}
else if (cmd == "callback")
{
int numRDDs = SerDe.ReadInt(s);
var jrdds = new List <JvmObjectReference>();
for (int i = 0; i < numRDDs; i++)
{
jrdds.Add(new JvmObjectReference(SerDe.ReadObjectId(s)));
}
double time = SerDe.ReadDouble(s);
IFormatter formatter = new BinaryFormatter();
object func = formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
string deserializer = SerDe.ReadString(s);
RDD <dynamic> rdd = null;
if (jrdds[0].Id != null)
{
rdd = new RDD <dynamic>(new RDDIpcProxy(jrdds[0]), sparkContext, (SerializedMode)Enum.Parse(typeof(SerializedMode), deserializer));
}
if (func is Func <double, RDD <dynamic>, RDD <dynamic> > )
{
JvmObjectReference jrdd = (((Func <double, RDD <dynamic>, RDD <dynamic> >)func)(time, rdd).RddProxy as RDDIpcProxy).JvmRddReference;
SerDe.Write(s, (byte)'j');
SerDe.Write(s, jrdd.Id);
}
else if (func is Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > )
{
string deserializer2 = SerDe.ReadString(s);
RDD <dynamic> rdd2 = new RDD <dynamic>(new RDDIpcProxy(jrdds[1]), sparkContext, (SerializedMode)Enum.Parse(typeof(SerializedMode), deserializer2));
JvmObjectReference jrdd = (((Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)func)(time, rdd, rdd2).RddProxy as RDDIpcProxy).JvmRddReference;
SerDe.Write(s, (byte)'j');
SerDe.Write(s, jrdd.Id);
}
else
{
((Action <double, RDD <dynamic> >)func)(time, rdd);
SerDe.Write(s, (byte)'n');
}
}
}
catch (Exception e)
{
//log exception only when callback socket is not shutdown explicitly
if (!callbackSocketShutdown)
{
logger.LogInfo(e.ToString());
}
}
}
}
}
catch (Exception e)
{
logger.LogInfo(e.ToString());
throw;
}
finally
{
if (callbackServer != null)
{
callbackServer.Stop();
}
}
});
return((callbackServer.LocalEndpoint as IPEndPoint).Port);
}
public void TestInitialize()
{
result = null;
// Create Mock object to mock implementation of T by new Mock<T>();
_mockSparkCLRProxy = new Mock <ISparkCLRProxy>();
_mockSparkContextProxy = new Mock <ISparkContextProxy>();
_mockStreamingContextProxy = new Mock <IStreamingContextProxy>();
_mockRddProxy = new Mock <IRDDProxy>();
SparkCLREnvironment.SparkCLRProxy = _mockSparkCLRProxy.Object;
// Mock method of T by Mock<T>.Setup(). For method with parameters, you can mock different method implementation for different method parameters.
// e.g., if you want to mock a method regardless of what values the method parameters are, you can use It.IsAny<T>() for each parameter; if you want
// to mock the method for certain criteria, use It.Is<T>(Func<T, bool>) can. You can mock the same method multiple times for different criteria of
// method parameters.
// If the method to mock has return value and you want to mock the return value only, Use Returns(TReturnValue); if you want to add logics and return,
// use Returns<T1, T2, ...>(Func<T1, T2, ..., TReturnValue>). If method is void, use CallBack<T1, T2, ...>(Action<T1, T2, ...>)
// for more info please visit https://github.com/Moq/moq4/wiki/Quickstart
_mockSparkCLRProxy.Setup(m => m.CreateSparkConf(It.IsAny <bool>())).Returns(new MockSparkConfProxy()); // some of mocks which rarely change can be kept
_mockSparkCLRProxy.Setup(m => m.CreateSparkContext(It.IsAny <ISparkConfProxy>())).Returns(_mockSparkContextProxy.Object);
_mockSparkCLRProxy.Setup(m => m.CreateStreamingContext(It.IsAny <SparkContext>(), It.IsAny <long>())).Returns(_mockStreamingContextProxy.Object);
_mockRddProxy.Setup(m => m.CollectAndServe()).Returns(() =>
{
var listener = SocketFactory.CreateSocket();
listener.Listen();
Task.Run(() =>
{
using (var socket = listener.Accept())
using (var ns = socket.GetStream())
{
foreach (var item in result)
{
var ms = new MemoryStream();
new BinaryFormatter().Serialize(ms, item);
byte[] buffer = ms.ToArray();
SerDe.Write(ns, buffer.Length);
SerDe.Write(ns, buffer);
}
ns.Flush();
}
});
return((listener.LocalEndPoint as IPEndPoint).Port);
});
_mockRddProxy.Setup(m => m.RDDCollector).Returns(new RDDCollector());
_mockSparkContextProxy.Setup(m => m.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(),
It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >()))
.Returns <IRDDProxy, byte[], Dictionary <string, string>, List <string>, bool, List <Broadcast>, List <byte[]> >(
(prefvJavaRddReference, command, environmentVariables, cSharpIncludes, preservePartitioning, broadcastVariables, accumulator) =>
{
IEnumerable <dynamic> input = result ?? (new[] {
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The dog lazy"
}).AsEnumerable();
var formatter = new BinaryFormatter();
using (MemoryStream s = new MemoryStream(command))
{
int rddId = SerDe.ReadInt(s);
int stageId = SerDe.ReadInt(s);
int partitionId = SerDe.ReadInt(s);
SerDe.ReadString(s);
SerDe.ReadString(s);
string runMode = SerDe.ReadString(s);
if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase))
{
string compilationDumpDir = SerDe.ReadString(s);
}
CSharpWorkerFunc workerFunc = (CSharpWorkerFunc)formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
var func = workerFunc.Func;
result = func(default(int), input);
}
if (result.FirstOrDefault() is byte[] && (result.First() as byte[]).Length == 8)
{
result = result.Where(e => (e as byte[]).Length != 8).Select(e => formatter.Deserialize(new MemoryStream(e as byte[])));
}
return(_mockRddProxy.Object);
});
_streamingContext = new StreamingContext(new SparkContext("", ""), 1000L);
}
public void Run()
{
Logger.LogInfo("TaskRunner [{0}] is running ...", TaskId);
try
{
while (!stop)
{
using (var inputStream = socket.GetInputStream())
using (var outputStream = socket.GetOutputStream())
{
if (!string.IsNullOrEmpty(secret))
{
SerDe.Write(outputStream, secret);
outputStream.Flush();
var reply = SerDe.ReadString(inputStream);
Logger.LogDebug("Connect back to JVM: " + reply);
secret = null;
}
byte[] bytes = SerDe.ReadBytes(inputStream, sizeof(int));
if (bytes != null)
{
int splitIndex = SerDe.ToInt(bytes);
bool readComplete = Worker.ProcessStream(inputStream, outputStream, splitIndex);
outputStream.Flush();
if (!readComplete) // if the socket is not read through completely, then it can't be reused
{
stop = true;
// wait for server to complete, otherwise server may get 'connection reset' exception
Logger.LogInfo("Sleep 500 millisecond to close socket ...");
Thread.Sleep(500);
}
else if (!socketReuse)
{
stop = true;
// wait for server to complete, otherwise server gets 'connection reset' exception
// Use SerDe.ReadBytes() to detect java side has closed socket properly
// ReadBytes() will block until the socket is closed
Logger.LogInfo("waiting JVM side to close socket...");
SerDe.ReadBytes(inputStream);
Logger.LogInfo("JVM side has closed socket");
}
}
else
{
stop = true;
Logger.LogWarn("read null splitIndex, socket is closed by JVM");
}
}
}
}
catch (Exception e)
{
stop = true;
Logger.LogError("TaskRunner [{0}] exeption, will dispose this TaskRunner", TaskId);
Logger.LogException(e);
}
finally
{
try
{
socket.Close();
}
catch (Exception ex)
{
Logger.LogWarn("close socket exception: {0}", ex);
}
Logger.LogInfo("TaskRunner [{0}] finished", TaskId);
}
}
private void ProcessCallbackRequest(object socket)
{
logger.LogDebug("New thread (id={0}) created to process callback request", Thread.CurrentThread.ManagedThreadId);
try
{
using (var sock = (ISocketWrapper)socket)
using (var s = sock.GetStream())
{
while (true)
{
try
{
string cmd = SerDe.ReadString(s);
if (cmd == "close")
{
logger.LogDebug("receive close cmd from Scala side");
break;
}
else if (cmd == "callback")
{
int numRDDs = SerDe.ReadInt(s);
var jrdds = new List <JvmObjectReference>();
for (int i = 0; i < numRDDs; i++)
{
jrdds.Add(new JvmObjectReference(SerDe.ReadObjectId(s)));
}
double time = SerDe.ReadDouble(s);
IFormatter formatter = new BinaryFormatter();
object func = formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
string serializedMode = SerDe.ReadString(s);
RDD <dynamic> rdd = null;
if (jrdds[0].Id != null)
{
rdd = new RDD <dynamic>(new RDDIpcProxy(jrdds[0]), sparkContext, (SerializedMode)Enum.Parse(typeof(SerializedMode), serializedMode));
}
if (func is Func <double, RDD <dynamic>, RDD <dynamic> > )
{
JvmObjectReference jrdd = ((((Func <double, RDD <dynamic>, RDD <dynamic> >)func)(time, rdd)).RddProxy as RDDIpcProxy).JvmRddReference;
SerDe.Write(s, (byte)'j');
SerDe.Write(s, jrdd.Id);
}
else if (func is Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > )
{
string serializedMode2 = SerDe.ReadString(s);
RDD <dynamic> rdd2 = new RDD <dynamic>(new RDDIpcProxy(jrdds[1]), sparkContext, (SerializedMode)Enum.Parse(typeof(SerializedMode), serializedMode2));
JvmObjectReference jrdd = ((((Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)func)(time, rdd, rdd2)).RddProxy as RDDIpcProxy).JvmRddReference;
SerDe.Write(s, (byte)'j');
SerDe.Write(s, jrdd.Id);
}
else
{
((Action <double, RDD <dynamic> >)func)(time, rdd);
SerDe.Write(s, (byte)'n');
}
}
}
catch (Exception e)
{
//log exception only when callback socket is not shutdown explicitly
if (!callbackSocketShutdown)
{
logger.LogError("Exception processing call back request. Thread id {0}", Thread.CurrentThread.ManagedThreadId);
logger.LogException(e);
// exit when exception happens
logger.LogError("ProcessCallbackRequest fail, will exit ...");
Thread.Sleep(1000);
System.Environment.Exit(1);
}
}
}
}
}
catch (Exception e)
{
logger.LogError("Exception in callback. Thread id {0}", Thread.CurrentThread.ManagedThreadId);
logger.LogException(e);
}
logger.LogDebug("Thread (id={0}) to process callback request exiting", Thread.CurrentThread.ManagedThreadId);
}
internal static bool ReadBoolean(Stream s)
{
byte[] buffer = SerDe.ReadBytes(s, 1);
return(BitConverter.ToBoolean(buffer, 0));
}