internal static int RunJob(IRDDProxy rdd)
{
var mockRdd = (rdd as MockRddProxy);
IEnumerable <byte[]> result = mockRdd.pickle ? mockRdd.result.Cast <byte[]>() :
mockRdd.result.Select(x =>
{
var ms = new MemoryStream();
formatter.Serialize(ms, x);
return(ms.ToArray());
});
TcpListener listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
Task.Run(() =>
{
using (Socket socket = listener.AcceptSocket())
using (Stream ns = new NetworkStream(socket))
{
foreach (var item in result)
{
SerDe.Write(ns, item.Length);
SerDe.Write(ns, item);
}
}
});
return((listener.LocalEndpoint as IPEndPoint).Port);
}
private static void ProcessBroadcastVariables(Stream networkStream)
{
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(networkStream);
logger.LogDebug("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(networkStream);
if (bid >= 0)
{
string path = SerDe.ReadString(networkStream);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast bc;
Broadcast.broadcastRegistry.TryRemove(bid, out bc);
}
}
}
}
public void TestWorkerIncompleteData()
{
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
SerDe.Write(s, command.Length);
s.Write(command, 0, command.Length);
for (int i = 0; i < 100; i++)
{
SerDe.Write(s, i.ToString());
}
s.Flush();
// Note: as send buffer is enabled by default, and CSharpWorker only flushes output after receives all data (receive END_OF_DATA_SECTION flag),
// so in current test we can't ensure expected number of result will be received at this point, validation for returned data is not enabled to avoid flaky test.
}
AssertWorker(worker, 0, "System.NullReferenceException: Object reference not set to an instance of an object.");
CSharpRDD_SocketServer.Close();
}
/// <summary>
/// read only data section from worker
/// </summary>
private IEnumerable <byte[]> ReadDataSection(Stream s, int expectedCount = 0)
{
int count = 0;
while (true)
{
int length = SerDe.ReadInt(s);
if (length > 0)
{
yield return(SerDe.ReadBytes(s, length));
if (expectedCount > 0 && ++count >= expectedCount)
{
break;
}
}
else if (length == (int)SpecialLengths.TIMING_DATA)
{
var bootTime = SerDe.ReadLong(s);
var initTime = SerDe.ReadLong(s);
var finishTime = SerDe.ReadLong(s);
var memoryBytesSpilled = SerDe.ReadLong(s);
var diskBytesSpilled = SerDe.ReadLong(s);
}
else if (length == (int)SpecialLengths.DOTNET_EXCEPTION_THROWN)
{
SerDe.ReadString(s);
break;
}
else if (length == (int)SpecialLengths.END_OF_DATA_SECTION)
{
break;
}
}
}
internal static SocketInfo RunJob(IRDDProxy rdd)
{
var mockRdd = (rdd as MockRddProxy);
IEnumerable <byte[]> result = mockRdd.pickle ? mockRdd.result.Cast <byte[]>() :
mockRdd.result.Select(x =>
{
var ms = new MemoryStream();
formatter.Serialize(ms, x);
return(ms.ToArray());
});
var listener = SocketFactory.CreateSocket();
listener.Listen();
Task.Run(() =>
{
using (var socket = listener.Accept())
using (var ns = socket.GetStream())
{
foreach (var item in result)
{
SerDe.Write(ns, item.Length);
SerDe.Write(ns, item);
}
ns.Flush();
}
});
SocketInfo socketInfo = new SocketInfo((listener.LocalEndPoint as IPEndPoint).Port, null);
return(socketInfo);
}
public void TestWorkerIncompleteBytes()
{
var originalReadBufferSize = Environment.GetEnvironmentVariable(ConfigurationService.CSharpWorkerReadBufferSizeEnvName);
try
{
if (SocketFactory.SocketWrapperType.Equals(SocketWrapperType.Rio))
{
Environment.SetEnvironmentVariable(ConfigurationService.CSharpWorkerReadBufferSizeEnvName, "0");
}
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
SerDe.Write(s, command.Length);
s.Write(command, 0, command.Length / 2);
s.Flush();
}
AssertWorker(worker, 0, "System.ArgumentException: Incomplete bytes read: ");
CSharpRDD_SocketServer.Close();
}
finally
{
Environment.SetEnvironmentVariable(ConfigurationService.CSharpWorkerReadBufferSizeEnvName, originalReadBufferSize);
}
}
private static UdfWrapperData GetUdfWrapperDataFromStream(
Stream stream,
out SerializedMode serializerMode,
out SerializedMode deserializerMode,
out string runMode)
{
if (!Enum.TryParse(SerDe.ReadString(stream), out serializerMode))
{
throw new InvalidDataException("Serializer mode is not valid.");
}
if (!Enum.TryParse(SerDe.ReadString(stream), out deserializerMode))
{
throw new InvalidDataException("Deserializer mode is not valid.");
}
runMode = SerDe.ReadString(stream);
byte[] serializedCommand = SerDe.ReadBytes(stream);
var bf = new BinaryFormatter();
var ms = new MemoryStream(serializedCommand, false);
return((UdfWrapperData)bf.Deserialize(ms));
}
internal void Run(ISocketWrapper listener)
{
try
{
listener.Listen();
// Communicate the server port back to the Spark using standard output.
Stream outputStream = Console.OpenStandardOutput();
var serverPort = ((IPEndPoint)listener.LocalEndPoint).Port;
SerDe.Write(outputStream, serverPort);
// Now the logger can be initialized after standard output's usage is done.
s_logger = LoggerServiceFactory.GetLogger(typeof(DaemonWorker));
s_logger.LogInfo($"Started .NET DaemonServer with port {serverPort}.");
// Start accepting connections from JVM.
new Thread(() => { StartServer(listener); }).Start();
WaitForSignal();
}
catch (Exception e)
{
s_logger.LogError($".NET DaemonWorker is exiting with exception: {e}.");
Environment.Exit(-1);
}
finally
{
_waitingTaskRunners.Dispose();
}
}
public void TestUndefinedAccumuator()
{
using (var s = sock.GetStream())
{
// write numUpdates
int numUpdates = 1;
SerDe.Write(s, numUpdates);
// write update
int key = 1;
int value = 1000;
Tuple <int, dynamic> update = new Tuple <int, dynamic>(key, value);
var ms = new MemoryStream();
var formatter = new BinaryFormatter();
formatter.Serialize(ms, update);
byte[] sendBuffer = ms.ToArray();
SerDe.Write(s, sendBuffer.Length);
SerDe.Write(s, sendBuffer);
s.Flush();
byte[] receiveBuffer = new byte[1];
s.Read(receiveBuffer, 0, 1);
Assert.IsTrue(Accumulator.accumulatorRegistry.ContainsKey(update.Item1));
var accumulator = Accumulator.accumulatorRegistry[update.Item1] as Accumulator <int>;
Assert.AreEqual(accumulator.Value, value);
}
}
public void TestAccumuatorSuccess()
{
Accumulator <int> accumulator = sc.Accumulator <int>(0);
using (var s = sock.GetStream())
{
// write numUpdates
int numUpdates = 1;
SerDe.Write(s, numUpdates);
// write update
int key = 0;
int value = 100;
Tuple <int, dynamic> update = new Tuple <int, dynamic>(key, value);
var ms = new MemoryStream();
var formatter = new BinaryFormatter();
formatter.Serialize(ms, update);
byte[] sendBuffer = ms.ToArray();
SerDe.Write(s, sendBuffer.Length);
SerDe.Write(s, sendBuffer);
s.Flush();
byte[] receiveBuffer = new byte[1];
s.Read(receiveBuffer, 0, 1);
Assert.AreEqual(accumulator.Value, value);
}
}
/// <summary>
/// Read one RDDCommand from the stream.
/// </summary>
/// <param name="stream">Stream to read from</param>
/// <returns>RDDCommand object</returns>
private static RDDCommand ReadRDDCommand(Stream stream)
{
int commandBytesCount = SerDe.ReadInt32(stream);
if (commandBytesCount <= 0)
{
throw new InvalidDataException(
$"Invalid command size: {commandBytesCount}");
}
var rddCommand = new RDDCommand
{
WorkerFunction = new RDD.WorkerFunction(
CommandSerDe.Deserialize <RDD.WorkerFunction.ExecuteDelegate>(
stream,
out CommandSerDe.SerializedMode serializerMode,
out CommandSerDe.SerializedMode deserializerMode,
out var runMode))
};
rddCommand.SerializerMode = serializerMode;
rddCommand.DeserializerMode = deserializerMode;
return(rddCommand);
}
public bool MoveNext()
{
watch.Start();
bool hasNext;
if ((items != null) && (pos < items.Length))
{
hasNext = true;
}
else
{
int messageLength = SerDe.ReadInt(inputStream);
if (messageLength == (int)SpecialLengths.END_OF_DATA_SECTION)
{
hasNext = false;
logger.LogDebug("END_OF_DATA_SECTION");
}
else if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL))
{
items = GetNext(messageLength);
Debug.Assert(items != null);
Debug.Assert(items.Any());
pos = 0;
hasNext = true;
}
else
{
throw new Exception(string.Format("unexpected messageLength: {0}", messageLength));
}
}
watch.Stop();
return hasNext;
}
internal static Dictionary <object, object> ReadDictionary(Stream s)
{
Dictionary <object, object> dict = new Dictionary <object, object>();
var len = SerDe.ReadInt(s);
if (len == 0)
{
return(dict);
}
var keysType = ReadObjectType(s);
var keysLen = SerDe.ReadInt(s);
var keys = new object[keysLen];
for (var i = 0; i < keys.Length; i++)
{
keys[i] = ReadTypedObject(s, keysType);
}
var valueLen = SerDe.ReadInt(s);
var values = new object[valueLen];
for (var i = 0; i < values.Length; i++)
{
values[i] = ReadTypedObject(s, ReadObjectType(s));
}
for (var i = 0; i < keys.Length; i++)
{
dict[keys[i]] = values[i];
}
return(dict);
}
public IRDDProxy CreateCSharpRdd(IRDDProxy prefvJavaRddReference, byte[] command, Dictionary <string, string> environmentVariables, List <string> cSharpIncludes, bool preservePartitioning, List <Broadcast> broadcastVariables, List <byte[]> accumulator)
{
IEnumerable <dynamic> input = (prefvJavaRddReference as MockRddProxy).result ??
(new string[] {
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The quick brown fox jumps over the lazy dog",
"The dog lazy"
}).AsEnumerable().Cast <dynamic>();
using (MemoryStream s = new MemoryStream(command))
{
string deserializerMode = SerDe.ReadString(s);
string serializerMode = SerDe.ReadString(s);
var func = (Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> >)formatter.Deserialize(new MemoryStream(SerDe.ReadBytes(s)));
IEnumerable <dynamic> output = func(default(int), input);
// number 8 indicates shuffling scenario's leading 8-byte hash code of each data row which should be filtered
if (output.FirstOrDefault() is byte[] && (output.First() as byte[]).Length == 8)
{
output = output.Where(e => (e as byte[]).Length != 8).Select(e => formatter.Deserialize(new MemoryStream(e as byte[])));
}
return(new MockRddProxy(output));
}
}
private static IFormatter ProcessCommand(Stream inputStream, Stream outputStream, int splitIndex, DateTime bootTime)
{
int isSqlUdf = SerDe.ReadInt(inputStream);
logger.LogDebug("Is func Sql UDF = {0}", isSqlUdf);
IFormatter formatter = new BinaryFormatter();
UDFCommand command = null;
if (isSqlUdf == 0)
{
command = ProcessNonUdfCommand(inputStream, outputStream, splitIndex, bootTime, formatter, isSqlUdf);
}
else
{
command = ProcessUdfCommand(inputStream, outputStream, splitIndex, bootTime, formatter, isSqlUdf);
}
if (command != null)
{
command.Execute();
}
return(formatter);
}
public void Write(Stream stream, TaskContext taskContext)
{
SerDe.Write(stream, taskContext.IsBarrier);
SerDe.Write(stream, taskContext.Port);
SerDe.Write(stream, taskContext.Secret);
SerDe.Write(stream, taskContext.StageId);
SerDe.Write(stream, taskContext.PartitionId);
SerDe.Write(stream, taskContext.AttemptNumber);
SerDe.Write(stream, taskContext.AttemptId);
SerDe.Write(stream, taskContext.Resources.Count());
foreach (TaskContext.Resource resource in taskContext.Resources)
{
SerDe.Write(stream, resource.Key);
SerDe.Write(stream, resource.Value);
SerDe.Write(stream, resource.Addresses.Count());
foreach (string address in resource.Addresses)
{
SerDe.Write(stream, address);
}
}
SerDe.Write(stream, taskContext.LocalProperties.Count);
foreach (KeyValuePair <string, string> kv in taskContext.LocalProperties)
{
SerDe.Write(stream, kv.Key);
SerDe.Write(stream, kv.Value);
}
}
public void TestWorkerSuccess()
{
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
SerDe.Write(s, command.Length);
SerDe.Write(s, command);
for (int i = 0; i < 100; i++)
{
SerDe.Write(s, i.ToString());
}
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
int count = 0;
foreach (var bytes in ReadWorker(s))
{
Assert.AreEqual(count++.ToString(), Encoding.UTF8.GetString(bytes));
}
Assert.AreEqual(100, count);
}
AssertWorker(worker);
CSharpRDD_SocketServer.Close();
}
public void TestWorkerIncompleteData()
{
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
SerDe.Write(s, command.Length);
s.Write(command, 0, command.Length);
for (int i = 0; i < 100; i++)
{
SerDe.Write(s, i.ToString());
}
int count = 0;
foreach (var bytes in ReadWorker(s, 100))
{
Assert.AreEqual(count++.ToString(), Encoding.UTF8.GetString(bytes));
}
Assert.AreEqual(100, count);
}
AssertWorker(worker, 0, "System.NullReferenceException: Object reference not set to an instance of an object.");
CSharpRDD_SocketServer.Close();
}
/// <summary>
/// Authenticates by writing secret to stream and validate the response.
/// </summary>
/// <param name="stream">Valid stream.</param>
/// <param name="secret">Secret string to authenticate against.</param>
/// <returns>True if authentication succeeds.</returns>
public static bool AuthenticateAsClient(Stream stream, string secret)
{
SerDe.Write(stream, secret);
stream.Flush();
return(SerDe.ReadString(stream) == s_validResponseCode);
}
internal static T Deserialize <T>(
Stream stream,
out SerializedMode serializerMode,
out SerializedMode deserializerMode,
out string runMode) where T : Delegate
{
if (!Enum.TryParse(SerDe.ReadString(stream), out serializerMode))
{
throw new InvalidDataException("Serializer mode is not valid.");
}
if (!Enum.TryParse(SerDe.ReadString(stream), out deserializerMode))
{
throw new InvalidDataException("Deserializer mode is not valid.");
}
runMode = SerDe.ReadString(stream);
var serializedCommand = SerDe.ReadBytes(stream);
var bf = new BinaryFormatter();
var ms = new MemoryStream(serializedCommand, false);
var udfWrapperData = (UdfWrapperData)bf.Deserialize(ms);
var nodeIndex = 0;
var udfIndex = 0;
var udf = (T)DeserializeUdfs <T>(udfWrapperData, ref nodeIndex, ref udfIndex);
// Check all the data is consumed.
Debug.Assert(nodeIndex == udfWrapperData.UdfWrapperNodes.Length);
Debug.Assert(udfIndex == udfWrapperData.Udfs.Length);
return(udf);
}
public void TestsSimpleWorkerTaskRunners(string version)
{
using ISocketWrapper serverListener = SocketFactory.CreateSocket();
var ipEndpoint = (IPEndPoint)serverListener.LocalEndPoint;
serverListener.Listen();
var typedVersion = new Version(version);
var simpleWorker = new SimpleWorker(typedVersion);
Task clientTask = Task.Run(() => simpleWorker.Run(ipEndpoint.Port));
PayloadWriter payloadWriter = new PayloadWriterFactory().Create(typedVersion);
using (ISocketWrapper serverSocket = serverListener.Accept())
{
if ((typedVersion.Major == 3 && typedVersion.Minor >= 2) || typedVersion.Major > 3)
{
int pid = SerDe.ReadInt32(serverSocket.InputStream);
}
TaskRunnerTests.TestTaskRunnerReadWrite(serverSocket, payloadWriter);
}
Assert.True(clientTask.Wait(5000));
}
public void TestWorkerWithDynamicLibrary()
{
var originalRunMode = Environment.GetEnvironmentVariable("SPARKCLR_RUN_MODE");
var originalCompilationDir = Environment.GetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR");
var compilationDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(compilationDir);
// copy dll
var currentDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
File.Copy(Path.Combine(currentDir, "Microsoft.Spark.CSharp.Adapter.dll"), Path.Combine(compilationDir, "ReplCompilation.1"));
try
{
Environment.SetEnvironmentVariable("SPARKCLR_RUN_MODE", "R");
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
WritePayloadHeaderToWorker(s);
Environment.SetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR", compilationDir);
byte[] commandWithDynamicLibraryPath = SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.String, SerializedMode.String);
SerDe.Write(s, commandWithDynamicLibraryPath.Length);
SerDe.Write(s, commandWithDynamicLibraryPath);
for (int i = 0; i < 100; i++)
{
SerDe.Write(s, i.ToString());
}
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
int count = 0;
foreach (var bytes in ReadWorker(s))
{
Assert.AreEqual(count++.ToString(), Encoding.UTF8.GetString(bytes));
}
Assert.AreEqual(100, count);
}
AssertWorker(worker);
CSharpRDD_SocketServer.Close();
}
finally
{
Environment.SetEnvironmentVariable("SPARKCLR_RUN_MODE", originalRunMode);
Environment.SetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR", originalCompilationDir);
Directory.Delete(compilationDir, true);
}
}
/// <summary>
/// Reads the given stream to construct a BroadcastVariables object.
/// </summary>
/// <param name="stream">The stream to read from</param>
/// <returns>BroadcastVariables object</returns>
internal BroadcastVariables Process(Stream stream)
{
var broadcastVars = new BroadcastVariables();
ISocketWrapper socket = null;
broadcastVars.DecryptionServerNeeded = SerDe.ReadBool(stream);
broadcastVars.Count = Math.Max(SerDe.ReadInt32(stream), 0);
if (broadcastVars.DecryptionServerNeeded)
{
broadcastVars.DecryptionServerPort = SerDe.ReadInt32(stream);
broadcastVars.Secret = SerDe.ReadString(stream);
if (broadcastVars.Count > 0)
{
socket = SocketFactory.CreateSocket();
socket.Connect(
IPAddress.Loopback,
broadcastVars.DecryptionServerPort,
broadcastVars.Secret);
}
}
var formatter = new BinaryFormatter();
for (int i = 0; i < broadcastVars.Count; ++i)
{
long bid = SerDe.ReadInt64(stream);
if (bid >= 0)
{
if (broadcastVars.DecryptionServerNeeded)
{
long readBid = SerDe.ReadInt64(socket.InputStream);
if (bid != readBid)
{
throw new Exception("The Broadcast Id received from the encryption " +
$"server {readBid} is different from the Broadcast Id received " +
$"from the payload {bid}.");
}
object value = formatter.Deserialize(socket.InputStream);
BroadcastRegistry.Add(bid, value);
}
else
{
string path = SerDe.ReadString(stream);
using FileStream fStream =
File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read);
object value = formatter.Deserialize(fStream);
BroadcastRegistry.Add(bid, value);
}
}
else
{
bid = -bid - 1;
BroadcastRegistry.Remove(bid);
}
}
socket?.Dispose();
return(broadcastVars);
}
protected override CommandExecutorStat ExecuteCore(
Stream inputStream,
Stream outputStream,
SqlCommand[] commands)
{
var stat = new CommandExecutorStat();
ICommandRunner commandRunner = CreateCommandRunner(commands);
// On the Spark side, each object in the following List<> is considered as a row.
// See the ICommandRunner comments above for the types for a row.
var outputRows = new List <object>();
// If the input is empty (no rows) or all rows have been read, then
// SpecialLengths.END_OF_DATA_SECTION is sent as the messageLength.
// For example, no rows:
// +---+----+
// |age|name|
// +---+----+
// +---+----+
int messageLength = 0;
while ((messageLength = SerDe.ReadInt32(inputStream)) !=
(int)SpecialLengths.END_OF_DATA_SECTION)
{
if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL))
{
if (messageLength <= 0)
{
throw new InvalidDataException(
$"Invalid message length: {messageLength}");
}
// Each row in inputRows is of type object[]. If a null is present in a row
// then the corresponding index column of the row object[] will be set to null.
// For example, (inputRows.Length == 2) and (inputRows[0][0] == null)
// +----+
// | age|
// +----+
// |null|
// | 11|
// +----+
object[] inputRows = PythonSerDe.GetUnpickledObjects(inputStream, messageLength);
for (int i = 0; i < inputRows.Length; ++i)
{
// Split id is not used for SQL UDFs, so 0 is passed.
outputRows.Add(commandRunner.Run(0, inputRows[i]));
}
// The initial (estimated) buffer size for pickling rows is set to the size of input pickled rows
// because the number of rows are the same for both input and output.
WriteOutput(outputStream, outputRows, messageLength);
stat.NumEntriesProcessed += inputRows.Length;
outputRows.Clear();
}
}
return(stat);
}
public void TestWriteDouble()
{
var ms = new MemoryStream();
SerDe.Write(ms, Math.PI);
ms.Position = 0;
Assert.AreEqual(Math.PI, SerDe.ReadDouble(ms), 1e-14);
}
public void TestBroadcastVariablesInWorker()
{
Process worker;
var CSharpRDD_SocketServer = CreateServer(out worker);
string assertMessage;
using (var serverSocket = CSharpRDD_SocketServer.Accept())
using (var s = serverSocket.GetStream())
{
SerDe.Write(s, splitIndex);
SerDe.Write(s, ver);
SerDe.Write(s, 0);
SerDe.Write(s, 0);
SerDe.Write(s, 0);
SerDe.Write(s, 0L);
SerDe.Write(s, sparkFilesDir);
SerDe.Write(s, numberOfIncludesItems);
// broadcastVariablesToAdd and broadcastVariablesToDelete are used to trigger broadcast variables operation(register and remove) in worker side,
// after worker exists, check wheather expected number of broadcast variables are processed.
var broadcastVariablesToAdd = new long[] { 101L, 102L, 103L };
var broadcastVariablesToDelete = new long[] { 10L, 20L };
SerDe.Write(s, broadcastVariablesToAdd.Length + broadcastVariablesToDelete.Length);
broadcastVariablesToAdd.ToList().ForEach(bid => { SerDe.Write(s, bid); SerDe.Write(s, "path" + bid); });
broadcastVariablesToDelete.ToList().ForEach(bid => SerDe.Write(s, -bid - 1));
SerDe.Write(s, 0); //flag for UDF
byte[] command = SparkContext.BuildCommand(new CSharpWorkerFunc((pid, iter) => iter), SerializedMode.String, SerializedMode.String);
SerDe.Write(s, command.Length);
SerDe.Write(s, command);
for (int i = 0; i < 100; i++)
{
SerDe.Write(s, i.ToString());
}
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
s.Flush();
int count = 0;
foreach (var bytes in ReadWorker(s))
{
Assert.AreEqual(count++.ToString(), Encoding.UTF8.GetString(bytes));
}
Assert.AreEqual(100, count);
// TODO verification should not depends on the output of worker
// we postpone the test of assertMessage after worker exit
assertMessage = "num_broadcast_variables: " + (broadcastVariablesToAdd.Length + broadcastVariablesToDelete.Length);
}
AssertWorker(worker, 0, assertMessage);
CSharpRDD_SocketServer.Close();
}
/// <summary>
/// write common header to worker
/// </summary>
/// <param name="s"></param>
private void WriteWorker(Stream s)
{
SerDe.Write(s, splitIndex);
SerDe.Write(s, ver);
SerDe.Write(s, sparkFilesDir);
SerDe.Write(s, numberOfIncludesItems);
SerDe.Write(s, numBroadcastVariables);
s.Flush();
}
private static int ReadDiagnosticsInfo(Stream networkStream)
{
int rddId = SerDe.ReadInt(networkStream);
int stageId = SerDe.ReadInt(networkStream);
int partitionId = SerDe.ReadInt(networkStream);
logger.LogInfo(string.Format("rddInfo: rddId {0}, stageId {1}, partitionId {2}", rddId, stageId, partitionId));
return(stageId);
}
protected override CommandExecutorStat ExecuteCore(
Stream inputStream,
Stream outputStream,
SqlCommand[] commands)
{
var stat = new CommandExecutorStat();
ICommandRunner commandRunner = CreateCommandRunner(commands);
SerDe.Write(outputStream, (int)SpecialLengths.START_ARROW_STREAM);
// TODO: Remove this MemoryStream once the arrow writer supports non-seekable streams.
// For now, we write to a temporary seekable MemoryStream which we then copy to
// the actual destination stream.
MemoryStream tmp = s_writeOutputStream ?? (s_writeOutputStream = new MemoryStream());
ArrowStreamWriter writer = null;
Schema resultSchema = null;
foreach (ReadOnlyMemory <IArrowArray> input in GetInputIterator(inputStream))
{
// Split id is currently not used, so 0 is passed.
IArrowArray[] results = commandRunner.Run(0, input);
// Assumes all columns have the same length, so uses 0th for num entries.
int numEntries = results[0].Length;
stat.NumEntriesProcessed += numEntries;
tmp.SetLength(0);
if (writer == null)
{
Debug.Assert(resultSchema == null);
resultSchema = BuildSchema(results);
writer = new ArrowStreamWriter(tmp, resultSchema, leaveOpen: true);
}
var recordBatch = new RecordBatch(resultSchema, results, numEntries);
// TODO: Remove sync-over-async once WriteRecordBatch exists.
writer.WriteRecordBatchAsync(recordBatch).GetAwaiter().GetResult();
tmp.Position = 0;
tmp.CopyTo(outputStream);
outputStream.Flush();
}
SerDe.Write(outputStream, 0);
if (writer != null)
{
writer.Dispose();
}
return(stat);
}
/// <summary>
/// write common header to worker
/// </summary>
/// <param name="s"></param>
private void WritePayloadHeaderToWorker(Stream s, int isSqlUdf = 0)
{
SerDe.Write(s, splitIndex);
SerDe.Write(s, ver);
SerDe.Write(s, sparkFilesDir);
SerDe.Write(s, numberOfIncludesItems);
SerDe.Write(s, numBroadcastVariables);
SerDe.Write(s, isSqlUdf); //flag for UDF
s.Flush();
}
