/// <summary>
/// Apply a function to each RDD in this DStream.
/// </summary>
/// <param name="f"></param>
public void ForeachRDD(Action <double, RDD <dynamic> > f)
{
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, f);
DStreamProxy.CallForeachRDD(stream.ToArray(), serializedMode.ToString());
}
internal static byte[] BuildCommand(object func, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte)
{
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, func);
List <byte[]> commandPayloadBytesList = new List <byte[]>();
// add deserializer mode
var modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString());
var length = modeBytes.Length;
var lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// add serializer mode
modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString());
length = modeBytes.Length;
lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// add func
var funcBytes = stream.ToArray();
var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length);
Array.Reverse(funcBytesLengthAsBytes);
commandPayloadBytesList.Add(funcBytesLengthAsBytes);
commandPayloadBytesList.Add(funcBytes);
return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray());
}
internal static byte[] BuildCommand(CSharpWorkerFunc workerFunc, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte)
{
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, workerFunc);
List <byte[]> commandPayloadBytesList = new List <byte[]>();
// reserve 12 bytes for RddId, stageId and partitionId, this info will be filled in CSharpRDD.scala
byte[] rddInfo = new byte[12];
for (int i = 0; i < rddInfo.Length; i++)
{
rddInfo[i] = 0;
}
commandPayloadBytesList.Add(rddInfo);
// add deserializer mode
var modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString());
var length = modeBytes.Length;
var lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// add serializer mode
modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString());
length = modeBytes.Length;
lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// add func
var funcBytes = stream.ToArray();
var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length);
Array.Reverse(funcBytesLengthAsBytes);
commandPayloadBytesList.Add(funcBytesLengthAsBytes);
commandPayloadBytesList.Add(funcBytes);
return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray());
}
internal static byte[] Serialize(
Delegate func,
SerializedMode deserializerMode = SerializedMode.Byte,
SerializedMode serializerMode = SerializedMode.Byte)
{
// TODO: Rework on the following List<Byte[]> to use MemoryStream!
var commandPayloadBytesList = new List <byte[]>();
// Add serializer mode.
byte[] modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString());
int length = modeBytes.Length;
byte[] lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// Add deserializer mode.
modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString());
length = modeBytes.Length;
lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// Add run mode:
// N - normal
// R - repl
string runMode = Environment.GetEnvironmentVariable("SPARK_NET_RUN_MODE") ?? "N";
byte[] runModeBytes = Encoding.UTF8.GetBytes(runMode);
lengthAsBytes = BitConverter.GetBytes(runModeBytes.Length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(runModeBytes);
if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase))
{
// add compilation dump directory
byte[] compilationDumpDirBytes = Encoding.UTF8.GetBytes(
Environment.GetEnvironmentVariable("SPARK_NET_SCRIPT_COMPILATION_DIR") ?? ".");
lengthAsBytes = BitConverter.GetBytes(compilationDumpDirBytes.Length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(compilationDumpDirBytes);
}
// Serialize the UDFs.
var udfWrapperNodes = new List <UdfWrapperNode>();
var udfs = new List <UdfSerDe.UdfData>();
SerializeUdfs(func, null, udfWrapperNodes, udfs);
// Run through UdfSerDe.Serialize once more to get serialization info
// on the actual UDF.
var udfWrapperData = new UdfWrapperData()
{
UdfWrapperNodes = udfWrapperNodes.ToArray(),
Udfs = udfs.ToArray()
};
var formatter = new BinaryFormatter();
using (var stream = new MemoryStream())
{
formatter.Serialize(stream, udfWrapperData);
byte[] udfBytes = stream.ToArray();
byte[] udfBytesLengthAsBytes = BitConverter.GetBytes(udfBytes.Length);
Array.Reverse(udfBytesLengthAsBytes);
commandPayloadBytesList.Add(udfBytesLengthAsBytes);
commandPayloadBytesList.Add(udfBytes);
}
return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray());
}
internal static byte[] BuildCommand(CSharpWorkerFunc workerFunc, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte)
{
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, workerFunc);
List <byte[]> commandPayloadBytesList = new List <byte[]>();
// reserve 12 bytes for RddId, stageId and partitionId, this info will be filled in CSharpRDD.scala
byte[] rddInfo = new byte[12];
for (int i = 0; i < rddInfo.Length; i++)
{
rddInfo[i] = 0;
}
commandPayloadBytesList.Add(rddInfo);
// add deserializer mode
var modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString());
var length = modeBytes.Length;
var lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// add serializer mode
modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString());
length = modeBytes.Length;
lengthAsBytes = BitConverter.GetBytes(length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(modeBytes);
// add run mode
// N - normal
// R - repl
var runMode = Environment.GetEnvironmentVariable("SPARKCLR_RUN_MODE") ?? "N";
var runModeBytes = Encoding.UTF8.GetBytes(runMode);
lengthAsBytes = BitConverter.GetBytes(runModeBytes.Length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(runModeBytes);
if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase))
{
// add compilation dump directory
var compilationDumpDirBytes = Encoding.UTF8.GetBytes(Environment.GetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR") ?? ".");
lengthAsBytes = BitConverter.GetBytes(compilationDumpDirBytes.Length);
Array.Reverse(lengthAsBytes);
commandPayloadBytesList.Add(lengthAsBytes);
commandPayloadBytesList.Add(compilationDumpDirBytes);
}
// add func
var funcBytes = stream.ToArray();
var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length);
Array.Reverse(funcBytesLengthAsBytes);
commandPayloadBytesList.Add(funcBytesLengthAsBytes);
commandPayloadBytesList.Add(funcBytes);
return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray());
}
