/// <summary> /// Apply a function to each RDD in this DStream. /// </summary> /// <param name="f"></param> public void ForeachRDD(Action <double, RDD <dynamic> > f) { var formatter = new BinaryFormatter(); var stream = new MemoryStream(); formatter.Serialize(stream, f); DStreamProxy.CallForeachRDD(stream.ToArray(), serializedMode.ToString()); }
internal static byte[] BuildCommand(object func, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte) { var formatter = new BinaryFormatter(); var stream = new MemoryStream(); formatter.Serialize(stream, func); List <byte[]> commandPayloadBytesList = new List <byte[]>(); // add deserializer mode var modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString()); var length = modeBytes.Length; var lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // add serializer mode modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString()); length = modeBytes.Length; lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // add func var funcBytes = stream.ToArray(); var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length); Array.Reverse(funcBytesLengthAsBytes); commandPayloadBytesList.Add(funcBytesLengthAsBytes); commandPayloadBytesList.Add(funcBytes); return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray()); }
internal static byte[] BuildCommand(CSharpWorkerFunc workerFunc, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte) { var formatter = new BinaryFormatter(); var stream = new MemoryStream(); formatter.Serialize(stream, workerFunc); List <byte[]> commandPayloadBytesList = new List <byte[]>(); // reserve 12 bytes for RddId, stageId and partitionId, this info will be filled in CSharpRDD.scala byte[] rddInfo = new byte[12]; for (int i = 0; i < rddInfo.Length; i++) { rddInfo[i] = 0; } commandPayloadBytesList.Add(rddInfo); // add deserializer mode var modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString()); var length = modeBytes.Length; var lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // add serializer mode modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString()); length = modeBytes.Length; lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // add func var funcBytes = stream.ToArray(); var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length); Array.Reverse(funcBytesLengthAsBytes); commandPayloadBytesList.Add(funcBytesLengthAsBytes); commandPayloadBytesList.Add(funcBytes); return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray()); }
internal static byte[] Serialize( Delegate func, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte) { // TODO: Rework on the following List<Byte[]> to use MemoryStream! var commandPayloadBytesList = new List <byte[]>(); // Add serializer mode. byte[] modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString()); int length = modeBytes.Length; byte[] lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // Add deserializer mode. modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString()); length = modeBytes.Length; lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // Add run mode: // N - normal // R - repl string runMode = Environment.GetEnvironmentVariable("SPARK_NET_RUN_MODE") ?? "N"; byte[] runModeBytes = Encoding.UTF8.GetBytes(runMode); lengthAsBytes = BitConverter.GetBytes(runModeBytes.Length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(runModeBytes); if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase)) { // add compilation dump directory byte[] compilationDumpDirBytes = Encoding.UTF8.GetBytes( Environment.GetEnvironmentVariable("SPARK_NET_SCRIPT_COMPILATION_DIR") ?? "."); lengthAsBytes = BitConverter.GetBytes(compilationDumpDirBytes.Length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(compilationDumpDirBytes); } // Serialize the UDFs. var udfWrapperNodes = new List <UdfWrapperNode>(); var udfs = new List <UdfSerDe.UdfData>(); SerializeUdfs(func, null, udfWrapperNodes, udfs); // Run through UdfSerDe.Serialize once more to get serialization info // on the actual UDF. var udfWrapperData = new UdfWrapperData() { UdfWrapperNodes = udfWrapperNodes.ToArray(), Udfs = udfs.ToArray() }; var formatter = new BinaryFormatter(); using (var stream = new MemoryStream()) { formatter.Serialize(stream, udfWrapperData); byte[] udfBytes = stream.ToArray(); byte[] udfBytesLengthAsBytes = BitConverter.GetBytes(udfBytes.Length); Array.Reverse(udfBytesLengthAsBytes); commandPayloadBytesList.Add(udfBytesLengthAsBytes); commandPayloadBytesList.Add(udfBytes); } return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray()); }
internal static byte[] BuildCommand(CSharpWorkerFunc workerFunc, SerializedMode deserializerMode = SerializedMode.Byte, SerializedMode serializerMode = SerializedMode.Byte) { var formatter = new BinaryFormatter(); var stream = new MemoryStream(); formatter.Serialize(stream, workerFunc); List <byte[]> commandPayloadBytesList = new List <byte[]>(); // reserve 12 bytes for RddId, stageId and partitionId, this info will be filled in CSharpRDD.scala byte[] rddInfo = new byte[12]; for (int i = 0; i < rddInfo.Length; i++) { rddInfo[i] = 0; } commandPayloadBytesList.Add(rddInfo); // add deserializer mode var modeBytes = Encoding.UTF8.GetBytes(deserializerMode.ToString()); var length = modeBytes.Length; var lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // add serializer mode modeBytes = Encoding.UTF8.GetBytes(serializerMode.ToString()); length = modeBytes.Length; lengthAsBytes = BitConverter.GetBytes(length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(modeBytes); // add run mode // N - normal // R - repl var runMode = Environment.GetEnvironmentVariable("SPARKCLR_RUN_MODE") ?? "N"; var runModeBytes = Encoding.UTF8.GetBytes(runMode); lengthAsBytes = BitConverter.GetBytes(runModeBytes.Length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(runModeBytes); if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase)) { // add compilation dump directory var compilationDumpDirBytes = Encoding.UTF8.GetBytes(Environment.GetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR") ?? "."); lengthAsBytes = BitConverter.GetBytes(compilationDumpDirBytes.Length); Array.Reverse(lengthAsBytes); commandPayloadBytesList.Add(lengthAsBytes); commandPayloadBytesList.Add(compilationDumpDirBytes); } // add func var funcBytes = stream.ToArray(); var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length); Array.Reverse(funcBytesLengthAsBytes); commandPayloadBytesList.Add(funcBytesLengthAsBytes); commandPayloadBytesList.Add(funcBytes); return(commandPayloadBytesList.SelectMany(byteArray => byteArray).ToArray()); }