/// <summary> /// Executes the commands on the input data read from input stream /// and writes results to the output stream. /// </summary> /// <param name="inputStream">Input stream to read data from</param> /// <param name="outputStream">Output stream to write results to</param> /// <param name="splitIndex">Split index for this task</param> /// <param name="command">Contains the commands to execute</param> /// <returns>Statistics captured during the Execute() run</returns> internal CommandExecutorStat Execute( Stream inputStream, Stream outputStream, int splitIndex, RDDCommand command) { var stat = new CommandExecutorStat(); CommandSerDe.SerializedMode serializerMode = command.SerializerMode; CommandSerDe.SerializedMode deserializerMode = command.DeserializerMode; RDD.WorkerFunction.ExecuteDelegate func = command.WorkerFunction.Func; foreach (object output in func( splitIndex, GetInputIterator(inputStream, deserializerMode))) { WriteOutput(outputStream, serializerMode, output); ++stat.NumEntriesProcessed; } return(stat); }
public void TestRDDCommandExecutor() { int mapUdf(int a) => a + 3; var command = new RDDCommand() { WorkerFunction = new RDD.WorkerFunction( new RDD <int> .MapUdfWrapper <int, int>(mapUdf).Execute), SerializerMode = CommandSerDe.SerializedMode.Byte, DeserializerMode = CommandSerDe.SerializedMode.Byte }; var commandPayload = new Worker.CommandPayload() { EvalType = UdfUtils.PythonEvalType.NON_UDF, Commands = new[] { command } }; using (var inputStream = new MemoryStream()) using (var outputStream = new MemoryStream()) { // Write test data to the input stream. var formatter = new BinaryFormatter(); var memoryStream = new MemoryStream(); var inputs = new[] { 0, 1, 2, 3, 4 }; var values = new List <byte[]>(); foreach (int input in inputs) { memoryStream.Position = 0; formatter.Serialize(memoryStream, input); values.Add(memoryStream.ToArray()); } foreach (byte[] value in values) { SerDe.Write(inputStream, value.Length); SerDe.Write(inputStream, value); } SerDe.Write(inputStream, (int)SpecialLengths.END_OF_DATA_SECTION); inputStream.Seek(0, SeekOrigin.Begin); // Execute the command. CommandExecutorStat stat = new CommandExecutor().Execute( inputStream, outputStream, 0, commandPayload); // Validate all the data on the stream is read. Assert.Equal(inputStream.Length, inputStream.Position); Assert.Equal(5, stat.NumEntriesProcessed); // Validate the output stream. outputStream.Seek(0, SeekOrigin.Begin); for (int i = 0; i < inputs.Length; ++i) { Assert.True(SerDe.ReadInt32(outputStream) > 0); Assert.Equal( mapUdf(i), formatter.Deserialize(outputStream)); } // Validate all the data on the stream is read. Assert.Equal(outputStream.Length, outputStream.Position); } }