/// <summary>
/// Executes the commands on the input data read from input stream
/// and writes results to the output stream.
/// </summary>
/// <param name="inputStream">Input stream to read data from</param>
/// <param name="outputStream">Output stream to write results to</param>
/// <param name="splitIndex">Split index for this task</param>
/// <param name="command">Contains the commands to execute</param>
/// <returns>Statistics captured during the Execute() run</returns>
internal CommandExecutorStat Execute(
Stream inputStream,
Stream outputStream,
int splitIndex,
RDDCommand command)
{
var stat = new CommandExecutorStat();
CommandSerDe.SerializedMode serializerMode = command.SerializerMode;
CommandSerDe.SerializedMode deserializerMode = command.DeserializerMode;
RDD.WorkerFunction.ExecuteDelegate func = command.WorkerFunction.Func;
foreach (object output in func(
splitIndex,
GetInputIterator(inputStream, deserializerMode)))
{
WriteOutput(outputStream, serializerMode, output);
++stat.NumEntriesProcessed;
}
return(stat);
}
public void TestRDDCommandExecutor()
{
int mapUdf(int a) => a + 3;
var command = new RDDCommand()
{
WorkerFunction = new RDD.WorkerFunction(
new RDD <int> .MapUdfWrapper <int, int>(mapUdf).Execute),
SerializerMode = CommandSerDe.SerializedMode.Byte,
DeserializerMode = CommandSerDe.SerializedMode.Byte
};
var commandPayload = new Worker.CommandPayload()
{
EvalType = UdfUtils.PythonEvalType.NON_UDF,
Commands = new[] { command }
};
using (var inputStream = new MemoryStream())
using (var outputStream = new MemoryStream())
{
// Write test data to the input stream.
var formatter = new BinaryFormatter();
var memoryStream = new MemoryStream();
var inputs = new[] { 0, 1, 2, 3, 4 };
var values = new List <byte[]>();
foreach (int input in inputs)
{
memoryStream.Position = 0;
formatter.Serialize(memoryStream, input);
values.Add(memoryStream.ToArray());
}
foreach (byte[] value in values)
{
SerDe.Write(inputStream, value.Length);
SerDe.Write(inputStream, value);
}
SerDe.Write(inputStream, (int)SpecialLengths.END_OF_DATA_SECTION);
inputStream.Seek(0, SeekOrigin.Begin);
// Execute the command.
CommandExecutorStat stat = new CommandExecutor().Execute(
inputStream,
outputStream,
0,
commandPayload);
// Validate all the data on the stream is read.
Assert.Equal(inputStream.Length, inputStream.Position);
Assert.Equal(5, stat.NumEntriesProcessed);
// Validate the output stream.
outputStream.Seek(0, SeekOrigin.Begin);
for (int i = 0; i < inputs.Length; ++i)
{
Assert.True(SerDe.ReadInt32(outputStream) > 0);
Assert.Equal(
mapUdf(i),
formatter.Deserialize(outputStream));
}
// Validate all the data on the stream is read.
Assert.Equal(outputStream.Length, outputStream.Position);
}
}
