/// <summary>
/// Writes the given message to the stream.
/// </summary>
/// <param name="stream">Stream to write to</param>
/// <param name="serializerMode">Mode for serialization</param>
/// <param name="message">Message to write to</param>
private void WriteOutput(
Stream stream,
CommandSerDe.SerializedMode serializerMode,
object message)
{
MemoryStream writeOutputStream = s_writeOutputStream ??= new MemoryStream();
writeOutputStream.Position = 0;
Serialize(serializerMode, message, writeOutputStream);
SerDe.Write(stream, (int)writeOutputStream.Position);
SerDe.Write(stream, writeOutputStream.GetBuffer(), (int)writeOutputStream.Position);
}
/// <summary>
/// Create input iterator from the given input stream.
/// </summary>
/// <param name="inputStream">Stream to read from</param>
/// <param name="deserializerMode">Mode for deserialization</param>
/// <returns></returns>
private IEnumerable <object> GetInputIterator(
Stream inputStream,
CommandSerDe.SerializedMode deserializerMode)
{
RDD.Collector.IDeserializer deserializer =
RDD.Collector.GetDeserializer(deserializerMode);
var messageLength = 0;
while ((messageLength = SerDe.ReadInt32(inputStream)) !=
(int)SpecialLengths.END_OF_DATA_SECTION)
{
if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL))
{
yield return(deserializer.Deserialize(inputStream, messageLength));
}
}
}
/// <summary>
/// Serialize a row based on the given serializer mode.
/// </summary>
/// <param name="serializerMode"></param>
/// <param name="message"></param>
/// <param name="stream"></param>
private void Serialize(
CommandSerDe.SerializedMode serializerMode,
object message,
MemoryStream stream)
{
switch (serializerMode)
{
case CommandSerDe.SerializedMode.Byte:
BinaryFormatter formatter = s_binaryFormatter ??= new BinaryFormatter();
formatter.Serialize(stream, message);
break;
case CommandSerDe.SerializedMode.None:
case CommandSerDe.SerializedMode.String:
case CommandSerDe.SerializedMode.Pair:
default:
throw new NotImplementedException(
$"Unsupported serializerMode: {serializerMode}");
}
}
/// <summary>
/// Executes the commands on the input data read from input stream
/// and writes results to the output stream.
/// </summary>
/// <param name="inputStream">Input stream to read data from</param>
/// <param name="outputStream">Output stream to write results to</param>
/// <param name="splitIndex">Split index for this task</param>
/// <param name="command">Contains the commands to execute</param>
/// <returns>Statistics captured during the Execute() run</returns>
internal CommandExecutorStat Execute(
Stream inputStream,
Stream outputStream,
int splitIndex,
RDDCommand command)
{
var stat = new CommandExecutorStat();
CommandSerDe.SerializedMode serializerMode = command.SerializerMode;
CommandSerDe.SerializedMode deserializerMode = command.DeserializerMode;
RDD.WorkerFunction.ExecuteDelegate func = command.WorkerFunction.Func;
foreach (object output in func(
splitIndex,
GetInputIterator(inputStream, deserializerMode)))
{
WriteOutput(outputStream, serializerMode, output);
++stat.NumEntriesProcessed;
}
return(stat);
}