/// <summary> /// Writes the given message to the stream. /// </summary> /// <param name="stream">Stream to write to</param> /// <param name="serializerMode">Mode for serialization</param> /// <param name="message">Message to write to</param> private void WriteOutput( Stream stream, CommandSerDe.SerializedMode serializerMode, object message) { MemoryStream writeOutputStream = s_writeOutputStream ??= new MemoryStream(); writeOutputStream.Position = 0; Serialize(serializerMode, message, writeOutputStream); SerDe.Write(stream, (int)writeOutputStream.Position); SerDe.Write(stream, writeOutputStream.GetBuffer(), (int)writeOutputStream.Position); }
/// <summary> /// Create input iterator from the given input stream. /// </summary> /// <param name="inputStream">Stream to read from</param> /// <param name="deserializerMode">Mode for deserialization</param> /// <returns></returns> private IEnumerable <object> GetInputIterator( Stream inputStream, CommandSerDe.SerializedMode deserializerMode) { RDD.Collector.IDeserializer deserializer = RDD.Collector.GetDeserializer(deserializerMode); var messageLength = 0; while ((messageLength = SerDe.ReadInt32(inputStream)) != (int)SpecialLengths.END_OF_DATA_SECTION) { if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL)) { yield return(deserializer.Deserialize(inputStream, messageLength)); } } }
/// <summary> /// Serialize a row based on the given serializer mode. /// </summary> /// <param name="serializerMode"></param> /// <param name="message"></param> /// <param name="stream"></param> private void Serialize( CommandSerDe.SerializedMode serializerMode, object message, MemoryStream stream) { switch (serializerMode) { case CommandSerDe.SerializedMode.Byte: BinaryFormatter formatter = s_binaryFormatter ??= new BinaryFormatter(); formatter.Serialize(stream, message); break; case CommandSerDe.SerializedMode.None: case CommandSerDe.SerializedMode.String: case CommandSerDe.SerializedMode.Pair: default: throw new NotImplementedException( $"Unsupported serializerMode: {serializerMode}"); } }
/// <summary> /// Executes the commands on the input data read from input stream /// and writes results to the output stream. /// </summary> /// <param name="inputStream">Input stream to read data from</param> /// <param name="outputStream">Output stream to write results to</param> /// <param name="splitIndex">Split index for this task</param> /// <param name="command">Contains the commands to execute</param> /// <returns>Statistics captured during the Execute() run</returns> internal CommandExecutorStat Execute( Stream inputStream, Stream outputStream, int splitIndex, RDDCommand command) { var stat = new CommandExecutorStat(); CommandSerDe.SerializedMode serializerMode = command.SerializerMode; CommandSerDe.SerializedMode deserializerMode = command.DeserializerMode; RDD.WorkerFunction.ExecuteDelegate func = command.WorkerFunction.Func; foreach (object output in func( splitIndex, GetInputIterator(inputStream, deserializerMode))) { WriteOutput(outputStream, serializerMode, output); ++stat.NumEntriesProcessed; } return(stat); }