/// <summary>
        /// Writes the given message to the stream.
        /// </summary>
        /// <param name="stream">Stream to write to</param>
        /// <param name="serializerMode">Mode for serialization</param>
        /// <param name="message">Message to write to</param>
        private void WriteOutput(
            Stream stream,
            CommandSerDe.SerializedMode serializerMode,
            object message)
        {
            MemoryStream writeOutputStream = s_writeOutputStream ??= new MemoryStream();

            writeOutputStream.Position = 0;
            Serialize(serializerMode, message, writeOutputStream);
            SerDe.Write(stream, (int)writeOutputStream.Position);
            SerDe.Write(stream, writeOutputStream.GetBuffer(), (int)writeOutputStream.Position);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Create input iterator from the given input stream.
        /// </summary>
        /// <param name="inputStream">Stream to read from</param>
        /// <param name="deserializerMode">Mode for deserialization</param>
        /// <returns></returns>
        private IEnumerable <object> GetInputIterator(
            Stream inputStream,
            CommandSerDe.SerializedMode deserializerMode)
        {
            RDD.Collector.IDeserializer deserializer =
                RDD.Collector.GetDeserializer(deserializerMode);

            var messageLength = 0;

            while ((messageLength = SerDe.ReadInt32(inputStream)) !=
                   (int)SpecialLengths.END_OF_DATA_SECTION)
            {
                if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL))
                {
                    yield return(deserializer.Deserialize(inputStream, messageLength));
                }
            }
        }
        /// <summary>
        /// Serialize a row based on the given serializer mode.
        /// </summary>
        /// <param name="serializerMode"></param>
        /// <param name="message"></param>
        /// <param name="stream"></param>
        private void Serialize(
            CommandSerDe.SerializedMode serializerMode,
            object message,
            MemoryStream stream)
        {
            switch (serializerMode)
            {
            case CommandSerDe.SerializedMode.Byte:
                BinaryFormatter formatter = s_binaryFormatter ??= new BinaryFormatter();
                formatter.Serialize(stream, message);
                break;

            case CommandSerDe.SerializedMode.None:
            case CommandSerDe.SerializedMode.String:
            case CommandSerDe.SerializedMode.Pair:
            default:
                throw new NotImplementedException(
                          $"Unsupported serializerMode: {serializerMode}");
            }
        }
Exemplo n.º 4
0
        /// <summary>
        /// Executes the commands on the input data read from input stream
        /// and writes results to the output stream.
        /// </summary>
        /// <param name="inputStream">Input stream to read data from</param>
        /// <param name="outputStream">Output stream to write results to</param>
        /// <param name="splitIndex">Split index for this task</param>
        /// <param name="command">Contains the commands to execute</param>
        /// <returns>Statistics captured during the Execute() run</returns>
        internal CommandExecutorStat Execute(
            Stream inputStream,
            Stream outputStream,
            int splitIndex,
            RDDCommand command)
        {
            var stat = new CommandExecutorStat();

            CommandSerDe.SerializedMode serializerMode   = command.SerializerMode;
            CommandSerDe.SerializedMode deserializerMode = command.DeserializerMode;

            RDD.WorkerFunction.ExecuteDelegate func = command.WorkerFunction.Func;
            foreach (object output in func(
                         splitIndex,
                         GetInputIterator(inputStream, deserializerMode)))
            {
                WriteOutput(outputStream, serializerMode, output);

                ++stat.NumEntriesProcessed;
            }

            return(stat);
        }