public object Deserialize(Stream stream, int length) { MaxLengthReadStream readStream = s_slicedReadStream ?? (s_slicedReadStream = new MaxLengthReadStream()); readStream.Reset(stream, length); return(_formater.Deserialize(stream)); }
/// <summary> /// Collects pickled row objects from the given socket. /// </summary> /// <param name="socket">Socket the get the stream from</param> /// <returns>Collection of row objects</returns> public IEnumerable <Row> Collect(ISocketWrapper socket) { Stream inputStream = socket.InputStream; int?length; while (((length = SerDe.ReadBytesLength(inputStream)) != null) && (length.GetValueOrDefault() > 0)) { MaxLengthReadStream readStream = s_slicedReadStream ?? (s_slicedReadStream = new MaxLengthReadStream()); readStream.Reset(inputStream, length.GetValueOrDefault()); var unpickledObjects = PythonSerDe.GetUnpickledObjects(readStream); foreach (object unpickled in unpickledObjects) { yield return((unpickled as RowConstructor).GetRow()); } } }
protected override CommandExecutorStat ExecuteCore( Stream inputStream, Stream outputStream, SqlCommand[] commands) { var stat = new CommandExecutorStat(); ICommandRunner commandRunner = CreateCommandRunner(commands); // On the Spark side, each object in the following List<> is considered as a row. // See the ICommandRunner comments above for the types for a row. var outputRows = new List <object>(); // If the input is empty (no rows) or all rows have been read, then // SpecialLengths.END_OF_DATA_SECTION is sent as the messageLength. // For example, no rows: // +---+----+ // |age|name| // +---+----+ // +---+----+ int messageLength = 0; while ((messageLength = SerDe.ReadInt32(inputStream)) != (int)SpecialLengths.END_OF_DATA_SECTION) { if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL)) { if (messageLength <= 0) { throw new InvalidDataException( $"Invalid message length: {messageLength}"); } MaxLengthReadStream readStream = s_slicedReadStream ?? (s_slicedReadStream = new MaxLengthReadStream()); readStream.Reset(inputStream, messageLength); // Each row in inputRows is of type object[]. If a null is present in a row // then the corresponding index column of the row object[] will be set to null. // For example, (inputRows.Length == 2) and (inputRows[0][0] == null) // +----+ // | age| // +----+ // |null| // | 11| // +----+ object[] inputRows = PythonSerDe.GetUnpickledObjects(readStream); for (int i = 0; i < inputRows.Length; ++i) { // Split id is not used for SQL UDFs, so 0 is passed. outputRows.Add(commandRunner.Run(0, inputRows[i])); } WriteOutput(outputStream, outputRows); stat.NumEntriesProcessed += inputRows.Length; outputRows.Clear(); } } return(stat); }