public object Deserialize(Stream stream, int length)
{
MaxLengthReadStream readStream = s_slicedReadStream ??
(s_slicedReadStream = new MaxLengthReadStream());
readStream.Reset(stream, length);
return(_formater.Deserialize(stream));
}
/// <summary>
/// Collects pickled row objects from the given socket.
/// </summary>
/// <param name="socket">Socket the get the stream from</param>
/// <returns>Collection of row objects</returns>
public IEnumerable <Row> Collect(ISocketWrapper socket)
{
Stream inputStream = socket.InputStream;
int?length;
while (((length = SerDe.ReadBytesLength(inputStream)) != null) &&
(length.GetValueOrDefault() > 0))
{
MaxLengthReadStream readStream = s_slicedReadStream ??
(s_slicedReadStream = new MaxLengthReadStream());
readStream.Reset(inputStream, length.GetValueOrDefault());
var unpickledObjects = PythonSerDe.GetUnpickledObjects(readStream);
foreach (object unpickled in unpickledObjects)
{
yield return((unpickled as RowConstructor).GetRow());
}
}
}
protected override CommandExecutorStat ExecuteCore(
Stream inputStream,
Stream outputStream,
SqlCommand[] commands)
{
var stat = new CommandExecutorStat();
ICommandRunner commandRunner = CreateCommandRunner(commands);
// On the Spark side, each object in the following List<> is considered as a row.
// See the ICommandRunner comments above for the types for a row.
var outputRows = new List <object>();
// If the input is empty (no rows) or all rows have been read, then
// SpecialLengths.END_OF_DATA_SECTION is sent as the messageLength.
// For example, no rows:
// +---+----+
// |age|name|
// +---+----+
// +---+----+
int messageLength = 0;
while ((messageLength = SerDe.ReadInt32(inputStream)) !=
(int)SpecialLengths.END_OF_DATA_SECTION)
{
if ((messageLength > 0) || (messageLength == (int)SpecialLengths.NULL))
{
if (messageLength <= 0)
{
throw new InvalidDataException(
$"Invalid message length: {messageLength}");
}
MaxLengthReadStream readStream = s_slicedReadStream ??
(s_slicedReadStream = new MaxLengthReadStream());
readStream.Reset(inputStream, messageLength);
// Each row in inputRows is of type object[]. If a null is present in a row
// then the corresponding index column of the row object[] will be set to null.
// For example, (inputRows.Length == 2) and (inputRows[0][0] == null)
// +----+
// | age|
// +----+
// |null|
// | 11|
// +----+
object[] inputRows = PythonSerDe.GetUnpickledObjects(readStream);
for (int i = 0; i < inputRows.Length; ++i)
{
// Split id is not used for SQL UDFs, so 0 is passed.
outputRows.Add(commandRunner.Run(0, inputRows[i]));
}
WriteOutput(outputStream, outputRows);
stat.NumEntriesProcessed += inputRows.Length;
outputRows.Clear();
}
}
return(stat);
}
