Пример #1
0
        public void TestChainingPicklingWorkerFunction()
        {
            var func1 = new PicklingWorkerFunction(
                new PicklingUdfWrapper <int, string, string>(
                    (number, str) => $"{str}:{number}").Execute);

            var func2 = new PicklingWorkerFunction(
                new PicklingUdfWrapper <string, string>(
                    (str) => $"outer1:{str}").Execute);

            var func3 = new PicklingWorkerFunction(
                new PicklingUdfWrapper <string, string>(
                    (str) => $"outer2:{str}").Execute);

            object[] input = { 100, "name" };

            // Validate one-level chaining.
            PicklingWorkerFunction chainedFunc1 = PicklingWorkerFunction.Chain(func1, func2);

            Assert.Equal("outer1:name:100", chainedFunc1.Func(0, input, new[] { 0, 1 }));

            // Validate two-level chaining.
            PicklingWorkerFunction chainedFunc2 = PicklingWorkerFunction.Chain(chainedFunc1, func3);

            Assert.Equal("outer2:outer1:name:100", chainedFunc2.Func(0, input, new[] { 0, 1 }));
        }
Пример #2
0
        public void TestInvalidChainingPickling()
        {
            var func1 = new PicklingWorkerFunction(
                new PicklingUdfWrapper <int, string, string>(
                    (number, str) => $"{str}:{number}").Execute);

            var func2 = new PicklingWorkerFunction(
                new PicklingUdfWrapper <string, string>(
                    (str) => $"outer1:{str}").Execute);

            object[] input = { 100, "name" };

            // The order does not align since workerFunction2 is executed first.
            PicklingWorkerFunction chainedFunc1 = PicklingWorkerFunction.Chain(func2, func1);

            Assert.ThrowsAny <Exception>(() => chainedFunc1.Func(0, input, new[] { 0, 1 }));
        }
Пример #3
0
        /// <summary>
        /// Read SqlCommands from the stream.
        /// </summary>
        /// <param name="stream">Stream to read from</param>
        /// <param name="evalType">Evaluation type for the current commands</param>
        /// <returns>SqlCommand objects</returns>
        private static SqlCommand[] ReadSqlCommands(
            PythonEvalType evalType,
            Stream stream)
        {
            int numUdfs  = SerDe.ReadInt32(stream);
            var commands = new SqlCommand[numUdfs];

            for (int i = 0; i < numUdfs; ++i)
            {
                var command = new SqlCommand();

                int numArgsOffsets = SerDe.ReadInt32(stream);
                command.ArgOffsets = new int[numArgsOffsets];
                for (int argIndex = 0; argIndex < numArgsOffsets; ++argIndex)
                {
                    command.ArgOffsets[argIndex] = SerDe.ReadInt32(stream);
                }

                command.NumChainedFunctions = SerDe.ReadInt32(stream);
                for (int funcIndex = 0; funcIndex < command.NumChainedFunctions; ++funcIndex)
                {
                    int commandBytesCount = SerDe.ReadInt32(stream);
                    if (commandBytesCount > 0)
                    {
                        CommandSerDe.SerializedMode serializerMode;
                        CommandSerDe.SerializedMode deserializerMode;
                        if (evalType == PythonEvalType.SQL_SCALAR_PANDAS_UDF)
                        {
                            var curWorkerFunction = new ArrowWorkerFunction(
                                CommandSerDe.Deserialize <ArrowWorkerFunction.ExecuteDelegate>(
                                    stream,
                                    out serializerMode,
                                    out deserializerMode,
                                    out string runMode));

                            command.WorkerFunction = (command.WorkerFunction == null) ?
                                                     curWorkerFunction :
                                                     ArrowWorkerFunction.Chain(
                                (ArrowWorkerFunction)command.WorkerFunction,
                                curWorkerFunction);
                        }
                        else if (evalType == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF)
                        {
                            if ((numUdfs != 1) || (command.WorkerFunction != null))
                            {
                                throw new InvalidDataException(
                                          "Grouped map UDFs do not support combining multiple UDFs");
                            }

                            command.WorkerFunction = new ArrowGroupedMapWorkerFunction(
                                CommandSerDe.Deserialize <ArrowGroupedMapWorkerFunction.ExecuteDelegate>(
                                    stream,
                                    out serializerMode,
                                    out deserializerMode,
                                    out string runMode));
                        }
                        else
                        {
                            var curWorkerFunction = new PicklingWorkerFunction(
                                CommandSerDe.Deserialize <PicklingWorkerFunction.ExecuteDelegate>(
                                    stream,
                                    out serializerMode,
                                    out deserializerMode,
                                    out string runMode));

                            command.WorkerFunction = (command.WorkerFunction == null) ?
                                                     curWorkerFunction :
                                                     PicklingWorkerFunction.Chain(
                                (PicklingWorkerFunction)command.WorkerFunction,
                                curWorkerFunction);
                        }

                        command.SerializerMode   = serializerMode;
                        command.DeserializerMode = deserializerMode;
                    }
                    else
                    {
                        throw new InvalidDataException(
                                  $"Invalid command size: {commandBytesCount}");
                    }
                }

                commands[i] = command;
            }

            return(commands);
        }