public void TestInvalidChainingArrow() { var func1 = new ArrowWorkerFunction( new ArrowUdfWrapper <Int32Array, StringArray, StringArray>( (numbers, strings) => (StringArray)ToArrowArray( Enumerable.Range(0, strings.Length) .Select(i => $"{strings.GetString(i)}:{numbers.Values[i]}") .ToArray())).Execute); var func2 = new ArrowWorkerFunction( new ArrowUdfWrapper <StringArray, StringArray>( (strings) => (StringArray)ToArrowArray( Enumerable.Range(0, strings.Length) .Select(i => $"outer1:{strings.GetString(i)}") .ToArray())).Execute); IArrowArray[] input = new[] { ToArrowArray(new[] { 100 }), ToArrowArray(new[] { "name" }) }; // The order does not align since workerFunction2 is executed first. ArrowWorkerFunction chainedFunc1 = ArrowWorkerFunction.Chain(func2, func1); Assert.ThrowsAny <Exception>(() => chainedFunc1.Func(input, new[] { 0, 1 })); }
public void TestChainingArrowWorkerFunction() { var func1 = new ArrowWorkerFunction( new ArrowUdfWrapper <int, string, string>( (number, str) => $"{str}:{number}").Execute); var func2 = new ArrowWorkerFunction( new ArrowUdfWrapper <string, string>( (str) => $"outer1:{str}").Execute); var func3 = new ArrowWorkerFunction( new ArrowUdfWrapper <string, string>( (str) => $"outer2:{str}").Execute); Apache.Arrow.IArrowArray[] input = new[] { ToArrowArray(new[] { 100 }), ToArrowArray(new[] { "name" }) }; // Validate one-level chaining. var chainedFunc1 = ArrowWorkerFunction.Chain(func1, func2); ArrowTestUtils.AssertEquals( "outer1:name:100", chainedFunc1.Func(0, input, new[] { 0, 1 })); // Validate two-level chaining. var chainedFunc2 = ArrowWorkerFunction.Chain(chainedFunc1, func3); ArrowTestUtils.AssertEquals( "outer2:outer1:name:100", chainedFunc2.Func(0, input, new[] { 0, 1 })); }
public void TestChainingArrowWorkerFunction() { var func1 = new ArrowWorkerFunction( new ArrowUdfWrapper <Int32Array, StringArray, StringArray>( (numbers, strings) => (StringArray)ToArrowArray( Enumerable.Range(0, strings.Length) .Select(i => $"{strings.GetString(i)}:{numbers.Values[i]}") .ToArray())).Execute); var func2 = new ArrowWorkerFunction( new ArrowUdfWrapper <StringArray, StringArray>( (strings) => (StringArray)ToArrowArray( Enumerable.Range(0, strings.Length) .Select(i => $"outer1:{strings.GetString(i)}") .ToArray())).Execute); var func3 = new ArrowWorkerFunction( new ArrowUdfWrapper <StringArray, StringArray>( (strings) => (StringArray)ToArrowArray( Enumerable.Range(0, strings.Length) .Select(i => $"outer2:{strings.GetString(i)}") .ToArray())).Execute); var input = new IArrowArray[] { ToArrowArray(new[] { 100 }), ToArrowArray(new[] { "name" }) }; // Validate one-level chaining. ArrowWorkerFunction chainedFunc1 = ArrowWorkerFunction.Chain(func1, func2); AssertEquals( "outer1:name:100", chainedFunc1.Func(input, new[] { 0, 1 })); // Validate two-level chaining. ArrowWorkerFunction chainedFunc2 = ArrowWorkerFunction.Chain(chainedFunc1, func3); AssertEquals( "outer2:outer1:name:100", chainedFunc2.Func(input, new[] { 0, 1 })); }
public void TestInvalidChainingArrow() { var func1 = new ArrowWorkerFunction( new ArrowUdfWrapper <int, string, string>( (number, str) => $"{str}:{number}").Execute); var func2 = new ArrowWorkerFunction( new ArrowUdfWrapper <string, string>( (str) => $"outer1:{str}").Execute); Apache.Arrow.IArrowArray[] input = new[] { ToArrowArray(new[] { 100 }), ToArrowArray(new[] { "name" }) }; // The order does not align since workerFunction2 is executed first. var chainedFunc1 = ArrowWorkerFunction.Chain(func2, func1); Assert.ThrowsAny <Exception>(() => chainedFunc1.Func(0, input, new[] { 0, 1 })); }
/// <summary> /// Read SqlCommands from the stream. /// </summary> /// <param name="stream">Stream to read from</param> /// <param name="evalType">Evaluation type for the current commands</param> /// <returns>SqlCommand objects</returns> private static SqlCommand[] ReadSqlCommands( PythonEvalType evalType, Stream stream) { int numUdfs = SerDe.ReadInt32(stream); var commands = new SqlCommand[numUdfs]; for (int i = 0; i < numUdfs; ++i) { var command = new SqlCommand(); int numArgsOffsets = SerDe.ReadInt32(stream); command.ArgOffsets = new int[numArgsOffsets]; for (int argIndex = 0; argIndex < numArgsOffsets; ++argIndex) { command.ArgOffsets[argIndex] = SerDe.ReadInt32(stream); } command.NumChainedFunctions = SerDe.ReadInt32(stream); for (int funcIndex = 0; funcIndex < command.NumChainedFunctions; ++funcIndex) { int commandBytesCount = SerDe.ReadInt32(stream); if (commandBytesCount > 0) { CommandSerDe.SerializedMode serializerMode; CommandSerDe.SerializedMode deserializerMode; if (evalType == PythonEvalType.SQL_SCALAR_PANDAS_UDF) { var curWorkerFunction = new ArrowWorkerFunction( CommandSerDe.Deserialize <ArrowWorkerFunction.ExecuteDelegate>( stream, out serializerMode, out deserializerMode, out string runMode)); command.WorkerFunction = (command.WorkerFunction == null) ? curWorkerFunction : ArrowWorkerFunction.Chain( (ArrowWorkerFunction)command.WorkerFunction, curWorkerFunction); } else if (evalType == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF) { if ((numUdfs != 1) || (command.WorkerFunction != null)) { throw new InvalidDataException( "Grouped map UDFs do not support combining multiple UDFs"); } command.WorkerFunction = new ArrowGroupedMapWorkerFunction( CommandSerDe.Deserialize <ArrowGroupedMapWorkerFunction.ExecuteDelegate>( stream, out serializerMode, out deserializerMode, out string runMode)); } else { var curWorkerFunction = new PicklingWorkerFunction( CommandSerDe.Deserialize <PicklingWorkerFunction.ExecuteDelegate>( stream, out serializerMode, out deserializerMode, out string runMode)); command.WorkerFunction = (command.WorkerFunction == null) ? curWorkerFunction : PicklingWorkerFunction.Chain( (PicklingWorkerFunction)command.WorkerFunction, curWorkerFunction); } command.SerializerMode = serializerMode; command.DeserializerMode = deserializerMode; } else { throw new InvalidDataException( $"Invalid command size: {commandBytesCount}"); } } commands[i] = command; } return(commands); }