/// <summary>
/// Registers the given delegate as a user-defined function with the specified name.
/// </summary>
/// <typeparam name="T1">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="T2">Specifies the type of the second argument to the UDF.</typeparam>
/// <typeparam name="T3">Specifies the type of the third argument to the UDF.</typeparam>
/// <typeparam name="T4">Specifies the type of the fourth argument to the UDF.</typeparam>
/// <typeparam name="T5">Specifies the type of the fifth argument to the UDF.</typeparam>
/// <typeparam name="T6">Specifies the type of the sixth argument to the UDF.</typeparam>
/// <typeparam name="T7">Specifies the type of the seventh argument to the UDF.</typeparam>
/// <typeparam name="T8">Specifies the type of the eighth argument to the UDF.</typeparam>
/// <typeparam name="T9">Specifies the type of the ninth argument to the UDF.</typeparam>
/// <typeparam name="T10">Specifies the type of the tenth argument to the UDF.</typeparam>
/// <param name="name">The UDF name.</param>
/// <param name="f">The UDF function implementation.</param>
/// <param name="returnType">Schema associated with this row</param>
public void Register <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>(
string name,
Func <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, Row> f,
StructType returnType)
{
Register(name, UdfUtils.CreateUdfWrapper(f), returnType);
}
private JvmObjectReference GetDataType <T>()
{
return((JvmObjectReference)_jvmObject.Jvm.CallStaticJavaMethod(
"org.apache.spark.sql.types.DataType",
"fromJson",
$"{UdfUtils.GetReturnType(typeof(T))}"));
}
/// <summary>
/// Registers the given delegate as a user-defined function with the specified name.
/// </summary>
/// <typeparam name="T1">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="T2">Specifies the type of the second argument to the UDF.</typeparam>
/// <typeparam name="T3">Specifies the type of the third argument to the UDF.</typeparam>
/// <param name="name">The UDF name.</param>
/// <param name="f">The UDF function implementation.</param>
/// <param name="returnType">Schema associated with this row</param>
public void Register <T1, T2, T3>(
string name,
Func <T1, T2, T3, Row> f,
StructType returnType)
{
Register(name, UdfUtils.CreateUdfWrapper(f), returnType);
}
/// <summary>
/// Registers the given delegate as a vector user-defined function with the specified name.
/// </summary>
/// <typeparam name="T">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="TResult">Specifies the return type of the UDF.</typeparam>
/// <param name="udf">The <see cref="UdfRegistration"/> object to invoke the register the Vector UDF.</param>
/// <param name="name">The UDF name.</param>
/// <param name="f">The UDF function implementation.</param>
public static void RegisterVector <T, TResult>(
this UdfRegistration udf, string name, Func <T, TResult> f)
where T : IArrowArray
where TResult : IArrowArray
{
RegisterVector <TResult>(udf, name, UdfUtils.CreateVectorUdfWrapper(f));
}
/// <summary>Creates a Vector UDF from the specified delegate.</summary>
/// <typeparam name="T">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="TResult">Specifies the return type of the UDF.</typeparam>
/// <param name="udf">The Vector UDF function implementation.</param>
/// <returns>
/// A delegate that returns a <see cref="Column"/> for the result of the Vector UDF.
/// </returns>
public static Func <Column, Column> VectorUdf <T, TResult>(Func <T, TResult> udf)
where T : IArrowArray
where TResult : IArrowArray
{
return(Functions.CreateVectorUdf <TResult>(
udf.Method.ToString(),
UdfUtils.CreateVectorUdfWrapper(udf)).Apply1);
}
/// <summary>
/// Registers the given delegate as a vector user-defined function with the specified name.
/// </summary>
/// <typeparam name="T1">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="T2">Specifies the type of the second argument to the UDF.</typeparam>
/// <typeparam name="T3">Specifies the type of the third argument to the UDF.</typeparam>
/// <typeparam name="T4">Specifies the type of the fourth argument to the UDF.</typeparam>
/// <typeparam name="T5">Specifies the type of the fifth argument to the UDF.</typeparam>
/// <typeparam name="TResult">Specifies the return type of the UDF.</typeparam>
/// <param name="udf">The <see cref="UdfRegistration"/> object to invoke the register the Vector UDF.</param>
/// <param name="name">The UDF name.</param>
/// <param name="f">The UDF function implementation.</param>
public static void RegisterVector <T1, T2, T3, T4, T5, TResult>(
this UdfRegistration udf, string name, Func <T1, T2, T3, T4, T5, TResult> f)
where T1 : IArrowArray
where T2 : IArrowArray
where T3 : IArrowArray
where T4 : IArrowArray
where T5 : IArrowArray
where TResult : IArrowArray
{
RegisterVector <TResult>(udf, name, UdfUtils.CreateVectorUdfWrapper(f));
}
/// <summary>Creates a Vector UDF from the specified delegate.</summary>
/// <typeparam name="T1">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="T2">Specifies the type of the second argument to the UDF.</typeparam>
/// <typeparam name="T3">Specifies the type of the third argument to the UDF.</typeparam>
/// <typeparam name="T4">Specifies the type of the fourth argument to the UDF.</typeparam>
/// <typeparam name="T5">Specifies the type of the fifth argument to the UDF.</typeparam>
/// <typeparam name="TResult">Specifies the return type of the UDF.</typeparam>
/// <param name="udf">The Vector UDF function implementation.</param>
/// <returns>
/// A delegate that returns a <see cref="Column"/> for the result of the Vector UDF.
/// </returns>
public static Func <Column, Column, Column, Column, Column, Column> VectorUdf <T1, T2, T3, T4, T5, TResult>(
Func <T1, T2, T3, T4, T5, TResult> udf)
where T1 : IArrowArray
where T2 : IArrowArray
where T3 : IArrowArray
where T4 : IArrowArray
where T5 : IArrowArray
where TResult : IArrowArray
{
return(Functions.CreateVectorUdf <TResult>(
udf.Method.ToString(),
UdfUtils.CreateVectorUdfWrapper(udf)).Apply5);
}
/// <summary>
/// Helper function to register wrapped udf.
/// </summary>
/// <typeparam name="TResult">Return type of the udf</typeparam>
/// <param name="name">Name of the udf</param>
/// <param name="func">Wrapped UDF function</param>
/// <param name="evalType">The EvalType of the function.</param>
internal void Register <TResult>(string name, Delegate func, UdfUtils.PythonEvalType evalType)
{
byte[] command = CommandSerDe.Serialize(
func,
CommandSerDe.SerializedMode.Row,
CommandSerDe.SerializedMode.Row);
var udf = UserDefinedFunction.Create(
_jvmObject.Jvm,
name,
command,
evalType,
UdfUtils.GetReturnType(typeof(TResult)));
_jvmObject.Invoke("registerPython", name, udf);
}
internal static UserDefinedFunction Create(
IJvmBridge jvm,
string name,
byte[] command,
UdfUtils.PythonEvalType evalType,
string returnType)
{
return(new UserDefinedFunction(
jvm.CallConstructor(
"org.apache.spark.sql.execution.python.UserDefinedPythonFunction",
name,
UdfUtils.CreatePythonFunction(jvm, command),
DataType.FromJson(jvm, returnType),
(int)evalType,
true // udfDeterministic
)));
}
public DataStreamWriter Foreach(IForeachWriter writer)
{
RDD.WorkerFunction.ExecuteDelegate wrapper =
new ForeachWriterWrapperUdfWrapper(
new ForeachWriterWrapper(writer).Process).Execute;
_jvmObject.Invoke(
"foreach",
_jvmObject.Jvm.CallConstructor(
"org.apache.spark.sql.execution.python.PythonForeachWriter",
UdfUtils.CreatePythonFunction(
_jvmObject.Jvm,
CommandSerDe.Serialize(
wrapper,
CommandSerDe.SerializedMode.Row,
CommandSerDe.SerializedMode.Row)),
DataType.FromJson(_jvmObject.Jvm, _df.Schema().Json)));
return(this);
}
/// <summary>
/// Helper function to register wrapped udf.
/// </summary>
/// <typeparam name="TResult">Return type of the udf</typeparam>
/// <param name="name">Name of the udf</param>
/// <param name="func">Wrapped UDF function</param>
private void Register <TResult>(string name, Delegate func)
{
byte[] command = CommandSerDe.Serialize(
func,
CommandSerDe.SerializedMode.Row,
CommandSerDe.SerializedMode.Row);
JvmObjectReference pythonFunction =
UdfUtils.CreatePythonFunction(_jvmObject.Jvm, command);
var udf = new UserDefinedFunction(
_jvmObject.Jvm.CallConstructor(
"org.apache.spark.sql.execution.python.UserDefinedPythonFunction",
name,
pythonFunction,
GetDataType <TResult>(),
(int)UdfUtils.GetPythonEvalType(),
true // udfDeterministic
));
_jvmObject.Invoke("registerPython", name, udf);
}
internal static UserDefinedFunction Create(
IJvmBridge jvm,
string name,
byte[] command,
UdfUtils.PythonEvalType evalType,
string returnType)
{
var pythonFunction = UdfUtils.CreatePythonFunction(jvm, command);
var dataType = (JvmObjectReference)jvm.CallStaticJavaMethod(
"org.apache.spark.sql.types.DataType",
"fromJson",
$"{returnType}");
return(new UserDefinedFunction(
jvm.CallConstructor(
"org.apache.spark.sql.execution.python.UserDefinedPythonFunction",
name,
pythonFunction,
dataType,
(int)evalType,
true // udfDeterministic
)));
}
/// <summary>
/// Registers the given delegate as a user-defined function with the specified name.
/// </summary>
/// <typeparam name="T1">Specifies the type of the first argument to the UDF.</typeparam>
/// <typeparam name="T2">Specifies the type of the second argument to the UDF.</typeparam>
/// <typeparam name="T3">Specifies the type of the third argument to the UDF.</typeparam>
/// <typeparam name="T4">Specifies the type of the fourth argument to the UDF.</typeparam>
/// <typeparam name="TResult">Specifies the return type of the UDF.</typeparam>
/// <param name="name">The UDF name.</param>
/// <param name="f">The UDF function implementation.</param>
public void Register <T1, T2, T3, T4, TResult>(string name, Func <T1, T2, T3, T4, TResult> f)
{
Register <TResult>(name, UdfUtils.CreateUdfWrapper(f));
}
private JvmObjectReference GetDataType <T>()
{
return(DataType.FromJson(_jvmObject.Jvm, UdfUtils.GetReturnType(typeof(T))));
}
/// <summary>
/// Helper function to register wrapped udf.
/// </summary>
/// <typeparam name="TResult">Return type of the udf</typeparam>
/// <param name="name">Name of the udf</param>
/// <param name="func">Wrapped UDF function</param>
/// <param name="evalType">The EvalType of the function</param>
internal void Register <TResult>(string name, Delegate func, UdfUtils.PythonEvalType evalType)
{
Register(name, func, evalType, UdfUtils.GetReturnType(typeof(TResult)));
}
