/// <summary>
/// Creates the PythonFunction object on the JVM side wrapping the given command bytes.
/// </summary>
/// <param name="jvm">JVM bridge to use</param>
/// <param name="command">Serialized command bytes</param>
/// <returns>JvmObjectReference object to the PythonFunction object</returns>
internal static JvmObjectReference CreatePythonFunction(IJvmBridge jvm, byte[] command)
{
JvmObjectReference hashTableReference = jvm.CallConstructor("java.util.Hashtable");
JvmObjectReference arrayListReference = jvm.CallConstructor("java.util.ArrayList");
return((JvmObjectReference)jvm.CallStaticJavaMethod(
"org.apache.spark.sql.api.dotnet.SQLUtils",
"createPythonFunction",
command,
hashTableReference, // Environment variables
arrayListReference, // Python includes
SparkEnvironment.ConfigurationService.GetWorkerExePath(),
"1.0",
arrayListReference, // Broadcast variables
null)); // Accumulator
}
internal static UserDefinedFunction Create(
string name,
byte[] command,
UdfUtils.PythonEvalType evalType,
string returnType)
{
IJvmBridge jvm = SparkEnvironment.JvmBridge;
JvmObjectReference hashTableReference = jvm.CallConstructor("java.util.Hashtable");
JvmObjectReference arrayListReference = jvm.CallConstructor("java.util.ArrayList");
var dataType = (JvmObjectReference)jvm.CallStaticJavaMethod(
"org.apache.spark.sql.types.DataType",
"fromJson",
$"\"{returnType}\"");
var pythonFunction = (JvmObjectReference)jvm.CallStaticJavaMethod(
"org.apache.spark.sql.api.dotnet.SQLUtils",
"createPythonFunction",
command,
hashTableReference, // Environment variables
arrayListReference, // Python includes
SparkEnvironment.ConfigurationService.GetWorkerExePath(),
"1.0",
arrayListReference, // Broadcast variables
null); // Accumulator
return(new UserDefinedFunction(
jvm.CallConstructor(
"org.apache.spark.sql.execution.python.UserDefinedPythonFunction",
name,
pythonFunction,
dataType,
(int)evalType,
true // udfDeterministic
)));
}
internal static UserDefinedFunction Create(
IJvmBridge jvm,
string name,
byte[] command,
UdfUtils.PythonEvalType evalType,
string returnType)
{
return(new UserDefinedFunction(
jvm.CallConstructor(
"org.apache.spark.sql.execution.python.UserDefinedPythonFunction",
name,
UdfUtils.CreatePythonFunction(jvm, command),
DataType.FromJson(jvm, returnType),
(int)evalType,
true // udfDeterministic
)));
}
private static JvmObjectReference CreateEnvVarsForPythonFunction(IJvmBridge jvm)
{
JvmObjectReference environmentVars = jvm.CallConstructor("java.util.Hashtable");
string assemblySearchPath = Environment.GetEnvironmentVariable(
AssemblySearchPathResolver.AssemblySearchPathsEnvVarName);
if (!string.IsNullOrEmpty(assemblySearchPath))
{
jvm.CallNonStaticJavaMethod(
environmentVars,
"put",
AssemblySearchPathResolver.AssemblySearchPathsEnvVarName,
assemblySearchPath);
}
return(environmentVars);
}
internal static UserDefinedFunction Create(
IJvmBridge jvm,
string name,
byte[] command,
UdfUtils.PythonEvalType evalType,
string returnType)
{
var pythonFunction = UdfUtils.CreatePythonFunction(jvm, command);
var dataType = (JvmObjectReference)jvm.CallStaticJavaMethod(
"org.apache.spark.sql.types.DataType",
"fromJson",
$"{returnType}");
return(new UserDefinedFunction(
jvm.CallConstructor(
"org.apache.spark.sql.execution.python.UserDefinedPythonFunction",
name,
pythonFunction,
dataType,
(int)evalType,
true // udfDeterministic
)));
}
public Hyperspace(SparkSession spark)
{
_spark = spark;
_jvmBridge = ((IJvmObjectReferenceProvider)spark).Reference.Jvm;
_jvmObject = _jvmBridge.CallConstructor(s_hyperspaceClassName, spark);
}
/// <summary>
/// Create a <c>java.util.ArrayList</c> JVM object
/// </summary>
/// <param name="jvm">JVM bridge to use</param>
internal ArrayList(IJvmBridge jvm)
{
Reference = jvm.CallConstructor("java.util.ArrayList");
}
public Hyperspace(SparkSession spark)
{
_spark = spark;
_jvmBridge = spark.Reference.Jvm;
Reference = _jvmBridge.CallConstructor(s_hyperspaceClassName, spark);
}
/// <summary>
/// Create a <c>java.util.ArrayList</c> JVM object
/// </summary>
/// <param name="jvm">JVM bridge to use</param>
internal ArrayList(IJvmBridge jvm)
{
_jvmObject = jvm.CallConstructor("java.util.ArrayList");
}