public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf)
{
JvmObjectReference jvmSparkContextReference = JvmBridge.CallConstructor("org.apache.spark.SparkContext", (conf as SparkConfIpcProxy).JvmSparkConfReference);
JvmObjectReference jvmJavaContextReference = JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference });
sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
return(sparkContextProxy);
}
public IStructFieldProxy CreateStructField(string name, string dataType, bool isNullable)
{
return(new StructFieldIpcProxy(
new JvmObjectReference(
JvmBridge.CallStaticJavaMethod(
"org.apache.spark.sql.api.csharp.SQLUtils", "createStructField",
new object[] { name, dataType, isNullable }).ToString()
)
));
}
public IStructTypeProxy CreateStructType(List <StructField> fields)
{
var fieldsReference = fields.Select(s => (s.StructFieldProxy as StructFieldIpcProxy).JvmStructFieldReference).ToList().Cast <JvmObjectReference>();
var seq =
new JvmObjectReference(
JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils",
"toSeq", new object[] { fieldsReference }).ToString());
return(new StructTypeIpcProxy(
new JvmObjectReference(
JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "createStructType", new object[] { seq }).ToString()
)
));
}
private static Version GetSparkVersion()
{
var sparkVersion = new Version((string)JvmBridge.CallStaticJavaMethod(
"org.apache.spark.deploy.dotnet.DotnetRunner",
"SPARK_VERSION"));
string sparkVersionOverride =
Environment.GetEnvironmentVariable("SPARK_VERSION_OVERRIDE");
if (!string.IsNullOrEmpty(sparkVersionOverride))
{
s_logger.LogInfo(
$"Overriding the Spark version from '{sparkVersion}' " +
$"to '{sparkVersionOverride}'.");
sparkVersion = new Version(sparkVersionOverride);
}
return(sparkVersion);
}
public ISparkConfProxy CreateSparkConf(bool loadDefaults = true)
{
return(new SparkConfIpcProxy(JvmBridge.CallConstructor("org.apache.spark.SparkConf", new object[] { loadDefaults })));
}
