public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf) { JvmObjectReference jvmSparkContextReference = JvmBridge.CallConstructor("org.apache.spark.SparkContext", (conf as SparkConfIpcProxy).JvmSparkConfReference); JvmObjectReference jvmJavaContextReference = JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference }); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); return(sparkContextProxy); }
public IStructFieldProxy CreateStructField(string name, string dataType, bool isNullable) { return(new StructFieldIpcProxy( new JvmObjectReference( JvmBridge.CallStaticJavaMethod( "org.apache.spark.sql.api.csharp.SQLUtils", "createStructField", new object[] { name, dataType, isNullable }).ToString() ) )); }
public IStructTypeProxy CreateStructType(List <StructField> fields) { var fieldsReference = fields.Select(s => (s.StructFieldProxy as StructFieldIpcProxy).JvmStructFieldReference).ToList().Cast <JvmObjectReference>(); var seq = new JvmObjectReference( JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "toSeq", new object[] { fieldsReference }).ToString()); return(new StructTypeIpcProxy( new JvmObjectReference( JvmBridge.CallStaticJavaMethod("org.apache.spark.sql.api.csharp.SQLUtils", "createStructType", new object[] { seq }).ToString() ) )); }
private static Version GetSparkVersion() { var sparkVersion = new Version((string)JvmBridge.CallStaticJavaMethod( "org.apache.spark.deploy.dotnet.DotnetRunner", "SPARK_VERSION")); string sparkVersionOverride = Environment.GetEnvironmentVariable("SPARK_VERSION_OVERRIDE"); if (!string.IsNullOrEmpty(sparkVersionOverride)) { s_logger.LogInfo( $"Overriding the Spark version from '{sparkVersion}' " + $"to '{sparkVersionOverride}'."); sparkVersion = new Version(sparkVersionOverride); } return(sparkVersion); }
public ISparkConfProxy CreateSparkConf(bool loadDefaults = true) { return(new SparkConfIpcProxy(JvmBridge.CallConstructor("org.apache.spark.SparkConf", new object[] { loadDefaults }))); }