public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf) { JvmObjectReference jvmSparkContextReference = JvmBridge.CallConstructor("org.apache.spark.SparkContext", (conf as SparkConfIpcProxy).JvmSparkConfReference); JvmObjectReference jvmJavaContextReference = JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference }); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); return sparkContextProxy; }
/// <summary> /// Create SparkConf /// </summary> /// <param name="loadDefaults">indicates whether to also load values from Java system properties</param> public SparkConf(bool loadDefaults = true) { sparkConfProxy = SparkCLREnvironment.SparkCLRProxy.CreateSparkConf(loadDefaults); //special handling for debug mode because //spark.master and spark.app.name will not be set in debug mode //driver code may override these values if SetMaster or SetAppName methods are used if (string.IsNullOrWhiteSpace(Get("spark.master", ""))) { logger.LogInfo("spark.master not set. Assuming debug mode."); SetMaster("local"); } if (string.IsNullOrWhiteSpace(Get("spark.app.name", ""))) { logger.LogInfo("spark.app.name not set. Assuming debug mode"); SetAppName("debug app"); } }
/// <summary> /// Create SparkConf /// </summary> /// <param name="loadDefaults">indicates whether to also load values from Java system properties</param> public SparkConf(bool loadDefaults = true) { sparkConfProxy = SparkCLREnvironment.SparkCLRProxy.CreateSparkConf(loadDefaults); //special handling for debug mode because //spark.master and spark.app.name will not be set in debug mode //driver code may override these values if SetMaster or SetAppName methods are used if (string.IsNullOrWhiteSpace(Get("spark.master", ""))) { logger.LogInfo("spark.master not set. Assuming debug mode."); SetMaster("local"); } if (string.IsNullOrWhiteSpace(Get("spark.app.name", ""))) { logger.LogInfo("spark.app.name not set. Assuming debug mode"); SetAppName("debug app"); } }
public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf) { string master = null; string appName = null; string sparkHome = null; if (conf != null) { MockSparkConfProxy proxy = conf as MockSparkConfProxy; if (proxy.stringConfDictionary.ContainsKey("mockmaster")) master = proxy.stringConfDictionary["mockmaster"]; if (proxy.stringConfDictionary.ContainsKey("mockappName")) appName = proxy.stringConfDictionary["mockappName"]; if (proxy.stringConfDictionary.ContainsKey("mockhome")) sparkHome = proxy.stringConfDictionary["mockhome"]; } return new MockSparkContextProxy(conf); }
public MockSparkContextProxy(ISparkConfProxy conf) { mockSparkContextReference = new object[] { conf }; }
public MockSparkContextProxy(ISparkConfProxy conf) { mockSparkContextReference = new object[] { conf }; }
/// <summary> /// when created from checkpoint /// </summary> /// <param name="sparkConfProxy"></param> internal SparkConf(ISparkConfProxy sparkConfProxy) { this.sparkConfProxy = sparkConfProxy; }
public void CreateSparkContext(string master, string appName, string sparkHome, ISparkConfProxy conf) { object[] args = (new object[] { master, appName, sparkHome, (conf == null ? null : (conf as SparkConfIpcProxy).JvmSparkConfReference) }).Where(x => x != null).ToArray(); jvmSparkContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.SparkContext", args); jvmJavaContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference }); }
/// <summary> /// when created from checkpoint /// </summary> /// <param name="sparkConfProxy"></param> internal SparkConf(ISparkConfProxy sparkConfProxy) { this.sparkConfProxy = sparkConfProxy; }
public void CreateSparkContext(string master, string appName, string sparkHome, ISparkConfProxy conf) { object[] args = (new object[] { master, appName, sparkHome, (conf == null ? null : (conf as SparkConfIpcProxy).JvmSparkConfReference) }).Where(x => x != null).ToArray(); jvmSparkContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.SparkContext", args); jvmJavaContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference }); }