public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf)
{
JvmObjectReference jvmSparkContextReference = JvmBridge.CallConstructor("org.apache.spark.SparkContext", (conf as SparkConfIpcProxy).JvmSparkConfReference);
JvmObjectReference jvmJavaContextReference = JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference });
sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
return sparkContextProxy;
}
/// <summary>
/// Create SparkConf
/// </summary>
/// <param name="loadDefaults">indicates whether to also load values from Java system properties</param>
public SparkConf(bool loadDefaults = true)
{
sparkConfProxy = SparkCLREnvironment.SparkCLRProxy.CreateSparkConf(loadDefaults);
//special handling for debug mode because
//spark.master and spark.app.name will not be set in debug mode
//driver code may override these values if SetMaster or SetAppName methods are used
if (string.IsNullOrWhiteSpace(Get("spark.master", "")))
{
logger.LogInfo("spark.master not set. Assuming debug mode.");
SetMaster("local");
}
if (string.IsNullOrWhiteSpace(Get("spark.app.name", "")))
{
logger.LogInfo("spark.app.name not set. Assuming debug mode");
SetAppName("debug app");
}
}
/// <summary>
/// Create SparkConf
/// </summary>
/// <param name="loadDefaults">indicates whether to also load values from Java system properties</param>
public SparkConf(bool loadDefaults = true)
{
sparkConfProxy = SparkCLREnvironment.SparkCLRProxy.CreateSparkConf(loadDefaults);
//special handling for debug mode because
//spark.master and spark.app.name will not be set in debug mode
//driver code may override these values if SetMaster or SetAppName methods are used
if (string.IsNullOrWhiteSpace(Get("spark.master", "")))
{
logger.LogInfo("spark.master not set. Assuming debug mode.");
SetMaster("local");
}
if (string.IsNullOrWhiteSpace(Get("spark.app.name", "")))
{
logger.LogInfo("spark.app.name not set. Assuming debug mode");
SetAppName("debug app");
}
}
public ISparkContextProxy CreateSparkContext(ISparkConfProxy conf)
{
string master = null;
string appName = null;
string sparkHome = null;
if (conf != null)
{
MockSparkConfProxy proxy = conf as MockSparkConfProxy;
if (proxy.stringConfDictionary.ContainsKey("mockmaster"))
master = proxy.stringConfDictionary["mockmaster"];
if (proxy.stringConfDictionary.ContainsKey("mockappName"))
appName = proxy.stringConfDictionary["mockappName"];
if (proxy.stringConfDictionary.ContainsKey("mockhome"))
sparkHome = proxy.stringConfDictionary["mockhome"];
}
return new MockSparkContextProxy(conf);
}
public MockSparkContextProxy(ISparkConfProxy conf)
{
mockSparkContextReference = new object[] { conf };
}
public MockSparkContextProxy(ISparkConfProxy conf)
{
mockSparkContextReference = new object[] { conf };
}
/// <summary>
/// when created from checkpoint
/// </summary>
/// <param name="sparkConfProxy"></param>
internal SparkConf(ISparkConfProxy sparkConfProxy)
{
this.sparkConfProxy = sparkConfProxy;
}
public void CreateSparkContext(string master, string appName, string sparkHome, ISparkConfProxy conf)
{
object[] args = (new object[] { master, appName, sparkHome, (conf == null ? null : (conf as SparkConfIpcProxy).JvmSparkConfReference) }).Where(x => x != null).ToArray();
jvmSparkContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.SparkContext", args);
jvmJavaContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference });
}
/// <summary>
/// when created from checkpoint
/// </summary>
/// <param name="sparkConfProxy"></param>
internal SparkConf(ISparkConfProxy sparkConfProxy)
{
this.sparkConfProxy = sparkConfProxy;
}
public void CreateSparkContext(string master, string appName, string sparkHome, ISparkConfProxy conf)
{
object[] args = (new object[] { master, appName, sparkHome, (conf == null ? null : (conf as SparkConfIpcProxy).JvmSparkConfReference) }).Where(x => x != null).ToArray();
jvmSparkContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.SparkContext", args);
jvmJavaContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.api.java.JavaSparkContext", new object[] { jvmSparkContextReference });
}
