/// <summary>
/// Create a SparkContext object with the given config.
/// </summary>
/// <param name="conf">a Spark config object describing the application configuration.
/// Any settings in this config overrides the default configs as well as system properties.
/// </param>
public SparkContext(SparkConf conf)
: this(
((IJvmObjectReferenceProvider)conf).Reference.Jvm.CallConstructor(
"org.apache.spark.SparkContext",
conf))
{
}
/// <summary>
/// This function may be used to get or instantiate a SparkContext and register it as a
/// singleton object. Because we can only have one active SparkContext per JVM,
/// this is useful when applications may wish to share a SparkContext.
/// </summary>
/// <param name="conf"><see cref="SparkConf"/> that will be used for creating SparkContext
/// </param>
/// <returns>
/// Current SparkContext (or a new one if it wasn't created before the function call)
/// </returns>
public static SparkContext GetOrCreate(SparkConf conf)
{
IJvmBridge jvm = ((IJvmObjectReferenceProvider)conf).Reference.Jvm;
return(new SparkContext(
(JvmObjectReference)jvm.CallStaticJavaMethod(
"org.apache.spark.SparkContext",
"getOrCreate",
conf)));
}
/// <summary>
/// Function that creates a temporary directory inside the given directory and returns the
/// absolute filepath of temporary file name in that directory.
/// </summary>
/// <param name="conf">SparkConf object</param>
/// <returns>Absolute filepath of the created random file</returns>
private string CreateTempFilePath(SparkConf conf)
{
var localDir = (string)conf.Reference.Jvm.CallStaticJavaMethod(
"org.apache.spark.util.Utils",
"getLocalDir",
conf);
string dir = Path.Combine(localDir, "sparkdotnet");
Directory.CreateDirectory(dir);
return(Path.Combine(dir, Path.GetRandomFileName()));
}
/// <summary>
/// Creates a modified version of <see cref="SparkConf"/> with the parameters that can be
/// passed separately to SparkContext, to make it easier to write SparkContext's
/// constructors.
/// </summary>
/// <param name="master">Cluster URL to connect to (e.g. spark://host:port, local)</param>
/// <param name="appName">A name for the application</param>
/// <param name="sparkHome">The path that holds spark bits</param>
/// <param name="conf">
/// A <see cref="SparkConf"/> object specifying other Spark parameters
/// </param>
/// <returns>Modified <see cref="SparkConf"/> object.</returns>
private static SparkConf GetUpdatedConf(
string master,
string appName,
string sparkHome,
SparkConf conf)
{
SparkConf sparkConf = conf ?? new SparkConf();
if (master != null)
{
sparkConf.SetMaster(master);
}
if (appName != null)
{
sparkConf.SetAppName(appName);
}
if (sparkHome != null)
{
sparkConf.SetSparkHome(sparkHome);
}
return(sparkConf);
}
/// <summary>
/// Constructor where SparkContext object is already created.
/// </summary>
/// <param name="jvmObject">JVM object reference for this SparkContext object</param>
internal SparkContext(JvmObjectReference jvmObject)
{
_jvmObject = jvmObject;
_conf = new SparkConf((JvmObjectReference)_jvmObject.Invoke("getConf"));
}
/// <summary>
/// Alternative constructor that allows setting common Spark properties directly.
/// </summary>
/// <param name="master">Cluster URL to connect to (e.g. spark://host:port, local)</param>
/// <param name="appName">A name for the application</param>
/// <param name="conf">
/// A <see cref="SparkConf"/> object specifying other Spark parameters
/// </param>
public SparkContext(string master, string appName, SparkConf conf)
: this(GetUpdatedConf(master, appName, null, conf))
{
}
/// <summary>
/// Create a SparkContext object with the given config.
/// </summary>
/// <param name="conf">a Spark config object describing the application configuration.
/// Any settings in this config overrides the default configs as well as system properties.
/// </param>
public SparkContext(SparkConf conf)
: this(conf.Reference.Jvm.CallConstructor("org.apache.spark.SparkContext", conf))
{
}