Example #1
0
 /// <summary>
 /// Create a SparkContext object with the given config.
 /// </summary>
 /// <param name="conf">a Spark config object describing the application configuration.
 /// Any settings in this config overrides the default configs as well as system properties.
 /// </param>
 public SparkContext(SparkConf conf)
     : this(
         ((IJvmObjectReferenceProvider)conf).Reference.Jvm.CallConstructor(
             "org.apache.spark.SparkContext",
             conf))
 {
 }
Example #2
0
        /// <summary>
        /// This function may be used to get or instantiate a SparkContext and register it as a
        /// singleton object. Because we can only have one active SparkContext per JVM,
        /// this is useful when applications may wish to share a SparkContext.
        /// </summary>
        /// <param name="conf"><see cref="SparkConf"/> that will be used for creating SparkContext
        /// </param>
        /// <returns>
        /// Current SparkContext (or a new one if it wasn't created before the function call)
        /// </returns>
        public static SparkContext GetOrCreate(SparkConf conf)
        {
            IJvmBridge jvm = ((IJvmObjectReferenceProvider)conf).Reference.Jvm;

            return(new SparkContext(
                       (JvmObjectReference)jvm.CallStaticJavaMethod(
                           "org.apache.spark.SparkContext",
                           "getOrCreate",
                           conf)));
        }
Example #3
0
        /// <summary>
        /// Function that creates a temporary directory inside the given directory and returns the
        /// absolute filepath of temporary file name in that directory.
        /// </summary>
        /// <param name="conf">SparkConf object</param>
        /// <returns>Absolute filepath of the created random file</returns>
        private string CreateTempFilePath(SparkConf conf)
        {
            var localDir = (string)conf.Reference.Jvm.CallStaticJavaMethod(
                "org.apache.spark.util.Utils",
                "getLocalDir",
                conf);
            string dir = Path.Combine(localDir, "sparkdotnet");

            Directory.CreateDirectory(dir);
            return(Path.Combine(dir, Path.GetRandomFileName()));
        }
Example #4
0
        /// <summary>
        /// Creates a modified version of <see cref="SparkConf"/> with the parameters that can be
        /// passed separately to SparkContext, to make it easier to write SparkContext's
        /// constructors.
        /// </summary>
        /// <param name="master">Cluster URL to connect to (e.g. spark://host:port, local)</param>
        /// <param name="appName">A name for the application</param>
        /// <param name="sparkHome">The path that holds spark bits</param>
        /// <param name="conf">
        /// A <see cref="SparkConf"/> object specifying other Spark parameters
        /// </param>
        /// <returns>Modified <see cref="SparkConf"/> object.</returns>
        private static SparkConf GetUpdatedConf(
            string master,
            string appName,
            string sparkHome,
            SparkConf conf)
        {
            SparkConf sparkConf = conf ?? new SparkConf();

            if (master != null)
            {
                sparkConf.SetMaster(master);
            }
            if (appName != null)
            {
                sparkConf.SetAppName(appName);
            }
            if (sparkHome != null)
            {
                sparkConf.SetSparkHome(sparkHome);
            }

            return(sparkConf);
        }
Example #5
0
 /// <summary>
 /// Constructor where SparkContext object is already created.
 /// </summary>
 /// <param name="jvmObject">JVM object reference for this SparkContext object</param>
 internal SparkContext(JvmObjectReference jvmObject)
 {
     _jvmObject = jvmObject;
     _conf      = new SparkConf((JvmObjectReference)_jvmObject.Invoke("getConf"));
 }
Example #6
0
 /// <summary>
 /// Alternative constructor that allows setting common Spark properties directly.
 /// </summary>
 /// <param name="master">Cluster URL to connect to (e.g. spark://host:port, local)</param>
 /// <param name="appName">A name for the application</param>
 /// <param name="conf">
 /// A <see cref="SparkConf"/> object specifying other Spark parameters
 /// </param>
 public SparkContext(string master, string appName, SparkConf conf)
     : this(GetUpdatedConf(master, appName, null, conf))
 {
 }
Example #7
0
 /// <summary>
 /// Create a SparkContext object with the given config.
 /// </summary>
 /// <param name="conf">a Spark config object describing the application configuration.
 /// Any settings in this config overrides the default configs as well as system properties.
 /// </param>
 public SparkContext(SparkConf conf)
     : this(conf.Reference.Jvm.CallConstructor("org.apache.spark.SparkContext", conf))
 {
 }