private static void InitializeSparkContext(string[] args) { var sparkConf = new SparkConf(); sparkConf.Set("spark.local.dir", args[0]); sparkConf.SetAppName("SparkCLR perf suite - C#"); SparkContext = new SparkContext(sparkConf); SqlContext = new SqlContext(PerfBenchmark.SparkContext); }
// Creates and returns a context private static SparkContext CreateSparkContext() { var conf = new SparkConf(); if (Configuration.SparkLocalDirectoryOverride != null) { conf.Set("spark.local.dir", Configuration.SparkLocalDirectoryOverride); } return new SparkContext(conf); }
// Creates and returns a context private static SparkContext CreateSparkContext() { var conf = new SparkConf() { }; conf.SetMaster(Env.SPARK_MASTER_URL); if (Configuration.SparkLocalDirectoryOverride != null) { conf.Set("spark.local.dir", Configuration.SparkLocalDirectoryOverride); } return new SparkContext(conf); }
/// <summary> /// Gets an existing [[SparkSession]] or, if there is no existing one, creates a new /// one based on the options set in this builder. /// </summary> /// <returns></returns> public SparkSession GetOrCreate() { var sparkConf = new SparkConf(); foreach (var option in options) { sparkConf.Set(option.Key, option.Value); } var sparkContext = SparkContext.GetOrCreate(sparkConf); return SqlContext.GetOrCreate(sparkContext).SparkSession; }
internal static void DStreamDirectKafkaWithRepartitionSample() { count = 0; string directory = SparkCLRSamples.Configuration.SampleDataLocation; string checkpointPath = Path.Combine(directory, "checkpoint"); StreamingContext ssc = StreamingContext.GetOrCreate(checkpointPath, () => { var conf = new SparkConf(); SparkContext sc = new SparkContext(conf); StreamingContext context = new StreamingContext(sc, 2000L); context.Checkpoint(checkpointPath); var kafkaParams = new Dictionary<string, string> { {"metadata.broker.list", brokers}, {"auto.offset.reset", "smallest"} }; conf.Set("spark.mobius.streaming.kafka.numPartitions." + topic, partitions.ToString()); var dstream = KafkaUtils.CreateDirectStream(context, new List<string> { topic }, kafkaParams, new Dictionary<string, long>()); dstream.ForeachRDD((time, rdd) => { long batchCount = rdd.Count(); int numPartitions = rdd.GetNumPartitions(); Console.WriteLine("-------------------------------------------"); Console.WriteLine("Time: {0}", time); Console.WriteLine("-------------------------------------------"); Console.WriteLine("Count: " + batchCount); Console.WriteLine("Partitions: " + numPartitions); // only first batch has data and is repartitioned into 10 partitions if (count++ == 0) { Assert.AreEqual(messages, batchCount); Assert.IsTrue(numPartitions >= partitions); } else { Assert.AreEqual(0, batchCount); Assert.IsTrue(numPartitions == 0); } }); return context; }); ssc.Start(); ssc.AwaitTermination(); }
private static SparkContext CreateSparkContext() { var conf = new SparkConf(); conf.Set("spark.local.dir", @"C:\temp"); return new SparkContext(conf); }
/// <summary> /// Creates and returns a context /// </summary> /// <returns>SparkContext</returns> private static SparkContext CreateSparkContext() { var conf = new SparkConf(); // set up local directory var tempDir = Environment.GetEnvironmentVariable("spark.local.dir"); if (string.IsNullOrEmpty(tempDir)) { tempDir = Path.GetTempPath(); } conf.Set("spark.local.dir", tempDir); Logger.DebugFormat("spark.local.dir is set to {0}", tempDir); return new SparkContext(conf); }