private static void InitializeSparkContext(string[] args)
{
var sparkConf = new SparkConf();
sparkConf.Set("spark.local.dir", args[0]);
sparkConf.SetAppName("SparkCLR perf suite - C#");
SparkContext = new SparkContext(sparkConf);
SqlContext = new SqlContext(PerfBenchmark.SparkContext);
}
// Creates and returns a context
private static SparkContext CreateSparkContext()
{
var conf = new SparkConf();
if (Configuration.SparkLocalDirectoryOverride != null)
{
conf.Set("spark.local.dir", Configuration.SparkLocalDirectoryOverride);
}
return new SparkContext(conf);
}
// Creates and returns a context
private static SparkContext CreateSparkContext()
{
var conf = new SparkConf() { };
conf.SetMaster(Env.SPARK_MASTER_URL);
if (Configuration.SparkLocalDirectoryOverride != null)
{
conf.Set("spark.local.dir", Configuration.SparkLocalDirectoryOverride);
}
return new SparkContext(conf);
}
/// <summary>
/// Gets an existing [[SparkSession]] or, if there is no existing one, creates a new
/// one based on the options set in this builder.
/// </summary>
/// <returns></returns>
public SparkSession GetOrCreate()
{
var sparkConf = new SparkConf();
foreach (var option in options)
{
sparkConf.Set(option.Key, option.Value);
}
var sparkContext = SparkContext.GetOrCreate(sparkConf);
return SqlContext.GetOrCreate(sparkContext).SparkSession;
}
internal static void DStreamDirectKafkaWithRepartitionSample()
{
count = 0;
string directory = SparkCLRSamples.Configuration.SampleDataLocation;
string checkpointPath = Path.Combine(directory, "checkpoint");
StreamingContext ssc = StreamingContext.GetOrCreate(checkpointPath,
() =>
{
var conf = new SparkConf();
SparkContext sc = new SparkContext(conf);
StreamingContext context = new StreamingContext(sc, 2000L);
context.Checkpoint(checkpointPath);
var kafkaParams = new Dictionary<string, string> {
{"metadata.broker.list", brokers},
{"auto.offset.reset", "smallest"}
};
conf.Set("spark.mobius.streaming.kafka.numPartitions." + topic, partitions.ToString());
var dstream = KafkaUtils.CreateDirectStream(context, new List<string> { topic }, kafkaParams, new Dictionary<string, long>());
dstream.ForeachRDD((time, rdd) =>
{
long batchCount = rdd.Count();
int numPartitions = rdd.GetNumPartitions();
Console.WriteLine("-------------------------------------------");
Console.WriteLine("Time: {0}", time);
Console.WriteLine("-------------------------------------------");
Console.WriteLine("Count: " + batchCount);
Console.WriteLine("Partitions: " + numPartitions);
// only first batch has data and is repartitioned into 10 partitions
if (count++ == 0)
{
Assert.AreEqual(messages, batchCount);
Assert.IsTrue(numPartitions >= partitions);
}
else
{
Assert.AreEqual(0, batchCount);
Assert.IsTrue(numPartitions == 0);
}
});
return context;
});
ssc.Start();
ssc.AwaitTermination();
}
private static SparkContext CreateSparkContext()
{
var conf = new SparkConf();
conf.Set("spark.local.dir", @"C:\temp");
return new SparkContext(conf);
}
/// <summary>
/// Creates and returns a context
/// </summary>
/// <returns>SparkContext</returns>
private static SparkContext CreateSparkContext()
{
var conf = new SparkConf();
// set up local directory
var tempDir = Environment.GetEnvironmentVariable("spark.local.dir");
if (string.IsNullOrEmpty(tempDir))
{
tempDir = Path.GetTempPath();
}
conf.Set("spark.local.dir", tempDir);
Logger.DebugFormat("spark.local.dir is set to {0}", tempDir);
return new SparkContext(conf);
}