public void TestSparkConf()
{
var sparkConf = new SparkConf(false);
sparkConf.SetMaster("master");
sparkConf.SetAppName("test");
sparkConf.SetSparkHome("test home");
sparkConf.Set("key_string", "value");
sparkConf.Set("key_int", "100");
var expectedConfigs = new Dictionary <string, string>()
{
{ "spark.master", "master" },
{ "spark.app.name", "test" },
{ "spark.home", "test home" },
{ "key_string", "value" },
{ "key_int", "100" }
};
foreach (KeyValuePair <string, string> kv in expectedConfigs)
{
Assert.Equal(kv.Value, sparkConf.Get(kv.Key, string.Empty));
}
Assert.Equal(100, sparkConf.GetInt("key_int", 0));
// Validate GetAll().
Dictionary <string, string> actualAllConfigs =
sparkConf.GetAll().ToDictionary(x => x.Key, x => x.Value);
Assert.Equal(expectedConfigs, actualAllConfigs);
}
/// <summary>
/// Sets a list of config options based on the given SparkConf
/// </summary>
public Builder Config(SparkConf conf)
{
foreach (KeyValuePair <string, string> keyValuePair in conf.GetAll())
{
_options[keyValuePair.Key] = keyValuePair.Value;
}
return(this);
}