public void TestSparkConf()
{
var sparkConf = new SparkConf(false);
sparkConf.SetMaster("master");
sparkConf.SetAppName("test");
sparkConf.SetSparkHome("test home");
sparkConf.Set("key_string", "value");
sparkConf.Set("key_int", "100");
var expectedConfigs = new Dictionary <string, string>()
{
{ "spark.master", "master" },
{ "spark.app.name", "test" },
{ "spark.home", "test home" },
{ "key_string", "value" },
{ "key_int", "100" }
};
foreach (KeyValuePair <string, string> kv in expectedConfigs)
{
Assert.Equal(kv.Value, sparkConf.Get(kv.Key, string.Empty));
}
Assert.Equal(100, sparkConf.GetInt("key_int", 0));
// Validate GetAll().
Dictionary <string, string> actualAllConfigs =
sparkConf.GetAll().ToDictionary(x => x.Key, x => x.Value);
Assert.Equal(expectedConfigs, actualAllConfigs);
}
public void TestSparkConfMethods()
{
var sparkConf = new SparkConf();
sparkConf.SetMaster("masterUrl");
Assert.AreEqual("masterUrl", sparkConf.Get(MockSparkConfProxy.MockMasterKey, ""));
sparkConf.SetAppName("app name ");
Assert.AreEqual("app name ", sparkConf.Get(MockSparkConfProxy.MockAppNameKey, ""));
sparkConf.SetSparkHome(@"c:\path\to\sparkfolder");
Assert.AreEqual(@"c:\path\to\sparkfolder", sparkConf.Get(MockSparkConfProxy.MockHomeKey, ""));
Assert.AreEqual("default value", sparkConf.Get("non existent key", "default value"));
Assert.AreEqual(3, sparkConf.GetInt("non existent key", 3));
}
public RoslynScriptEngine(SparkContext sc, SqlContext sqlContext)
{
this.sc = sc;
sparkConf = sc.GetConf();
host = new SparkCLRHost
{
sc = sc,
sqlContext = sqlContext
};
var sparkLocalDir = sparkConf.Get("spark.local.dir", Path.GetTempPath());
compilationDumpDirectory = Path.Combine(sparkLocalDir, Path.GetRandomFileName());
Directory.CreateDirectory(compilationDumpDirectory);
options = new CSharpParseOptions(LanguageVersion.CSharp6, DocumentationMode.Parse, SourceCodeKind.Script);
}
public ScriptResult Execute(string code)
{
if (string.IsNullOrWhiteSpace(code))
{
return(ScriptResult.Empty);
}
if (!IsCompleteSubmission(code))
{
return(ScriptResult.Incomplete);
}
Script <object> script;
if (previousState == null)
{
script = CreateScript(code);
Environment.SetEnvironmentVariable("SPARKCLR_RUN_MODE", "R");
if (sparkConf.Get("spark.master", "local").StartsWith("local", StringComparison.InvariantCultureIgnoreCase))
{
Environment.SetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR", compilationDumpDirectory);
}
}
else
{
script = previousState.Script.ContinueWith(code);
}
var diagnostics = script.Compile();
bool hasErrors = Enumerable.Any(diagnostics, diagnostic => diagnostic.Severity == DiagnosticSeverity.Error);
if (hasErrors)
{
var diagnosticMessages = diagnostics.Select(diagnostic => "[" + diagnostic.Severity + "] " + diagnostic).ToList();
return(new ScriptResult(compilationException: new Exception(string.Join("\r\n", diagnosticMessages))));
}
var compilationDump = DumpCompilation(script.GetCompilation());
if (new FileInfo(compilationDump).Length > 0)
{
// Ship compilation binary to executor side leveraging sparkContext.AddFile() method.
sc.AddFile(new Uri(compilationDump).ToString());
}
try
{
ScriptState <object> endState = null;
if (previousState == null)
{
endState = script.RunAsync(host).Result;
}
else
{
// Currently "ContinueAsync" is a internal methold, might go public in 1.2.0(https://github.com/dotnet/roslyn/issues/6612)
const string methodName = "ContinueAsync";
var m = script.GetType().GetMethod(methodName, System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
if (m != null)
{
endState = ((Task <ScriptState <object> >)m.Invoke(script, new object[] { previousState, default(CancellationToken) })).Result;
}
else
{
throw new InvalidOperationException(string.Format("Can't find method {0}", methodName));
}
}
previousState = endState;
return(new ScriptResult(returnValue: endState.ReturnValue));
}
catch (Exception e)
{
return(new ScriptResult(executionException: e));
}
}