public StreamingContextIpcProxy(SparkContext sparkContext, long durationMs) { this.sparkContext = sparkContext; sparkContextProxy = sparkContext.SparkContextProxy; var jduration = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.Duration", new object[] { durationMs }); JvmObjectReference jvmSparkContextReference = (sparkContextProxy as SparkContextIpcProxy).JvmSparkContextReference; jvmStreamingContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.StreamingContext", new object[] { jvmSparkContextReference, jduration }); jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { jvmStreamingContextReference }); }
public StreamingContextIpcProxy(string checkpointPath) { jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath }); jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc")); JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc")); JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf")); JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext")); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference); sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy)); }
public StreamingContextIpcProxy(SparkContext sparkContext, long durationMs) { this.sparkContext = sparkContext; sparkContextProxy = sparkContext.SparkContextProxy; var jduration = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.Duration", new object[] { durationMs }); JvmObjectReference jvmSparkContextReference = (sparkContextProxy as SparkContextIpcProxy).JvmSparkContextReference; jvmStreamingContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.StreamingContext", new object[] { jvmSparkContextReference, jduration }); jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { jvmStreamingContextReference }); int port = StartCallback(); SparkCLRIpcProxy.JvmBridge.CallStaticJavaMethod("SparkCLRHandler", "connectCallback", port); //className and methodName hardcoded in CSharpBackendHandler }
public StreamingContextIpcProxy(SparkContext sparkContext, int durationSeconds) { this.sparkContext = sparkContext; sparkContextProxy = sparkContext.SparkContextProxy; var jduration = JvmBridgeUtils.GetJavaDuration(durationSeconds); JvmObjectReference jvmSparkContextReference = (sparkContextProxy as SparkContextIpcProxy).JvmSparkContextReference; jvmStreamingContextReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.StreamingContext", new object[] { jvmSparkContextReference, jduration }); jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { jvmStreamingContextReference }); StartAccumulatorServer(sparkContext); StartCallbackServer(); }
public StreamingContextIpcProxy(string checkpointPath) { jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath }); jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc")); JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc")); JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf")); JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext")); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference); sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy)); // TODO: We don't know whether accumulator variable is used before restart. We just start accumuator server for safety. sparkContext.StartAccumulatorServer(); }
public StreamingContextIpcProxy(string checkpointPath) { jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath }); jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc")); sparkContext = SparkContext.GetActiveSparkContext(); if (sparkContext == null) { JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc")); JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf")); JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext")); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference); sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy)); } else { sparkContextProxy = sparkContext.SparkContextProxy; } StartAccumulatorServer(sparkContext); }
public StreamingContextIpcProxy(string checkpointPath) { sparkContext = SparkContext.GetActiveSparkContext(); StartCallbackServer(); jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath }); jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc")); if (sparkContext == null) { JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc")); JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf")); JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext")); sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference); var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference); sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy)); } else { sparkContextProxy = sparkContext.SparkContextProxy; } StartAccumulatorServer(sparkContext); }
/// <summary> /// when created from checkpoint /// </summary> /// <param name="sparkContextProxy"></param> /// <param name="conf"></param> internal SparkContext(ISparkContextProxy sparkContextProxy, SparkConf conf) { SparkContextProxy = sparkContextProxy; SparkConf = conf; }
public MockSqlContextProxy(ISparkContextProxy scProxy) { mockSqlContextReference = new object(); mockSparkContextProxy = scProxy; }
/// <summary> /// when created from checkpoint /// </summary> /// <param name="sparkContextProxy"></param> /// <param name="conf"></param> internal SparkContext(ISparkContextProxy sparkContextProxy, SparkConf conf) { SparkContextProxy = sparkContextProxy; SparkConf = conf; _activeSparkContext = this; }