Ejemplos de Microsoft.Spark.CSharp.Proxy.Ipc SparkConfIpcProxy en C# (CSharp)

Ejemplo n.º 1

Archivo: StreamingContextIpcProxy.cs Proyecto: nkiranrao/SparkCLR

 public StreamingContextIpcProxy(string checkpointPath)
 {
     jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath });
     jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc"));
     JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc"));
     JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf"));
     JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext"));
     sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
     var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference);
     sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy));
 }

Ejemplo n.º 2

Archivo: StreamingContextIpcProxy.cs Proyecto: terencecraig/SparkCLR

        public StreamingContextIpcProxy(string checkpointPath)
        {
            jvmJavaStreamingReference    = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath });
            jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc"));
            JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc"));
            JvmObjectReference jvmSparkConfReference    = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf"));
            JvmObjectReference jvmJavaContextReference  = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext"));

            sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
            var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference);

            sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy));
        }

Ejemplo n.º 3

Archivo: StreamingContextIpcProxy.cs Proyecto: edepaula/SparkCLR

        public StreamingContextIpcProxy(string checkpointPath)
        {
            jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath });
            jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc"));
            JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc"));
            JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf"));
            JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext"));
            sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
            var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference);
            sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy));

            // TODO: We don't know whether accumulator variable is used before restart. We just start accumuator server for safety.
            sparkContext.StartAccumulatorServer();
        }

Ejemplo n.º 4

        public StreamingContextIpcProxy(string checkpointPath)
        {
            jvmJavaStreamingReference    = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath });
            jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc"));
            JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc"));
            JvmObjectReference jvmSparkConfReference    = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf"));
            JvmObjectReference jvmJavaContextReference  = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext"));

            sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
            var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference);

            sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy));

            // TODO: We don't know whether accumulator variable is used before restart. We just start accumuator server for safety.
            sparkContext.StartAccumulatorServer();
        }

Ejemplo n.º 5

 public StreamingContextIpcProxy(string checkpointPath)
 {
     jvmJavaStreamingReference    = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath });
     jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc"));
     sparkContext = SparkContext.GetActiveSparkContext();
     if (sparkContext == null)
     {
         JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc"));
         JvmObjectReference jvmSparkConfReference    = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf"));
         JvmObjectReference jvmJavaContextReference  = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext"));
         sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
         var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference);
         sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy));
     }
     else
     {
         sparkContextProxy = sparkContext.SparkContextProxy;
     }
     StartAccumulatorServer(sparkContext);
 }

Ejemplo n.º 6

Archivo: StreamingContextIpcProxy.cs Proyecto: cyruszhang/Mobius

        public StreamingContextIpcProxy(string checkpointPath)
        {
            sparkContext = SparkContext.GetActiveSparkContext();
            StartCallbackServer();

            jvmJavaStreamingReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.java.JavaStreamingContext", new object[] { checkpointPath });
            jvmStreamingContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "ssc"));
            if (sparkContext == null)
            {
                JvmObjectReference jvmSparkContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "sc"));
                JvmObjectReference jvmSparkConfReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmStreamingContextReference, "conf"));
                JvmObjectReference jvmJavaContextReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "sparkContext"));
                sparkContextProxy = new SparkContextIpcProxy(jvmSparkContextReference, jvmJavaContextReference);
                var sparkConfProxy = new SparkConfIpcProxy(jvmSparkConfReference);
                sparkContext = new SparkContext(sparkContextProxy, new SparkConf(sparkConfProxy));
            }
            else
            {
                sparkContextProxy = sparkContext.SparkContextProxy;
            }
            StartAccumulatorServer(sparkContext);
        }