Exemplo n.º 1
0
 public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string deserializer)
 {
     Func<double, RDD<dynamic>, RDD<dynamic>> f = (Func<double, RDD<dynamic>, RDD<dynamic>>)formatter.Deserialize(new MemoryStream(func));
     RDD<dynamic> rdd = f(DateTime.UtcNow.Ticks,
         new RDD<dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")));
     return new MockDStreamProxy(rdd.RddProxy);
 }
Exemplo n.º 2
0
 public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string deserializer)
 {
     return new DStreamIpcProxy(
         SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpDStream",
         new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, deserializer })
     );
 }
 public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string serializationMode)
 {
     Func<double, RDD<dynamic>, RDD<dynamic>> f = (Func<double, RDD<dynamic>, RDD<dynamic>>)formatter.Deserialize(new MemoryStream(func));
     RDD<dynamic> rdd = f(DateTime.UtcNow.Ticks,
         new RDD<dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")));
     return new MockDStreamProxy(rdd.RddProxy);
 }
Exemplo n.º 4
0
 internal DStream(IDStreamProxy dstreamProxy, StreamingContext streamingContext, SerializedMode serializedMode = SerializedMode.Byte)
 {
     this.streamingContext = streamingContext;
     this.dstreamProxy     = dstreamProxy;
     this.serializedMode   = serializedMode;
     isCached       = false;
     isCheckpointed = false;
 }
Exemplo n.º 5
0
        public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string serializationMode)
        {
            Func <double, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(func));
            RDD <dynamic> rdd = f(DateTime.UtcNow.Ticks,
                                  new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")));

            return(new MockDStreamProxy(rdd.RddProxy));
        }
Exemplo n.º 6
0
        public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string deserializer)
        {
            var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpDStream",
                new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, deserializer });

            var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));
            return new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference);
        }
Exemplo n.º 7
0
        public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string deserializer)
        {
            Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(func));
            RDD <dynamic> rdd = f(DateTime.UtcNow.Ticks,
                                  new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")),
                                  new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")));

            return(new MockDStreamProxy(rdd.RddProxy));
        }
Exemplo n.º 8
0
        public IDStreamProxy CreateCSharpStateDStream(IDStreamProxy jdstream, byte[] func, string className, string serializationMode, string serializationMode2)
        {
            var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp." + className,
                                                                                 new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, serializationMode, serializationMode2 });

            var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));

            return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference));
        }
Exemplo n.º 9
0
        public IDStreamProxy CreateCSharpTransformed2DStream(IDStreamProxy jdstream, IDStreamProxy jother, byte[] func, string deserializer, string deserializerOther)
        {
            var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpTransformed2DStream",
                                                                                 new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, (jother as DStreamIpcProxy).jvmDStreamReference, func, deserializer, deserializerOther });

            var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));

            return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference));
        }
Exemplo n.º 10
0
        public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string deserializer)
        {
            var windowDurationReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.Duration", new object[] { windowSeconds * 1000 });
            var slideDurationReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.Duration", new object[] { slideSeconds * 1000 });

            return new DStreamIpcProxy(
                SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpReducedWindowedDStream",
                new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, invFunc, windowDurationReference, slideDurationReference, deserializer })
            );
        }
Exemplo n.º 11
0
        public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string serializationMode)
        {
            var windowDurationReference = JvmBridgeUtils.GetJavaDuration(windowSeconds);
            var slideDurationReference  = JvmBridgeUtils.GetJavaDuration(slideSeconds);

            var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpReducedWindowedDStream",
                                                                                 new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, invFunc, windowDurationReference, slideDurationReference, serializationMode });

            var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));

            return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference));
        }
Exemplo n.º 12
0
 public IDStreamProxy Union(IDStreamProxy firstDStream, IDStreamProxy[] otherDStreams)
 {
     return(new DStreamIpcProxy(
                new JvmObjectReference(
                    (string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "union",
                                                                               new object[]
     {
         (firstDStream as DStreamIpcProxy).javaDStreamReference,
         JvmBridgeUtils.GetJavaList <JvmObjectReference>(otherDStreams.Select(x => (x as DStreamIpcProxy).javaDStreamReference))
     }
                                                                               ))));
 }
Exemplo n.º 13
0
        public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string serializationMode)
        {
            Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(func));

            var           ticks = DateTime.UtcNow.Ticks;
            RDD <dynamic> rdd   = f(ticks,
                                    new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")),
                                    new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")));

            if (invFunc == null)
            {
                return(new MockDStreamProxy(rdd.RddProxy));
            }

            Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > invf = (Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(invFunc));
            RDD <dynamic> invRdd = invf(ticks,
                                        new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")),
                                        new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")));
            var difference = rdd.Subtract(invRdd);

            return(new MockDStreamProxy(difference.RddProxy));
        }
Exemplo n.º 14
0
 internal DStream(IDStreamProxy dstreamProxy, StreamingContext streamingContext, SerializedMode serializedMode = SerializedMode.Byte)
 {
     this.streamingContext = streamingContext;
     this.dstreamProxy     = dstreamProxy;
     this.serializedMode   = serializedMode;
 }
Exemplo n.º 15
0
        public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string serializationMode)
        {
            var windowDurationReference = JvmBridgeUtils.GetJavaDuration(windowSeconds);
            var slideDurationReference = JvmBridgeUtils.GetJavaDuration(slideSeconds);

            var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpReducedWindowedDStream",
                new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, invFunc, windowDurationReference, slideDurationReference, serializationMode });

            var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream"));
            return new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference);
        }
Exemplo n.º 16
0
 public IDStreamProxy Union(IDStreamProxy firstDStreams, IDStreamProxy[] otherDStreams)
 {
     return(new MockDStreamProxy());
 }
Exemplo n.º 17
0
 public IDStreamProxy Union(IDStreamProxy firstDStreams, IDStreamProxy[] otherDStreams)
 {
     return new MockDStreamProxy();
 }
 public IDStreamProxy Union(IDStreamProxy firstDStream, IDStreamProxy[] otherDStreams)
 {
     return new DStreamIpcProxy(
         new JvmObjectReference(
             (string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "union",
                 new object[]
                 {
                     (firstDStream as DStreamIpcProxy).javaDStreamReference,
                     otherDStreams.Select(x => (x as DStreamIpcProxy).javaDStreamReference).ToArray()
                 }
             )));
 }
Exemplo n.º 19
0
        public IDStreamProxy CreateCSharpTransformed2DStream(IDStreamProxy jdstream, IDStreamProxy jother, byte[] func, string deserializer, string deserializerOther)
        {
            var dstream = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((jdstream as DStreamIpcProxy).jvmDStreamReference, "dstream"));
            var dother = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((jother as DStreamIpcProxy).jvmDStreamReference, "dstream"));

            return new DStreamIpcProxy(
                SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpTransformed2DStream",
                new object[] { dstream, dother, func, deserializer, deserializerOther })
            );
        }