public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string deserializer) { Func<double, RDD<dynamic>, RDD<dynamic>> f = (Func<double, RDD<dynamic>, RDD<dynamic>>)formatter.Deserialize(new MemoryStream(func)); RDD<dynamic> rdd = f(DateTime.UtcNow.Ticks, new RDD<dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", ""))); return new MockDStreamProxy(rdd.RddProxy); }
public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string deserializer) { return new DStreamIpcProxy( SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpDStream", new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, deserializer }) ); }
public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string serializationMode) { Func<double, RDD<dynamic>, RDD<dynamic>> f = (Func<double, RDD<dynamic>, RDD<dynamic>>)formatter.Deserialize(new MemoryStream(func)); RDD<dynamic> rdd = f(DateTime.UtcNow.Ticks, new RDD<dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", ""))); return new MockDStreamProxy(rdd.RddProxy); }
internal DStream(IDStreamProxy dstreamProxy, StreamingContext streamingContext, SerializedMode serializedMode = SerializedMode.Byte) { this.streamingContext = streamingContext; this.dstreamProxy = dstreamProxy; this.serializedMode = serializedMode; isCached = false; isCheckpointed = false; }
public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string serializationMode) { Func <double, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(func)); RDD <dynamic> rdd = f(DateTime.UtcNow.Ticks, new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", ""))); return(new MockDStreamProxy(rdd.RddProxy)); }
public IDStreamProxy CreateCSharpDStream(IDStreamProxy jdstream, byte[] func, string deserializer) { var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpDStream", new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, deserializer }); var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream")); return new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference); }
public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string deserializer) { Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(func)); RDD <dynamic> rdd = f(DateTime.UtcNow.Ticks, new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")), new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", ""))); return(new MockDStreamProxy(rdd.RddProxy)); }
public IDStreamProxy CreateCSharpStateDStream(IDStreamProxy jdstream, byte[] func, string className, string serializationMode, string serializationMode2) { var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp." + className, new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, serializationMode, serializationMode2 }); var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream")); return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference)); }
public IDStreamProxy CreateCSharpTransformed2DStream(IDStreamProxy jdstream, IDStreamProxy jother, byte[] func, string deserializer, string deserializerOther) { var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpTransformed2DStream", new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, (jother as DStreamIpcProxy).jvmDStreamReference, func, deserializer, deserializerOther }); var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream")); return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference)); }
public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string deserializer) { var windowDurationReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.Duration", new object[] { windowSeconds * 1000 }); var slideDurationReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.Duration", new object[] { slideSeconds * 1000 }); return new DStreamIpcProxy( SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpReducedWindowedDStream", new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, invFunc, windowDurationReference, slideDurationReference, deserializer }) ); }
public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string serializationMode) { var windowDurationReference = JvmBridgeUtils.GetJavaDuration(windowSeconds); var slideDurationReference = JvmBridgeUtils.GetJavaDuration(slideSeconds); var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpReducedWindowedDStream", new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, invFunc, windowDurationReference, slideDurationReference, serializationMode }); var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream")); return(new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference)); }
public IDStreamProxy Union(IDStreamProxy firstDStream, IDStreamProxy[] otherDStreams) { return(new DStreamIpcProxy( new JvmObjectReference( (string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "union", new object[] { (firstDStream as DStreamIpcProxy).javaDStreamReference, JvmBridgeUtils.GetJavaList <JvmObjectReference>(otherDStreams.Select(x => (x as DStreamIpcProxy).javaDStreamReference)) } )))); }
public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string serializationMode) { Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > f = (Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(func)); var ticks = DateTime.UtcNow.Ticks; RDD <dynamic> rdd = f(ticks, new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")), new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", ""))); if (invFunc == null) { return(new MockDStreamProxy(rdd.RddProxy)); } Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > invf = (Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> >)formatter.Deserialize(new MemoryStream(invFunc)); RDD <dynamic> invRdd = invf(ticks, new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", "")), new RDD <dynamic>((jdstream as MockDStreamProxy).rddProxy ?? new MockRddProxy(null), new SparkContext("", ""))); var difference = rdd.Subtract(invRdd); return(new MockDStreamProxy(difference.RddProxy)); }
internal DStream(IDStreamProxy dstreamProxy, StreamingContext streamingContext, SerializedMode serializedMode = SerializedMode.Byte) { this.streamingContext = streamingContext; this.dstreamProxy = dstreamProxy; this.serializedMode = serializedMode; }
public IDStreamProxy CreateCSharpReducedWindowedDStream(IDStreamProxy jdstream, byte[] func, byte[] invFunc, int windowSeconds, int slideSeconds, string serializationMode) { var windowDurationReference = JvmBridgeUtils.GetJavaDuration(windowSeconds); var slideDurationReference = JvmBridgeUtils.GetJavaDuration(slideSeconds); var jvmDStreamReference = SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpReducedWindowedDStream", new object[] { (jdstream as DStreamIpcProxy).jvmDStreamReference, func, invFunc, windowDurationReference, slideDurationReference, serializationMode }); var javaDStreamReference = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDStreamReference, "asJavaDStream")); return new DStreamIpcProxy(javaDStreamReference, jvmDStreamReference); }
public IDStreamProxy Union(IDStreamProxy firstDStreams, IDStreamProxy[] otherDStreams) { return(new MockDStreamProxy()); }
public IDStreamProxy Union(IDStreamProxy firstDStreams, IDStreamProxy[] otherDStreams) { return new MockDStreamProxy(); }
public IDStreamProxy Union(IDStreamProxy firstDStream, IDStreamProxy[] otherDStreams) { return new DStreamIpcProxy( new JvmObjectReference( (string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaStreamingReference, "union", new object[] { (firstDStream as DStreamIpcProxy).javaDStreamReference, otherDStreams.Select(x => (x as DStreamIpcProxy).javaDStreamReference).ToArray() } ))); }
public IDStreamProxy CreateCSharpTransformed2DStream(IDStreamProxy jdstream, IDStreamProxy jother, byte[] func, string deserializer, string deserializerOther) { var dstream = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((jdstream as DStreamIpcProxy).jvmDStreamReference, "dstream")); var dother = new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod((jother as DStreamIpcProxy).jvmDStreamReference, "dstream")); return new DStreamIpcProxy( SparkCLRIpcProxy.JvmBridge.CallConstructor("org.apache.spark.streaming.api.csharp.CSharpTransformed2DStream", new object[] { dstream, dother, func, deserializer, deserializerOther }) ); }