/// <summary> /// Return a new DStream in which each RDD is generated by applying a function /// on each RDD of this DStream and 'other' DStream. /// /// `func` can have two arguments of (`rdd_a`, `rdd_b`) or have three /// arguments of (`time`, `rdd_a`, `rdd_b`) /// </summary> /// <typeparam name="U"></typeparam> /// <typeparam name="V"></typeparam> /// <param name="f"></param> /// <param name="other"></param> /// <param name="keepSerializer"></param> /// <returns></returns> public DStream <V> TransformWith <U, V>(Func <double, RDD <T>, RDD <U>, RDD <V> > f, DStream <U> other, bool keepSerializer = false) { Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > func = new TransformWithDynamicHelper <T, U, V>(f).Execute; var formatter = new BinaryFormatter(); var stream = new MemoryStream(); formatter.Serialize(stream, func); return(new DStream <V>(SparkCLREnvironment.SparkCLRProxy.CreateCSharpTransformed2DStream( DStreamProxy, other.DStreamProxy, stream.ToArray(), serializedMode.ToString(), other.serializedMode.ToString()), streamingContext, keepSerializer ? serializedMode : SerializedMode.Byte)); }
/// <summary> /// Return a new DStream in which each RDD is generated by applying a function /// on each RDD of this DStream and 'other' DStream. /// /// `func` can have two arguments of (`rdd_a`, `rdd_b`) or have three /// arguments of (`time`, `rdd_a`, `rdd_b`) /// </summary> /// <typeparam name="U"></typeparam> /// <typeparam name="V"></typeparam> /// <param name="f"></param> /// <param name="other"></param> /// <param name="keepSerializer"></param> /// <returns></returns> public DStream <V> TransformWith <U, V>(Func <double, RDD <T>, RDD <U>, RDD <V> > f, DStream <U> other, bool keepSerializer = false) { Func <double, RDD <dynamic>, RDD <dynamic> > prevF = Piplinable ? (this as TransformedDStream <T>).func : null; Func <double, RDD <dynamic>, RDD <dynamic> > otherF = other.Piplinable ? (other as TransformedDStream <U>).func : null; Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > func = new TransformWithDynamicHelper <T, U, V>(f, prevF, otherF).Execute; var formatter = new BinaryFormatter(); var stream = new MemoryStream(); formatter.Serialize(stream, func); return(new DStream <V>(SparkCLREnvironment.SparkCLRProxy.StreamingContextProxy.CreateCSharpTransformed2DStream( Piplinable ? prevDStreamProxy : DStreamProxy, other.Piplinable ? other.prevDStreamProxy : other.DStreamProxy, stream.ToArray(), (Piplinable ? prevSerializedMode : serializedMode).ToString(), (other.Piplinable ? other.prevSerializedMode : other.serializedMode).ToString()), streamingContext, keepSerializer ? serializedMode : SerializedMode.Byte)); }