/// <summary>
/// Return a new DStream in which each RDD is generated by applying a function
/// on each RDD of this DStream and 'other' DStream.
///
/// `func` can have two arguments of (`rdd_a`, `rdd_b`) or have three
/// arguments of (`time`, `rdd_a`, `rdd_b`)
/// </summary>
/// <typeparam name="U"></typeparam>
/// <typeparam name="V"></typeparam>
/// <param name="f"></param>
/// <param name="other"></param>
/// <param name="keepSerializer"></param>
/// <returns></returns>
public DStream <V> TransformWith <U, V>(Func <double, RDD <T>, RDD <U>, RDD <V> > f, DStream <U> other, bool keepSerializer = false)
{
Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > func = new TransformWithDynamicHelper <T, U, V>(f).Execute;
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, func);
return(new DStream <V>(SparkCLREnvironment.SparkCLRProxy.CreateCSharpTransformed2DStream(
DStreamProxy,
other.DStreamProxy,
stream.ToArray(),
serializedMode.ToString(),
other.serializedMode.ToString()),
streamingContext,
keepSerializer ? serializedMode : SerializedMode.Byte));
}
/// <summary>
/// Return a new DStream in which each RDD is generated by applying a function
/// on each RDD of this DStream and 'other' DStream.
///
/// `func` can have two arguments of (`rdd_a`, `rdd_b`) or have three
/// arguments of (`time`, `rdd_a`, `rdd_b`)
/// </summary>
/// <typeparam name="U"></typeparam>
/// <typeparam name="V"></typeparam>
/// <param name="f"></param>
/// <param name="other"></param>
/// <param name="keepSerializer"></param>
/// <returns></returns>
public DStream <V> TransformWith <U, V>(Func <double, RDD <T>, RDD <U>, RDD <V> > f, DStream <U> other, bool keepSerializer = false)
{
Func <double, RDD <dynamic>, RDD <dynamic> > prevF = Piplinable ? (this as TransformedDStream <T>).func : null;
Func <double, RDD <dynamic>, RDD <dynamic> > otherF = other.Piplinable ? (other as TransformedDStream <U>).func : null;
Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > func = new TransformWithDynamicHelper <T, U, V>(f, prevF, otherF).Execute;
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, func);
return(new DStream <V>(SparkCLREnvironment.SparkCLRProxy.StreamingContextProxy.CreateCSharpTransformed2DStream(
Piplinable ? prevDStreamProxy : DStreamProxy,
other.Piplinable ? other.prevDStreamProxy : other.DStreamProxy,
stream.ToArray(),
(Piplinable ? prevSerializedMode : serializedMode).ToString(),
(other.Piplinable ? other.prevSerializedMode : other.serializedMode).ToString()),
streamingContext,
keepSerializer ? serializedMode : SerializedMode.Byte));
}
