/// <summary> /// Return a new DStream by applying 'join' between RDDs of this DStream and `other` DStream. /// Hash partitioning is used to generate the RDDs with `numPartitions` partitions. /// </summary> /// <typeparam name="K"></typeparam> /// <typeparam name="V"></typeparam> /// <typeparam name="W"></typeparam> /// <param name="self"></param> /// <param name="other"></param> /// <param name="numPartitions"></param> /// <returns></returns> public static DStream <Tuple <K, Tuple <V, W> > > Join <K, V, W>(this DStream <Tuple <K, V> > self, DStream <Tuple <K, W> > other, int numPartitions = 0) { if (numPartitions <= 0) { numPartitions = self.streamingContext.SparkContext.DefaultParallelism; } return(self.TransformWith <Tuple <K, W>, Tuple <K, Tuple <V, W> > >(new JoinHelper <K, V, W>(numPartitions).Execute, other)); }
/// <summary> /// Return a new DStream by applying 'full outer join' between RDDs of this DStream and `other` DStream. /// Hash partitioning is used to generate the RDDs with `numPartitions` partitions. /// </summary> /// <typeparam name="K"></typeparam> /// <typeparam name="V"></typeparam> /// <typeparam name="W"></typeparam> /// <param name="self"></param> /// <param name="other"></param> /// <param name="numPartitions"></param> /// <returns></returns> public static DStream <KeyValuePair <K, Tuple <Option <V>, Option <W> > > > FullOuterJoin <K, V, W>(this DStream <KeyValuePair <K, V> > self, DStream <KeyValuePair <K, W> > other, int numPartitions = 0) { if (numPartitions <= 0) { numPartitions = self.streamingContext.SparkContext.DefaultParallelism; } return(self.TransformWith <KeyValuePair <K, W>, KeyValuePair <K, Tuple <Option <V>, Option <W> > > >(new FullOuterJoinHelper <K, V, W>(numPartitions).Execute, other)); }
/// <summary> /// Return a new DStream by applying 'cogroup' between RDDs of this DStream and `other` DStream. /// Hash partitioning is used to generate the RDDs with `numPartitions` partitions. /// </summary> /// <typeparam name="K"></typeparam> /// <typeparam name="V"></typeparam> /// <typeparam name="W"></typeparam> /// <param name="self"></param> /// <param name="other"></param> /// <param name="numPartitions"></param> /// <returns></returns> public static DStream <KeyValuePair <K, Tuple <List <V>, List <W> > > > GroupWith <K, V, W>(this DStream <KeyValuePair <K, V> > self, DStream <KeyValuePair <K, W> > other, int numPartitions = 0) { if (numPartitions <= 0) { numPartitions = self.streamingContext.sparkContext.DefaultParallelism; } return(self.TransformWith <KeyValuePair <K, W>, KeyValuePair <K, Tuple <List <V>, List <W> > > >(new GroupWithHelper <K, V, W>(numPartitions).Execute, other)); }