コード例 #1
0
        /// <summary>
        /// Return a new "state" DStream where the state for each key is updated by applying
        /// the given function on the previous state of the key and the new values of the key.
        /// </summary>
        public static MapWithStateDStream <K, V, S, M> MapWithState <K, V, S, M>(this DStream <Tuple <K, V> > self, StateSpec <K, V, S, M> stateSpec)
        {
            if (stateSpec.numPartitions <= 0)
            {
                stateSpec = stateSpec.NumPartitions(self.streamingContext.SparkContext.DefaultParallelism);
            }

            Func <double, RDD <dynamic>, RDD <dynamic> > prevFunc = self.Piplinable ? (self as TransformedDStream <Tuple <K, V> >).func : null;

            Func <double, RDD <dynamic>, RDD <dynamic>, RDD <dynamic> > func = new MapWithStateHelper <K, V, S, M>(prevFunc, stateSpec).Execute;

            var formatter = new BinaryFormatter();
            var stream    = new MemoryStream();

            formatter.Serialize(stream, func);

            var mapWithStateDStream = new DStream <MapWithStateRDDRecord <K, S, M> >(SparkCLREnvironment.SparkCLRProxy.StreamingContextProxy.CreateCSharpStateDStream(
                                                                                         self.Piplinable ? self.prevDStreamProxy : self.DStreamProxy,
                                                                                         stream.ToArray(),
                                                                                         "CSharpStateDStream",
                                                                                         self.serializedMode.ToString(),
                                                                                         (self.Piplinable ? self.prevSerializedMode : self.serializedMode).ToString()),
                                                                                     self.streamingContext);

            DStream <M>             mappedDataDStream = mapWithStateDStream.FlatMap(r => r.mappedData);
            DStream <Tuple <K, S> > snapshotsDStream  = mapWithStateDStream.FlatMap(
                r => r.stateMap.Select(entry => new Tuple <K, S>(entry.Key, entry.Value.state)));

            return(new MapWithStateDStream <K, V, S, M>(mappedDataDStream, snapshotsDStream));
        }
コード例 #2
0
 internal MapWithStateHelper(Func <double, RDD <dynamic>, RDD <dynamic> > prevF, StateSpec <K, V, S, M> stateSpec)
 {
     prevFunc       = prevF;
     this.stateSpec = stateSpec;
 }