Esempio n. 1
0
        /// <summary>
        /// Return all the RDDs between 'fromTime' to 'toTime' (both included)
        /// </summary>
        /// <param name="fromTimeUtc"></param>
        /// <param name="toTimeUtc"></param>
        /// <returns></returns>
        public RDD <T>[] Slice(DateTime fromTimeUtc, DateTime toTimeUtc)
        {
            long fromUnixTime = (long)(fromTimeUtc - startUtc).TotalMilliseconds;
            long toUnixTime   = (long)(toTimeUtc - startUtc).TotalMilliseconds;

            return(DStreamProxy.Slice(fromUnixTime, toUnixTime).Select(r => new RDD <T>(r, streamingContext.SparkContext, serializedMode)).ToArray());
        }
Esempio n. 2
0
        /// <summary>
        /// Apply a function to each RDD in this DStream.
        /// </summary>
        /// <param name="f"></param>
        public void ForeachRDD(Action <double, RDD <dynamic> > f)
        {
            var formatter = new BinaryFormatter();
            var stream    = new MemoryStream();

            formatter.Serialize(stream, f);
            DStreamProxy.CallForeachRDD(stream.ToArray(), serializedMode.ToString());
        }
Esempio n. 3
0
 /// <summary>
 /// Return a new DStream in which each RDD contains all the elements in seen in a
 /// sliding window of time over this DStream.
 ///
 /// @param windowDuration: width of the window; must be a multiple of this DStream's
 ///                      batching interval
 /// @param slideDuration:  sliding interval of the window (i.e., the interval after which
 ///                      the new DStream will generate RDDs); must be a multiple of this
 ///                      DStream's batching interval
 /// </summary>
 /// <param name="windowSeconds"></param>
 /// <param name="slideSeconds"></param>
 /// <returns></returns>
 public DStream <T> Window(int windowSeconds, int slideSeconds)
 {
     ValidateWindowParam(windowSeconds, slideSeconds);
     return(new DStream <T>(DStreamProxy.Window(windowSeconds, slideSeconds), streamingContext, serializedMode));
 }
Esempio n. 4
0
 /// <summary>
 /// Enable periodic checkpointing of RDDs of this DStream
 /// </summary>
 /// <param name="intervalMs">time in milliseconds, after each period of that, generated RDD will be checkpointed</param>
 /// <returns></returns>
 public DStream <T> Checkpoint(long intervalMs)
 {
     isCheckpointed = true;
     DStreamProxy.Checkpoint(intervalMs);
     return(this);
 }
Esempio n. 5
0
 /// <summary>
 /// Persist the RDDs of this DStream with the given storage level
 /// </summary>
 /// <param name="storageLevelType"></param>
 /// <returns></returns>
 public DStream <T> Persist(StorageLevelType storageLevelType)
 {
     isCached = true;
     DStreamProxy.Persist(storageLevelType);
     return(this);
 }
Esempio n. 6
0
 /// <summary>
 /// Print the first num elements of each RDD generated in this DStream.
 ///
 /// @param num: the number of elements from the first will be printed.
 /// </summary>
 /// <param name="num"></param>
 public void Print(int num = 10)
 {
     DStreamProxy.Print(num);
 }
Esempio n. 7
0
 /// <summary>
 /// Enable periodic checkpointing of RDDs of this DStream
 /// </summary>
 /// <param name="intervalSeconds">time in seconds, after each period of that, generated RDD will be checkpointed</param>
 /// <returns></returns>
 public DStream <T> Checkpoint(int intervalSeconds)
 {
     isCheckpointed = true;
     DStreamProxy.Checkpoint(intervalSeconds);
     return(this);
 }