/// <summary>
/// Return all the RDDs between 'fromTime' to 'toTime' (both included)
/// </summary>
/// <param name="fromTimeUtc"></param>
/// <param name="toTimeUtc"></param>
/// <returns></returns>
public RDD <T>[] Slice(DateTime fromTimeUtc, DateTime toTimeUtc)
{
long fromUnixTime = (long)(fromTimeUtc - startUtc).TotalMilliseconds;
long toUnixTime = (long)(toTimeUtc - startUtc).TotalMilliseconds;
return(DStreamProxy.Slice(fromUnixTime, toUnixTime).Select(r => new RDD <T>(r, streamingContext.SparkContext, serializedMode)).ToArray());
}
/// <summary>
/// Apply a function to each RDD in this DStream.
/// </summary>
/// <param name="f"></param>
public void ForeachRDD(Action <double, RDD <dynamic> > f)
{
var formatter = new BinaryFormatter();
var stream = new MemoryStream();
formatter.Serialize(stream, f);
DStreamProxy.CallForeachRDD(stream.ToArray(), serializedMode.ToString());
}
/// <summary>
/// Return a new DStream in which each RDD contains all the elements in seen in a
/// sliding window of time over this DStream.
///
/// @param windowDuration: width of the window; must be a multiple of this DStream's
/// batching interval
/// @param slideDuration: sliding interval of the window (i.e., the interval after which
/// the new DStream will generate RDDs); must be a multiple of this
/// DStream's batching interval
/// </summary>
/// <param name="windowSeconds"></param>
/// <param name="slideSeconds"></param>
/// <returns></returns>
public DStream <T> Window(int windowSeconds, int slideSeconds)
{
ValidateWindowParam(windowSeconds, slideSeconds);
return(new DStream <T>(DStreamProxy.Window(windowSeconds, slideSeconds), streamingContext, serializedMode));
}
/// <summary>
/// Enable periodic checkpointing of RDDs of this DStream
/// </summary>
/// <param name="intervalMs">time in milliseconds, after each period of that, generated RDD will be checkpointed</param>
/// <returns></returns>
public DStream <T> Checkpoint(long intervalMs)
{
isCheckpointed = true;
DStreamProxy.Checkpoint(intervalMs);
return(this);
}
/// <summary>
/// Persist the RDDs of this DStream with the given storage level
/// </summary>
/// <param name="storageLevelType"></param>
/// <returns></returns>
public DStream <T> Persist(StorageLevelType storageLevelType)
{
isCached = true;
DStreamProxy.Persist(storageLevelType);
return(this);
}
/// <summary>
/// Print the first num elements of each RDD generated in this DStream.
///
/// @param num: the number of elements from the first will be printed.
/// </summary>
/// <param name="num"></param>
public void Print(int num = 10)
{
DStreamProxy.Print(num);
}
/// <summary>
/// Enable periodic checkpointing of RDDs of this DStream
/// </summary>
/// <param name="intervalSeconds">time in seconds, after each period of that, generated RDD will be checkpointed</param>
/// <returns></returns>
public DStream <T> Checkpoint(int intervalSeconds)
{
isCheckpointed = true;
DStreamProxy.Checkpoint(intervalSeconds);
return(this);
}
