public void TestCSharpInputDStream()
{
// test create CSharpInputDStream
var sc = new SparkContext("", "");
var ssc = new StreamingContext(sc, 1000L);
Func <double, int, IEnumerable <string> > func =
(double time, int pid) =>
{
var list = new List <string>()
{
string.Format("PluggableInputDStream-{0}-{1}", pid, time)
};
return(list.AsEnumerable());
};
const int numPartitions = 5;
var inputDStream = CSharpInputDStreamUtils.CreateStream <string>(
ssc,
numPartitions,
func);
Assert.IsNotNull(inputDStream);
Assert.AreEqual(ssc, inputDStream.streamingContext);
// test CSharpInputDStreamMapPartitionWithIndexHelper
int[] array = new int[numPartitions];
int partitionIndex = 0;
new CSharpInputDStreamMapPartitionWithIndexHelper <string>(0.0, func).Execute(partitionIndex, array.AsEnumerable());
// test CSharpInputDStreamGenerateRDDHelper
new CSharpInputDStreamGenerateRDDHelper <string>(numPartitions, func).Execute(0.0);
}
internal static void DStreamCSharpInputSample()
{
const int numPartitions = 5;
var sc = SparkCLRSamples.SparkContext;
var ssc = new StreamingContext(sc, 2000L); // batch interval is in milliseconds
var inputDStream = CSharpInputDStreamUtils.CreateStream <string>(
ssc,
numPartitions,
(double time, int pid) =>
{
var list = new List <string>()
{
string.Format("PluggableInputDStream-{0}-{1}", pid, time)
};
return(list.AsEnumerable());
});
inputDStream.ForeachRDD((time, rdd) =>
{
var taken = rdd.Collect();
int partitions = rdd.GetNumPartitions();
Console.WriteLine("-------------------------------------------");
Console.WriteLine("Time: {0}", time);
Console.WriteLine("-------------------------------------------");
Console.WriteLine("Count: " + taken.Length);
Console.WriteLine("Partitions: " + partitions);
foreach (object record in taken)
{
Console.WriteLine(record);
}
});
ssc.Start();
ssc.AwaitTermination();
}
