public void TestDStreamMapWithStateMapWithStateHelper()
{
// test when initialStateRdd is null
var stateSpec = new StateSpec <string, int, int, int>((k, v, s) => v).NumPartitions(2).Timeout(TimeSpan.FromSeconds(100));
var helper = new MapWithStateHelper <string, int, int, int>((t, rdd) => rdd, stateSpec);
var sparkContextProxy = new Mock <ISparkContextProxy>();
var sc = new SparkContext(sparkContextProxy.Object, null);
var pairwiseRddProxy = new Mock <IRDDProxy>();
sparkContextProxy.Setup(p => p.CreatePairwiseRDD(It.IsAny <IRDDProxy>(), It.IsAny <int>(), It.IsAny <long>())).Returns(pairwiseRddProxy.Object);
var pipelinedRddProxy = new Mock <IRDDProxy>();
pipelinedRddProxy.Setup(p => p.Union(It.IsAny <IRDDProxy>())).Returns(new Mock <IRDDProxy>().Object);
sparkContextProxy.Setup(p =>
p.CreateCSharpRdd(It.IsAny <IRDDProxy>(), It.IsAny <byte[]>(), It.IsAny <Dictionary <string, string> >(), It.IsAny <List <string> >(), It.IsAny <bool>(), It.IsAny <List <Broadcast> >(), It.IsAny <List <byte[]> >()))
.Returns(pipelinedRddProxy.Object);
var valueRddProxy = new Mock <IRDDProxy>();
var valuesRdd = new RDD <dynamic>(valueRddProxy.Object, sc);
var resultRdd = helper.Execute(DateTime.UtcNow.Millisecond, null, valuesRdd);
Assert.IsNotNull(resultRdd);
// test when initialStateRdd is not null
var initialStateRdd = new RDD <KeyValuePair <string, int> >(new Mock <IRDDProxy>().Object, null);
var stateSpec2 = new StateSpec <string, int, int, int>((k, v, s) => v).InitialState(initialStateRdd).NumPartitions(2);
var helper2 = new MapWithStateHelper <string, int, int, int>((t, rdd) => rdd, stateSpec2);
var resultRdd2 = helper2.Execute(DateTime.UtcNow.Millisecond, null, valuesRdd);
Assert.IsNotNull(resultRdd2);
}
public void TestDStreamMapWithStateMapWithStateHelper()
{
// test when initialStateRdd is null
var stateSpec = new StateSpec<string, int, int, int>((k, v, s) => v).NumPartitions(2).Timeout(TimeSpan.FromSeconds(100));
var helper = new MapWithStateHelper<string, int, int, int>((t, rdd) => rdd, stateSpec);
var sparkContextProxy = new Mock<ISparkContextProxy>();
var sc = new SparkContext(sparkContextProxy.Object, null);
var pairwiseRddProxy = new Mock<IRDDProxy>();
sparkContextProxy.Setup(p => p.CreatePairwiseRDD(It.IsAny<IRDDProxy>(), It.IsAny<int>(), It.IsAny<long>())).Returns(pairwiseRddProxy.Object);
var pipelinedRddProxy = new Mock<IRDDProxy>();
pipelinedRddProxy.Setup(p => p.Union(It.IsAny<IRDDProxy>())).Returns(new Mock<IRDDProxy>().Object);
sparkContextProxy.Setup(p =>
p.CreateCSharpRdd(It.IsAny<IRDDProxy>(), It.IsAny<byte[]>(), It.IsAny<Dictionary<string, string>>(), It.IsAny<List<string>>(), It.IsAny<bool>(), It.IsAny<List<Broadcast>>(), It.IsAny<List<byte[]>>()))
.Returns(pipelinedRddProxy.Object);
var valueRddProxy = new Mock<IRDDProxy>();
var valuesRdd = new RDD<dynamic>(valueRddProxy.Object, sc);
var resultRdd = helper.Execute(DateTime.UtcNow.Millisecond, null, valuesRdd);
Assert.IsNotNull(resultRdd);
// test when initialStateRdd is not null
var initialStateRdd = new RDD<KeyValuePair<string, int>>(new Mock<IRDDProxy>().Object, null);
var stateSpec2 = new StateSpec<string, int, int, int>((k, v, s) => v).InitialState(initialStateRdd).NumPartitions(2);
var helper2 = new MapWithStateHelper<string, int, int, int>((t, rdd) => rdd, stateSpec2);
var resultRdd2 = helper2.Execute(DateTime.UtcNow.Millisecond, null, valuesRdd);
Assert.IsNotNull(resultRdd2);
}
