public void TestSparkContextProxy() { var sparkContext = new SparkContext("masterUrl", "appName"); sparkContext.AddFile(null); sparkContext.BinaryFiles(null, null); sparkContext.CancelAllJobs(); sparkContext.CancelJobGroup(null); sparkContext.EmptyRDD <string>(); sparkContext.GetLocalProperty(null); sparkContext.HadoopFile(null, null, null, null); sparkContext.HadoopRDD(null, null, null); sparkContext.NewAPIHadoopFile(null, null, null, null); sparkContext.NewAPIHadoopRDD(null, null, null); sparkContext.Parallelize <int>(new int[] { 1, 2, 3, 4, 5 }); sparkContext.SequenceFile(null, null, null, null, null, null); sparkContext.SetCheckpointDir(null); sparkContext.SetJobGroup(null, null); sparkContext.SetLocalProperty(null, null); sparkContext.SetLogLevel(null); sparkContext.TextFile(null); sparkContext.WholeTextFiles(null); sparkContext.Stop(); sparkContext.Union <string>(null); }
public void TestUnionWhenRddsHaveOnlyOneElement() { var rdds = new RDD <int>[] { new RDD <int>() }; Mock <ISparkContextProxy> sparkContextProxy = new Mock <ISparkContextProxy>(); SparkContext sc = new SparkContext(sparkContextProxy.Object, null); RDD <int> result = sc.Union <int>(rdds); Assert.IsNotNull(result); Assert.AreEqual(rdds[0], result); }
public RDD <T> TestUnion <T>(IEnumerable <RDD <T> > rdds) { Mock <IRDDProxy> rddProxy = new Mock <IRDDProxy>(); Mock <ISparkContextProxy> sparkContextProxy = new Mock <ISparkContextProxy>(); sparkContextProxy.Setup(m => m.EmptyRDD()).Returns(rddProxy.Object); SparkContext sc = new SparkContext(sparkContextProxy.Object, null); RDD <T> result = sc.Union <T>(rdds); Assert.IsNotNull(result); sparkContextProxy.Verify(m => m.EmptyRDD(), Times.Once); return(result); }
public void TestUnion() { var rdds = new RDD <string>[] { new RDD <string>(new Mock <IRDDProxy>().Object, null, SerializedMode.String), new RDD <string>(new Mock <IRDDProxy>().Object, null, SerializedMode.String) }; Mock <IRDDProxy> rddProxy = new Mock <IRDDProxy>(); Mock <ISparkContextProxy> sparkContextProxy = new Mock <ISparkContextProxy>(); sparkContextProxy.Setup(m => m.Union(It.IsAny <IEnumerable <IRDDProxy> >())).Returns(rddProxy.Object); SparkContext sc = new SparkContext(sparkContextProxy.Object, null); RDD <string> result = sc.Union <string>(rdds); Assert.IsNotNull(result); Assert.AreEqual(rdds[0].serializedMode, result.serializedMode); Assert.AreEqual(rddProxy.Object, result.RddProxy); }
public void TestSparkContextProxy() { var sparkContext = new SparkContext("masterUrl", "appName"); sparkContext.AddFile(null); sparkContext.BinaryFiles(null, null); sparkContext.CancelAllJobs(); sparkContext.CancelJobGroup(null); sparkContext.EmptyRDD<string>(); sparkContext.GetLocalProperty(null); sparkContext.HadoopFile(null, null, null, null); sparkContext.HadoopRDD(null, null, null); sparkContext.NewAPIHadoopFile(null, null, null, null); sparkContext.NewAPIHadoopRDD(null, null, null); sparkContext.Parallelize<int>(new int[] { 1, 2, 3, 4, 5 }); sparkContext.SequenceFile(null, null, null, null, null, null); sparkContext.SetCheckpointDir(null); sparkContext.SetJobGroup(null, null); sparkContext.SetLocalProperty(null, null); sparkContext.SetLogLevel(null); sparkContext.TextFile(null); sparkContext.WholeTextFiles(null); sparkContext.Stop(); sparkContext.Union<string>(null); }