Exemplo n.º 1
0
        public void TestSparkContextProxy()
        {
            var sparkContext = new SparkContext("masterUrl", "appName");

            sparkContext.AddFile(null);
            sparkContext.BinaryFiles(null, null);
            sparkContext.CancelAllJobs();
            sparkContext.CancelJobGroup(null);
            sparkContext.EmptyRDD <string>();
            sparkContext.GetLocalProperty(null);
            sparkContext.HadoopFile(null, null, null, null);
            sparkContext.HadoopRDD(null, null, null);
            sparkContext.NewAPIHadoopFile(null, null, null, null);
            sparkContext.NewAPIHadoopRDD(null, null, null);
            sparkContext.Parallelize <int>(new int[] { 1, 2, 3, 4, 5 });
            sparkContext.SequenceFile(null, null, null, null, null, null);
            sparkContext.SetCheckpointDir(null);
            sparkContext.SetJobGroup(null, null);
            sparkContext.SetLocalProperty(null, null);
            sparkContext.SetLogLevel(null);
            sparkContext.TextFile(null);
            sparkContext.WholeTextFiles(null);
            sparkContext.Stop();
            sparkContext.Union <string>(null);
        }
Exemplo n.º 2
0
        public void TestSequenceFiles()
        {
            // Arrange
            const string filePath = @"hdfs://path/to/files";

            const int    defaultParallelism  = 10;
            const string keyClass            = "java.lang.Long";
            const string valueClass          = "java.lang.String";
            const string keyConverterClass   = "xyz.KeyConveter";
            const string valueConverterClass = "xyz.valueConveter";

            Mock <IRDDProxy>          rddProxy          = new Mock <IRDDProxy>();
            Mock <ISparkContextProxy> sparkContextProxy = new Mock <ISparkContextProxy>();

            sparkContextProxy.Setup(m => m.SequenceFile(filePath, keyClass, valueClass, keyConverterClass, valueConverterClass, It.IsAny <int>(), It.IsAny <int>()))
            .Returns(rddProxy.Object);
            sparkContextProxy.Setup(m => m.DefaultParallelism).Returns(defaultParallelism);
            SparkContext sc = new SparkContext(sparkContextProxy.Object, null);

            // Act
            RDD <byte[]> rdd = sc.SequenceFile(filePath, keyClass, valueClass, keyConverterClass, valueConverterClass, null);

            // Assert
            Assert.IsNotNull(rdd);
            Assert.AreEqual(rddProxy.Object, rdd.RddProxy);
            Assert.AreEqual(sc, rdd.sparkContext);
            Assert.AreEqual(SerializedMode.None, rdd.serializedMode);
        }
Exemplo n.º 3
0
 public void TestSparkContextProxy()
 {
     var sparkContext = new SparkContext("masterUrl", "appName");
     sparkContext.AddFile(null);
     sparkContext.BinaryFiles(null, null);
     sparkContext.CancelAllJobs();
     sparkContext.CancelJobGroup(null);
     sparkContext.EmptyRDD<string>();
     sparkContext.GetLocalProperty(null);
     sparkContext.HadoopFile(null, null, null, null);
     sparkContext.HadoopRDD(null, null, null);
     sparkContext.NewAPIHadoopFile(null, null, null, null);
     sparkContext.NewAPIHadoopRDD(null, null, null);
     sparkContext.Parallelize<int>(new int[] { 1, 2, 3, 4, 5 });
     sparkContext.SequenceFile(null, null, null, null, null, null);
     sparkContext.SetCheckpointDir(null);
     sparkContext.SetJobGroup(null, null);
     sparkContext.SetLocalProperty(null, null);
     sparkContext.SetLogLevel(null);
     sparkContext.TextFile(null);
     sparkContext.WholeTextFiles(null);
     sparkContext.Stop();
     sparkContext.Union<string>(null);
 }