Esempio n. 1
0
        public void TestHiveContextRefreshTable()
        {
            // arrange
            var mockSparkContextProxy = new Mock <ISparkContextProxy>();
            var mockSparkSessionProxy = new Mock <ISparkSessionProxy>();
            var mockCatalogProxy      = new Mock <ICatalogProxy>();

            mockCatalogProxy.Setup(m => m.RefreshTable(It.IsAny <string>()));
            mockSparkSessionProxy.Setup(m => m.GetCatalog()).Returns(mockCatalogProxy.Object);
            mockSparkContextProxy.Setup(m => m.CreateSparkSession()).Returns(mockSparkSessionProxy.Object);

            var mockSparkConfProxy = new Mock <ISparkConfProxy>();

            mockSparkConfProxy.Setup(m => m.GetSparkConfAsString())
            .Returns("spark.master=master;spark.app.name=appname;config1=value1;config2=value2;");

            var conf        = new SparkConf(mockSparkConfProxy.Object);
            var hiveContext = new HiveContext(new SparkContext(mockSparkContextProxy.Object, conf));

            hiveContext.SparkSession.SparkSessionProxy = mockSparkSessionProxy.Object;

            // act
            hiveContext.RefreshTable("table");

            // assert
            mockCatalogProxy.Verify(m => m.RefreshTable("table"));
        }
Esempio n. 2
0
        static void Main(string[] args)
        {
            LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
            var logger = LoggerServiceFactory.GetLogger(typeof(HiveDataFrameExample));

            var sparkConf       = new SparkConf();
            var sparkContext    = new SparkContext(sparkConf);
            var hiveContext     = new HiveContext(sparkContext);
            var peopleDataFrame = hiveContext.Read().Json(Path.Combine(Environment.CurrentDirectory, @"data\people.json"));

            const string dbName    = "SampleHiveDataBaseForMobius";
            const string tableName = "people";

            hiveContext.Sql(string.Format("CREATE DATABASE IF NOT EXISTS {0}", dbName)); // create database if not exists
            hiveContext.Sql(string.Format("USE {0}", dbName));
            hiveContext.Sql(string.Format("DROP TABLE {0}", tableName));                 // drop table if exists

            peopleDataFrame.Write().Mode(SaveMode.Overwrite).SaveAsTable(tableName);     // create table
            var tablesDataFrame = hiveContext.Tables(dbName);                            // get all tables in database

            logger.LogInfo(string.Format("table count in database {0}: {1}", dbName, tablesDataFrame.Count()));
            tablesDataFrame.Show();

            hiveContext.Sql(string.Format("SELECT * FROM {0}", tableName)).Show(); // select from table
        }
Esempio n. 3
0
        public void TestHiveContextSql()
        {
            mockSqlContextProxy.Setup(m => m.Sql(It.IsAny <string>()));
            var hiveContext = new HiveContext(new SparkContext("", ""), mockSqlContextProxy.Object);

            hiveContext.Sql("SELECT * FROM ABC");
            mockSqlContextProxy.Verify(m => m.Sql("SELECT * FROM ABC"));
        }
Esempio n. 4
0
        public void TestHiveContextRefreshTable()
        {
            // arrange
            mockSqlContextProxy.Setup(m => m.RefreshTable(It.IsAny <string>()));
            var hiveContext = new HiveContext(new SparkContext("", ""), mockSqlContextProxy.Object);

            // act
            hiveContext.RefreshTable("table");

            // assert
            mockSqlContextProxy.Verify(m => m.RefreshTable("table"));
        }
Esempio n. 5
0
        public void TestHiveContextConstructor()
        {
            var mockSparkContextProxy = new Mock <ISparkContextProxy>();

            var mockSparkSessionProxy = new Mock <ISparkSessionProxy>();
            var mockCatalogProxy      = new Mock <ICatalogProxy>();

            mockCatalogProxy.Setup(m => m.RefreshTable(It.IsAny <string>()));
            mockSparkSessionProxy.Setup(m => m.GetCatalog()).Returns(mockCatalogProxy.Object);
            mockSparkContextProxy.Setup(m => m.CreateSparkSession()).Returns(mockSparkSessionProxy.Object);

            var mockSparkConfProxy = new Mock <ISparkConfProxy>();

            mockSparkConfProxy.Setup(m => m.GetSparkConfAsString())
            .Returns("spark.master=master;spark.app.name=appname;config1=value1;config2=value2;");

            var conf        = new SparkConf(mockSparkConfProxy.Object);
            var hiveContext = new HiveContext(new SparkContext(mockSparkContextProxy.Object, conf));

            Assert.IsNotNull(hiveContext.SparkSession);
        }
Esempio n. 6
0
 public void TestHiveContextConstructor()
 {
     var hiveContext = new HiveContext(new SparkContext("", ""));
     Assert.IsNotNull((hiveContext.SqlContextProxy as MockSqlContextProxy).mockSqlContextReference);
 }
Esempio n. 7
0
 public BlockService(HiveContext hiveContext, IMapper mapper)
 {
     _hiveContext = hiveContext;
     _mapper      = mapper;
 }