public void TestHiveContextRefreshTable() { // arrange var mockSparkContextProxy = new Mock <ISparkContextProxy>(); var mockSparkSessionProxy = new Mock <ISparkSessionProxy>(); var mockCatalogProxy = new Mock <ICatalogProxy>(); mockCatalogProxy.Setup(m => m.RefreshTable(It.IsAny <string>())); mockSparkSessionProxy.Setup(m => m.GetCatalog()).Returns(mockCatalogProxy.Object); mockSparkContextProxy.Setup(m => m.CreateSparkSession()).Returns(mockSparkSessionProxy.Object); var mockSparkConfProxy = new Mock <ISparkConfProxy>(); mockSparkConfProxy.Setup(m => m.GetSparkConfAsString()) .Returns("spark.master=master;spark.app.name=appname;config1=value1;config2=value2;"); var conf = new SparkConf(mockSparkConfProxy.Object); var hiveContext = new HiveContext(new SparkContext(mockSparkContextProxy.Object, conf)); hiveContext.SparkSession.SparkSessionProxy = mockSparkSessionProxy.Object; // act hiveContext.RefreshTable("table"); // assert mockCatalogProxy.Verify(m => m.RefreshTable("table")); }
static void Main(string[] args) { LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set var logger = LoggerServiceFactory.GetLogger(typeof(HiveDataFrameExample)); var sparkConf = new SparkConf(); var sparkContext = new SparkContext(sparkConf); var hiveContext = new HiveContext(sparkContext); var peopleDataFrame = hiveContext.Read().Json(Path.Combine(Environment.CurrentDirectory, @"data\people.json")); const string dbName = "SampleHiveDataBaseForMobius"; const string tableName = "people"; hiveContext.Sql(string.Format("CREATE DATABASE IF NOT EXISTS {0}", dbName)); // create database if not exists hiveContext.Sql(string.Format("USE {0}", dbName)); hiveContext.Sql(string.Format("DROP TABLE {0}", tableName)); // drop table if exists peopleDataFrame.Write().Mode(SaveMode.Overwrite).SaveAsTable(tableName); // create table var tablesDataFrame = hiveContext.Tables(dbName); // get all tables in database logger.LogInfo(string.Format("table count in database {0}: {1}", dbName, tablesDataFrame.Count())); tablesDataFrame.Show(); hiveContext.Sql(string.Format("SELECT * FROM {0}", tableName)).Show(); // select from table }
public void TestHiveContextSql() { mockSqlContextProxy.Setup(m => m.Sql(It.IsAny <string>())); var hiveContext = new HiveContext(new SparkContext("", ""), mockSqlContextProxy.Object); hiveContext.Sql("SELECT * FROM ABC"); mockSqlContextProxy.Verify(m => m.Sql("SELECT * FROM ABC")); }
public void TestHiveContextRefreshTable() { // arrange mockSqlContextProxy.Setup(m => m.RefreshTable(It.IsAny <string>())); var hiveContext = new HiveContext(new SparkContext("", ""), mockSqlContextProxy.Object); // act hiveContext.RefreshTable("table"); // assert mockSqlContextProxy.Verify(m => m.RefreshTable("table")); }
public void TestHiveContextConstructor() { var mockSparkContextProxy = new Mock <ISparkContextProxy>(); var mockSparkSessionProxy = new Mock <ISparkSessionProxy>(); var mockCatalogProxy = new Mock <ICatalogProxy>(); mockCatalogProxy.Setup(m => m.RefreshTable(It.IsAny <string>())); mockSparkSessionProxy.Setup(m => m.GetCatalog()).Returns(mockCatalogProxy.Object); mockSparkContextProxy.Setup(m => m.CreateSparkSession()).Returns(mockSparkSessionProxy.Object); var mockSparkConfProxy = new Mock <ISparkConfProxy>(); mockSparkConfProxy.Setup(m => m.GetSparkConfAsString()) .Returns("spark.master=master;spark.app.name=appname;config1=value1;config2=value2;"); var conf = new SparkConf(mockSparkConfProxy.Object); var hiveContext = new HiveContext(new SparkContext(mockSparkContextProxy.Object, conf)); Assert.IsNotNull(hiveContext.SparkSession); }
public void TestHiveContextConstructor() { var hiveContext = new HiveContext(new SparkContext("", "")); Assert.IsNotNull((hiveContext.SqlContextProxy as MockSqlContextProxy).mockSqlContextReference); }
public BlockService(HiveContext hiveContext, IMapper mapper) { _hiveContext = hiveContext; _mapper = mapper; }