public void TestHiveContextRefreshTable()
{
// arrange
var mockSparkContextProxy = new Mock <ISparkContextProxy>();
var mockSparkSessionProxy = new Mock <ISparkSessionProxy>();
var mockCatalogProxy = new Mock <ICatalogProxy>();
mockCatalogProxy.Setup(m => m.RefreshTable(It.IsAny <string>()));
mockSparkSessionProxy.Setup(m => m.GetCatalog()).Returns(mockCatalogProxy.Object);
mockSparkContextProxy.Setup(m => m.CreateSparkSession()).Returns(mockSparkSessionProxy.Object);
var mockSparkConfProxy = new Mock <ISparkConfProxy>();
mockSparkConfProxy.Setup(m => m.GetSparkConfAsString())
.Returns("spark.master=master;spark.app.name=appname;config1=value1;config2=value2;");
var conf = new SparkConf(mockSparkConfProxy.Object);
var hiveContext = new HiveContext(new SparkContext(mockSparkContextProxy.Object, conf));
hiveContext.SparkSession.SparkSessionProxy = mockSparkSessionProxy.Object;
// act
hiveContext.RefreshTable("table");
// assert
mockCatalogProxy.Verify(m => m.RefreshTable("table"));
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
var logger = LoggerServiceFactory.GetLogger(typeof(HiveDataFrameExample));
var sparkConf = new SparkConf();
var sparkContext = new SparkContext(sparkConf);
var hiveContext = new HiveContext(sparkContext);
var peopleDataFrame = hiveContext.Read().Json(Path.Combine(Environment.CurrentDirectory, @"data\people.json"));
const string dbName = "SampleHiveDataBaseForMobius";
const string tableName = "people";
hiveContext.Sql(string.Format("CREATE DATABASE IF NOT EXISTS {0}", dbName)); // create database if not exists
hiveContext.Sql(string.Format("USE {0}", dbName));
hiveContext.Sql(string.Format("DROP TABLE {0}", tableName)); // drop table if exists
peopleDataFrame.Write().Mode(SaveMode.Overwrite).SaveAsTable(tableName); // create table
var tablesDataFrame = hiveContext.Tables(dbName); // get all tables in database
logger.LogInfo(string.Format("table count in database {0}: {1}", dbName, tablesDataFrame.Count()));
tablesDataFrame.Show();
hiveContext.Sql(string.Format("SELECT * FROM {0}", tableName)).Show(); // select from table
}
public void TestHiveContextSql()
{
mockSqlContextProxy.Setup(m => m.Sql(It.IsAny <string>()));
var hiveContext = new HiveContext(new SparkContext("", ""), mockSqlContextProxy.Object);
hiveContext.Sql("SELECT * FROM ABC");
mockSqlContextProxy.Verify(m => m.Sql("SELECT * FROM ABC"));
}
public void TestHiveContextRefreshTable()
{
// arrange
mockSqlContextProxy.Setup(m => m.RefreshTable(It.IsAny <string>()));
var hiveContext = new HiveContext(new SparkContext("", ""), mockSqlContextProxy.Object);
// act
hiveContext.RefreshTable("table");
// assert
mockSqlContextProxy.Verify(m => m.RefreshTable("table"));
}
public void TestHiveContextConstructor()
{
var mockSparkContextProxy = new Mock <ISparkContextProxy>();
var mockSparkSessionProxy = new Mock <ISparkSessionProxy>();
var mockCatalogProxy = new Mock <ICatalogProxy>();
mockCatalogProxy.Setup(m => m.RefreshTable(It.IsAny <string>()));
mockSparkSessionProxy.Setup(m => m.GetCatalog()).Returns(mockCatalogProxy.Object);
mockSparkContextProxy.Setup(m => m.CreateSparkSession()).Returns(mockSparkSessionProxy.Object);
var mockSparkConfProxy = new Mock <ISparkConfProxy>();
mockSparkConfProxy.Setup(m => m.GetSparkConfAsString())
.Returns("spark.master=master;spark.app.name=appname;config1=value1;config2=value2;");
var conf = new SparkConf(mockSparkConfProxy.Object);
var hiveContext = new HiveContext(new SparkContext(mockSparkContextProxy.Object, conf));
Assert.IsNotNull(hiveContext.SparkSession);
}
public void TestHiveContextConstructor()
{
var hiveContext = new HiveContext(new SparkContext("", ""));
Assert.IsNotNull((hiveContext.SqlContextProxy as MockSqlContextProxy).mockSqlContextReference);
}
public BlockService(HiveContext hiveContext, IMapper mapper)
{
_hiveContext = hiveContext;
_mapper = mapper;
}