public HBaseWriter()
{
//Get the Hadoop Cluster info and create connection
this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
SecureString pw = new SecureString();
for(int i = 0; i < HadoopUserPassword.Length; i++){
pw.InsertAt(i, HadoopUserPassword[i]);
}
Uri clusterUri = new Uri(this.ClusterName);
ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
this.client = new HBaseClient(creds);
//create table and enable the hbase writer
if (!client.ListTables().name.Contains(this.HBaseTableName))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName);
}
WriterThread = new Thread(new ThreadStart(WriterThreadFunction));
WriterThread.Start();
}
public TableSchema CreateTableSchema(string name)
{
//Create a new HBase table.
TableSchema testTableSchema = new TableSchema();
testTableSchema.name = name;
testTableSchema.columns.Add(new ColumnSchema() { name = "CF1" });
return testTableSchema;
}
public void CreateTable()
{
//Create a new HBase table.
TableSchema testTableSchema = new TableSchema();
testTableSchema.name = _sampleTableName;
testTableSchema.columns.Add(new ColumnSchema() { name = "CF1" });
testTableSchema.columns.Add(new ColumnSchema() { name = "CF2" });
_client.CreateTable(testTableSchema);
}
/// <summary>
///
/// </summary>
/// <param name="context"></param>
/// <param name="tablename"></param>
public EventHBaseWriter(Context context, Dictionary<string, Object> parms = null)
{
this.context = context;
this.appConfig = new AppConfig();
Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>();
inputSchema.Add(Constants.DEFAULT_STREAM_ID, AggregatedTupleFields);
this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null));
//Setup the credentials for HBase cluster
HBaseClusterCredentials =
new ClusterCredentials(
new Uri(this.appConfig.HBaseClusterUrl),
this.appConfig.HBaseClusterUserName,
this.appConfig.HBaseClusterUserPassword);
HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);
//Query HBase for existing tables
var tabs = HBaseClusterClient.ListTables();
Context.Logger.Info("HBase Tables (" + tabs.name.Count + "): " + String.Join(", ", tabs.name));
this.PrimaryKey = this.appConfig.PrimaryKey;
this.SecondaryKey = this.appConfig.SecondaryKey;
this.HBaseTableName =
this.appConfig.HBaseTableNamePrefix +
this.appConfig.PrimaryKey + this.appConfig.SecondaryKey +
this.appConfig.HBaseTableNameSuffix;
Context.Logger.Info("HBaseTableName = " + this.HBaseTableName);
//Create a HBase table if it not exists
if (!tabs.name.Contains(this.HBaseTableName))
{
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema() { name = "v" });
HBaseClusterClient.CreateTable(tableSchema);
Context.Logger.Info("Created HBase Table: " + this.HBaseTableName);
}
Context.Logger.Info("HBaseOverwrite: " + this.appConfig.HBaseOverwrite);
globalstopwatch = new Stopwatch();
globalstopwatch.Start();
emitstopwatch = new Stopwatch();
emitstopwatch.Start();
}
/// <summary>
/// Create a new HBase table by async
/// </summary>
/// <param name="table"></param>
/// <returns></returns>
public static async Task<bool> CreateTableAsync(string tableName, string[] columnFamilysKeys)
{
TableSchema tableSchema = new TableSchema();
tableSchema.name = tableName;
foreach (var columnFamilySKey in columnFamilysKeys)
{
tableSchema.columns.Add(new ColumnSchema() { name = columnFamilySKey });
}
if (hbaseClient == null)
hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword);
return await hbaseClient.CreateTableAsync(tableSchema);
}
public HBaseStore()
{
// Initialize HBase connection
var credentials = CreateFromFile(@"credentials.txt");
client = new HBaseClient(credentials);
if (!client.ListTables().name.Contains(TABLE_BY_WORDS_NAME))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = TABLE_BY_WORDS_NAME;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
}
}
/// <summary>
/// Create a new HBase table
/// </summary>
/// <param name="table"></param>
/// <returns></returns>
public static bool CreateTable(string tableName, string[] columnFamilysKeys)
{
TableSchema tableSchema = new TableSchema();
tableSchema.name = tableName;
foreach (var columnFamilySKey in columnFamilysKeys)
{
tableSchema.columns.Add(new ColumnSchema() { name = columnFamilySKey });
}
if (hbaseClient == null)
hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword);
int millisecondsTimeout = 3000;
return hbaseClient.CreateTableAsync(tableSchema).Wait(millisecondsTimeout);
}
/// <summary>
/// Create a new HBase table
/// </summary>
/// <param name="hbaseClient">client used to connect to HBase</param>
public static void CreateHBaseTable(HBaseClient hbaseClient)
{
//Define the 'cf family
//Set versions to retain to 5
ColumnSchema cf = new ColumnSchema() {
name = Properties.Settings.Default.HBaseTableColumnFamily,
maxVersions = 5
};
//Define the table
TableSchema tableSchema = new TableSchema();
tableSchema.name = Properties.Settings.Default.HBaseTableName;
tableSchema.columns.Add(cf);
//Create the table
hbaseClient.CreateTable(tableSchema);
Console.WriteLine("Table created.");
}
protected override void Context()
{
var client = CreateClient();
// ensure tables from previous tests are cleaned up
TableList tables = client.ListTables();
foreach (string name in tables.name)
{
if (name.StartsWith(TestTablePrefix, StringComparison.Ordinal))
{
client.DeleteTable(name);
}
}
// add a table specific to this test
_testTableName = TestTablePrefix + _random.Next(10000);
_testTableSchema = new TableSchema();
_testTableSchema.name = _testTableName;
_testTableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(_testTableSchema);
}
public HBaseWriter()
{
var credentials = CreateFromFile(@"..\..\credentials.txt");
client = new HBaseClient(credentials);
if (!client.ListTables().name.Contains(TABLE_BY_WORDS_NAME))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = TABLE_BY_WORDS_NAME;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" created.", TABLE_BY_WORDS_NAME);
}
// Read current row count cell
rowCount = GetRowCount();
// Load sentiment dictionary file
LoadDictionary();
writerThread = new Thread(new ThreadStart(WriterThreadFunction));
writerThread.Start();
}
// This function connects to HBase, loads the sentiment dictionary, and starts the thread for writting.
public HBaseWriter()
{
ClusterCredentials credentials = new ClusterCredentials(new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD);
client = new HBaseClient(credentials);
// create the HBase table if it doesn't exist
if (!client.ListTables().name.Contains(HBASETABLENAME))
{
TableSchema tableSchema = new TableSchema();
tableSchema.name = HBASETABLENAME;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" is created.", HBASETABLENAME);
}
// Load sentiment dictionary from a file
LoadDictionary();
// Start a thread for writting to HBase
writerThread = new Thread(new ThreadStart(WriterThreadFunction));
writerThread.Start();
}
private static void checkCreateTable(string tableName)
{
try
{
List<string> tables = HadoopContext.HBaseClient.ListTables().name;
if (!tables.Contains(tableName))
{
var tableSchema = new TableSchema();
tableSchema.name = tableName;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
HadoopContext.HBaseClient.CreateTable(tableSchema);
}
}
catch(Exception e)
{
Console.WriteLine(e.Message);
}
}
private void AddTable()
{
// add a table specific to this test
var client = new HBaseClient(_credentials);
_tableName = TableNamePrefix + Guid.NewGuid().ToString("N");
_tableSchema = new TableSchema { name = _tableName };
_tableSchema.columns.Add(new ColumnSchema { name = ColumnFamilyName1 });
_tableSchema.columns.Add(new ColumnSchema { name = ColumnFamilyName2 });
client.CreateTableAsync(_tableSchema).Wait();
}
/// <summary>
/// Initialize the HBase settings and connections
/// </summary>
public void InitializeHBase()
{
this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl");
}
this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName");
}
this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword");
}
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
if (String.IsNullOrWhiteSpace(this.HBaseTableName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName");
}
this.HBaseTableColumnFamily = ConfigurationManager.AppSettings["HBaseTableColumnFamily"];
if (String.IsNullOrWhiteSpace(this.HBaseTableColumnFamily))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnFamily");
}
//TODO - DO NOT include the ROWKEY field in the column list as it is assumed to be at index 0 in tuple fields
//Rest of the tuple fields are mapped with the column names provided
var hbaseTableColumnNames = ConfigurationManager.AppSettings["HBaseTableColumnNames"];
if (String.IsNullOrWhiteSpace(hbaseTableColumnNames))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnNames");
}
//Setup the credentials for HBase cluster
this.HBaseClusterCredentials =
new ClusterCredentials(
new Uri(this.HBaseClusterUrl),
this.HBaseClusterUserName,
this.HBaseClusterPassword);
this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials);
//Query HBase for existing tables
var tables = this.HBaseClusterClient.ListTables();
Context.Logger.Info("Existing HBase tables - Count: {0}, Tables: {1}", tables.name.Count, String.Join(", ", tables.name));
//Read the HBaseTable Columns and convert them to byte[]
this.HBaseTableColumns = hbaseTableColumnNames.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries).
Select(c => c.Trim()).ToList();
//Create a HBase table if it not exists
if (!tables.name.Contains(this.HBaseTableName))
{
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema() { name = this.HBaseTableColumnFamily });
HBaseClusterClient.CreateTable(tableSchema);
Context.Logger.Info("Created HBase table: {0}", this.HBaseTableName);
}
else
{
Context.Logger.Info("Found an existing HBase table: {0}", this.HBaseTableName);
}
}
public async Task TestCreateTable()
{
try
{
var tableSchema = new TableSchema {name = "users", readOnly = false, inMemory = false};
var columnSchema = new ColumnSchema {name = "profile"};
tableSchema.columns.Add(columnSchema);
var succeed = await _hbaseClient.CreateTableAsync(tableSchema);
Assert.IsTrue(succeed);
}
catch (Exception e)
{
Assert.Fail(e.Message);
}
}