public HBaseWriter()
{
//Get the Hadoop Cluster info and create connection
this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
SecureString pw = new SecureString();
for(int i = 0; i < HadoopUserPassword.Length; i++){
pw.InsertAt(i, HadoopUserPassword[i]);
}
Uri clusterUri = new Uri(this.ClusterName);
ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
this.client = new HBaseClient(creds);
//create table and enable the hbase writer
if (!client.ListTables().name.Contains(this.HBaseTableName))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName);
}
WriterThread = new Thread(new ThreadStart(WriterThreadFunction));
WriterThread.Start();
}
// The constructor
public HBaseReader()
{
ClusterCredentials creds = new ClusterCredentials(
new Uri(CLUSTERNAME),
HADOOPUSERNAME,
HADOOPUSERPASSWORD);
client = new HBaseClient(creds);
}
public HBaseReader()
{
var creds = new ClusterCredentials(
new Uri(ConfigurationManager.AppSettings["Cluster"]),
ConfigurationManager.AppSettings["User"],
ConfigurationManager.AppSettings["Pwd"]);
client = new HBaseClient(creds);
}
/// <summary>
/// Initializes a new instance of the <see cref="WebRequesterSecure"/> class.
/// </summary>
/// <param name="credentials">The credentials.</param>
/// <param name="contentType">Type of the content.</param>
public WebRequesterSecure(ClusterCredentials credentials, string contentType = "application/x-protobuf")
{
credentials.ArgumentNotNull("credentials");
_credentials = credentials;
_contentType = contentType;
_credentialCache = new CredentialCache();
InitCache();
}
public HBaseReaderClient(string hbaseclusterurl, string hbaseclusterusername, string hbaseclusteruserpassword)
{
this.HBaseClusterUrl = hbaseclusterurl;
this.HBaseClusterUserName = hbaseclusterusername;
this.HBaseClusterUserPassword = hbaseclusteruserpassword;
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true;
this.HBaseClusterCredentials = new ClusterCredentials(new Uri(HBaseClusterUrl), HBaseClusterUserName, HBaseClusterUserPassword);
this.HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);
}
/// <summary>
///
/// </summary>
/// <param name="context"></param>
/// <param name="tablename"></param>
public EventHBaseWriter(Context context, Dictionary<string, Object> parms = null)
{
this.context = context;
this.appConfig = new AppConfig();
Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>();
inputSchema.Add(Constants.DEFAULT_STREAM_ID, AggregatedTupleFields);
this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null));
//Setup the credentials for HBase cluster
HBaseClusterCredentials =
new ClusterCredentials(
new Uri(this.appConfig.HBaseClusterUrl),
this.appConfig.HBaseClusterUserName,
this.appConfig.HBaseClusterUserPassword);
HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);
//Query HBase for existing tables
var tabs = HBaseClusterClient.ListTables();
Context.Logger.Info("HBase Tables (" + tabs.name.Count + "): " + String.Join(", ", tabs.name));
this.PrimaryKey = this.appConfig.PrimaryKey;
this.SecondaryKey = this.appConfig.SecondaryKey;
this.HBaseTableName =
this.appConfig.HBaseTableNamePrefix +
this.appConfig.PrimaryKey + this.appConfig.SecondaryKey +
this.appConfig.HBaseTableNameSuffix;
Context.Logger.Info("HBaseTableName = " + this.HBaseTableName);
//Create a HBase table if it not exists
if (!tabs.name.Contains(this.HBaseTableName))
{
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema() { name = "v" });
HBaseClusterClient.CreateTable(tableSchema);
Context.Logger.Info("Created HBase Table: " + this.HBaseTableName);
}
Context.Logger.Info("HBaseOverwrite: " + this.appConfig.HBaseOverwrite);
globalstopwatch = new Stopwatch();
globalstopwatch.Start();
emitstopwatch = new Stopwatch();
emitstopwatch.Start();
}
public HBaseClient(ClusterCredentials credentials, RequestOptions globalRequestOptions = null, ILoadBalancer loadBalancer = null)
{
_globalRequestOptions = globalRequestOptions ?? RequestOptions.GetDefaultOptions();
_globalRequestOptions.Validate();
if (credentials != null) // gateway mode
{
_requester = new GatewayWebRequester(credentials);
}
else // vnet mode
{
_requester = new VNetWebRequester(loadBalancer);
}
}
static void Main(string[] args)
{
while (true)
{
Random rnd = new Random();
Console.Clear();
string clusterURL = "https://hb12345.azurehdinsight.net";
string userName = "******";
string password = "******";
// Connect to HBase cluster
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL),
userName, password);
HBaseClient hbaseClient = new HBaseClient(creds);
// Get all stocks
Scanner scanSettings = new Scanner()
{
batch = 10,
startRow = Encoding.UTF8.GetBytes("AAA"),
endRow = Encoding.UTF8.GetBytes("ZZZ")
};
ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings);
CellSet stockCells = null;
while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null)
{
foreach (var row in stockCells.rows)
{
string stock = Encoding.UTF8.GetString(row.key);
Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1);
Cell c = new Cell
{
column = Encoding.UTF8.GetBytes("Current:Price"),
data =
Encoding.UTF8.GetBytes(newPrice.ToString())
};
row.values.Insert(2, c);
Console.WriteLine(stock + ": " + currentPrice.ToString() + " := "
+ newPrice.ToString());
}
hbaseClient.StoreCells("Stocks", stockCells);
}
}
}
public HBaseReader()
{
//Get the Hadoop Cluster info and create connection
this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
SecureString pw = new SecureString();
for (int i = 0; i < HadoopUserPassword.Length; i++)
{
pw.InsertAt(i, HadoopUserPassword[i]);
}
Uri clusterUri = new Uri(this.ClusterName);
ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
this.client = new HBaseClient(creds);
}
/// <summary>
/// Initializes a new instance of the <see cref="HBaseClient"/> class.
/// </summary>
/// <param name="credentials">The credentials.</param>
/// <param name="retryPolicyFactory">The retry policy factory.</param>
public HBaseClient(ClusterCredentials credentials, IRetryPolicyFactory retryPolicyFactory, ILoadBalancer loadBalancer = null)
{
retryPolicyFactory.ArgumentNotNull("retryPolicyFactory");
if (credentials != null)
{
_requester = new WebRequesterSecure(credentials);
}
else
{
_requester = new WebRequesterBasic();
_loadBalancer = loadBalancer;
}
_retryPolicyFactory = retryPolicyFactory;
}
static void Main(string[] args)
{
bool quit = false;
while (!quit)
{
Console.ResetColor();
Console.WriteLine("Enter a stock code, or enter 'quit' to exit");
// Connect to HBase cluster
string clusterURL = "https://hb12345.azurehdinsight.net";
string userName = "******";
string password = "******";
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password);
HBaseClient hbaseClient = new HBaseClient(creds);
string input = Console.ReadLine();
if (input.ToLower() == "quit")
{
quit = true;
}
else
{
CellSet cellSet = hbaseClient.GetCells("Stocks", input);
var row = cellSet.rows[0];
Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
Double closingPrice = Double.Parse(Encoding.UTF8.GetString(row.values[0].data));
if (currentPrice > closingPrice)
{
Console.ForegroundColor = ConsoleColor.Green;
}
else
if (currentPrice < closingPrice)
{
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine(input + ": " + currentPrice.ToString());
}
}
}
static void Main(string[] args)
{
//Connect to HBase
var credentials = new ClusterCredentials(
new Uri(Properties.Settings.Default.HBaseClusterUrl),
Properties.Settings.Default.HBaseClusterUserName,
Properties.Settings.Default.HBaseClusterPassword);
//Get the client
var hbaseClient = new HBaseClient(credentials);
char operation = ' ';
//Loop until 'q' is selected
do {
Console.Write("(C)reate table, (d)elete table, or search for (s)tart or (e)nd events. Or (q)uit: ");
//Get the character
operation = Console.ReadKey().KeyChar;
Console.WriteLine();
//Which operation?
switch (operation)
{
case 'c':
CreateHBaseTable(hbaseClient);
break;
case 'd':
DeleteHBaseTable(hbaseClient);
break;
case 's':
//Fall through since both Start and Event searches
//have the same logic/signature
case 'e':
DateTime start = GetDateTime("start");
DateTime end = GetDateTime("end");
//Was it start or end we want to look for?
string eventType = operation == 's' ? "START" : "END";
//Find sessions for the event type, by start and end times
GetSessionsByTime(hbaseClient, eventType, start, end);
break;
default:
break;
}
} while (operation != 'q');
}
/// <summary>
/// Create a new instance of an HBase client.
/// </summary>
/// <param name="clusterURL"></param>
/// <param name="httpName"></param>
/// <param name="httpUserPassword"></param>
/// <returns></returns>
private static HBaseClient CreateHBaseClient(string clusterURL, string httpName, string httpUserPassword)
{
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), httpName, httpUserPassword);
return new HBaseClient(creds);
}
// This function connects to HBase, loads the sentiment dictionary, and starts the thread for writting.
public HBaseWriter()
{
ClusterCredentials credentials = new ClusterCredentials(new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD);
client = new HBaseClient(credentials);
// create the HBase table if it doesn't exist
if (!client.ListTables().name.Contains(HBASETABLENAME))
{
TableSchema tableSchema = new TableSchema();
tableSchema.name = HBASETABLENAME;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" is created.", HBASETABLENAME);
}
// Load sentiment dictionary from a file
LoadDictionary();
// Start a thread for writting to HBase
writerThread = new Thread(new ThreadStart(WriterThreadFunction));
writerThread.Start();
}
/// <summary>
/// Initializes a new instance of the <see cref="HBaseClient"/> class.
/// </summary>
/// <param name="credentials">The credentials.</param>
public HBaseClient(ClusterCredentials credentials)
: this(credentials, RequestOptions.GetDefaultOptions())
{
}
internal static ClusterCredentials CreateFromList(List<string> lines)
{
if (lines.Count() != 3)
{
throw new ArgumentException(
string.Format(
CultureInfo.InvariantCulture,
"Expected the credentials file to have exactly three lines, " +
"first containing the cluster URL, second the username, third the password. " + "Given {0} lines!",
lines.Count()),
"lines");
}
var rv = new ClusterCredentials(new Uri(lines[0]), lines[1], lines[2]);
return rv;
}
public bool StartListening()
{
Background = Task.Run(async () =>
{
while (true)
{
var clusterURL = ConfigurationManager.AppSettings["ClusterUrl"];
var hadoopUsername = ConfigurationManager.AppSettings["Username"];
var hadoopUserPassword = ConfigurationManager.AppSettings["Password"];
var hbaseTableName =ConfigurationManager.AppSettings["TableName"];
// Create a new instance of an HBase client.
var creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword);
var hbaseClient = new HBaseClient(creds);
//Scan over rows in a table. Assume the table has integer keys and you want data between keys 25 and 35.
var scanSettings = new Scanner()
{
batch = 10
};
var scannerInfo = await hbaseClient.CreateScannerAsync(hbaseTableName, scanSettings);
CellSet next = null;
var res = new List<HBaseEntry>();
var rowNum = 0;
while ((next = hbaseClient.ScannerGetNext(scannerInfo)) != null)
{
res.AddRange(from row in next.rows
let key = Encoding.UTF8.GetString(row.key)
let value = Encoding.UTF8.GetString(row.values[0].data)
let parts = key.Split(',')
where parts.Length == 3
select new HBaseEntry()
{
Type = parts[0], Date = parts[1].Replace("0000000", "000"), RoomNumber = parts[2], Reading = value
});
}
var dateRangedList =
res.Where(a => DateTime.Parse(a.Date) >= DateTime.UtcNow.AddDays(-5)).Select(a => a).ToList();
var map = new Dictionary<String, Dictionary<String, Dictionary<String, String>>> ();
var tempList = dateRangedList.Where(a => a.Type == "Temperature").ToList();
var engyList = dateRangedList.Where(a => a.Type == "Energy").ToList();
var humList = dateRangedList.Where(a => a.Type == "Humidity").ToList();
var lghtList = dateRangedList.Where(a => a.Type == "Light").ToList();
map.Add("Temperature", GetDataAsMap(tempList));
map.Add("Energy", GetDataAsMap(engyList));
map.Add("Humidity", GetDataAsMap(humList));
map.Add("Light", GetDataAsMap(lghtList));
//get data
var ctx = GlobalHost.ConnectionManager.GetHubContext<DashHub>();
ctx.Clients.All.acceptData(JsonConvert.SerializeObject(map));
Thread.Sleep(15000);
}
});
Background.Wait();
return true;
}
/// <summary>
/// Initialize the HBase settings and connections
/// </summary>
public void InitializeHBase()
{
this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl");
}
this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName");
}
this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword");
}
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
if (String.IsNullOrWhiteSpace(this.HBaseTableName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName");
}
//Setup the credentials for HBase cluster
this.HBaseClusterCredentials =
new ClusterCredentials(
new Uri(this.HBaseClusterUrl),
this.HBaseClusterUserName,
this.HBaseClusterPassword);
this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials);
//Query HBase for existing tables
var tables = this.HBaseClusterClient.ListTables();
Context.Logger.Info("HBase Tables (" + tables.name.Count + "): " + String.Join(", ", tables.name));
//Create a HBase table if it not exists
if (!tables.name.Contains(this.HBaseTableName))
{
throw new Exception("Cannot find HBase table: " + this.HBaseTableName);
}
}
/// <summary>
/// Initialize the HBase settings and connections
/// </summary>
public void InitializeHBase()
{
this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl");
}
this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName");
}
this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"];
if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword");
}
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
if (String.IsNullOrWhiteSpace(this.HBaseTableName))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName");
}
this.HBaseTableColumnFamily = ConfigurationManager.AppSettings["HBaseTableColumnFamily"];
if (String.IsNullOrWhiteSpace(this.HBaseTableColumnFamily))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnFamily");
}
//TODO - DO NOT include the ROWKEY field in the column list as it is assumed to be at index 0 in tuple fields
//Rest of the tuple fields are mapped with the column names provided
var hbaseTableColumnNames = ConfigurationManager.AppSettings["HBaseTableColumnNames"];
if (String.IsNullOrWhiteSpace(hbaseTableColumnNames))
{
throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnNames");
}
//Setup the credentials for HBase cluster
this.HBaseClusterCredentials =
new ClusterCredentials(
new Uri(this.HBaseClusterUrl),
this.HBaseClusterUserName,
this.HBaseClusterPassword);
this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials);
//Query HBase for existing tables
var tables = this.HBaseClusterClient.ListTables();
Context.Logger.Info("Existing HBase tables - Count: {0}, Tables: {1}", tables.name.Count, String.Join(", ", tables.name));
//Read the HBaseTable Columns and convert them to byte[]
this.HBaseTableColumns = hbaseTableColumnNames.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries).
Select(c => c.Trim()).ToList();
//Create a HBase table if it not exists
if (!tables.name.Contains(this.HBaseTableName))
{
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema() { name = this.HBaseTableColumnFamily });
HBaseClusterClient.CreateTable(tableSchema);
Context.Logger.Info("Created HBase table: {0}", this.HBaseTableName);
}
else
{
Context.Logger.Info("Found an existing HBase table: {0}", this.HBaseTableName);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="HBaseClient"/> class.
/// </summary>
/// <param name="credentials">The credentials.</param>
public HBaseClient(ClusterCredentials credentials)
: this(credentials, new DefaultRetryPolicyFactory())
{
}