public HBaseWriter() { //Get the Hadoop Cluster info and create connection this.ClusterName = ConfigurationManager.AppSettings["ClusterName"]; this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"]; string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"]; this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"]; SecureString pw = new SecureString(); for(int i = 0; i < HadoopUserPassword.Length; i++){ pw.InsertAt(i, HadoopUserPassword[i]); } Uri clusterUri = new Uri(this.ClusterName); ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw); this.client = new HBaseClient(creds); //create table and enable the hbase writer if (!client.ListTables().name.Contains(this.HBaseTableName)) { // Create the table var tableSchema = new TableSchema(); tableSchema.name = this.HBaseTableName; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName); } WriterThread = new Thread(new ThreadStart(WriterThreadFunction)); WriterThread.Start(); }
// The constructor public HBaseReader() { ClusterCredentials creds = new ClusterCredentials( new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD); client = new HBaseClient(creds); }
public HBaseReader() { var creds = new ClusterCredentials( new Uri(ConfigurationManager.AppSettings["Cluster"]), ConfigurationManager.AppSettings["User"], ConfigurationManager.AppSettings["Pwd"]); client = new HBaseClient(creds); }
/// <summary> /// Initializes a new instance of the <see cref="WebRequesterSecure"/> class. /// </summary> /// <param name="credentials">The credentials.</param> /// <param name="contentType">Type of the content.</param> public WebRequesterSecure(ClusterCredentials credentials, string contentType = "application/x-protobuf") { credentials.ArgumentNotNull("credentials"); _credentials = credentials; _contentType = contentType; _credentialCache = new CredentialCache(); InitCache(); }
public HBaseReaderClient(string hbaseclusterurl, string hbaseclusterusername, string hbaseclusteruserpassword) { this.HBaseClusterUrl = hbaseclusterurl; this.HBaseClusterUserName = hbaseclusterusername; this.HBaseClusterUserPassword = hbaseclusteruserpassword; ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true; this.HBaseClusterCredentials = new ClusterCredentials(new Uri(HBaseClusterUrl), HBaseClusterUserName, HBaseClusterUserPassword); this.HBaseClusterClient = new HBaseClient(HBaseClusterCredentials); }
/// <summary> /// /// </summary> /// <param name="context"></param> /// <param name="tablename"></param> public EventHBaseWriter(Context context, Dictionary<string, Object> parms = null) { this.context = context; this.appConfig = new AppConfig(); Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>(); inputSchema.Add(Constants.DEFAULT_STREAM_ID, AggregatedTupleFields); this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null)); //Setup the credentials for HBase cluster HBaseClusterCredentials = new ClusterCredentials( new Uri(this.appConfig.HBaseClusterUrl), this.appConfig.HBaseClusterUserName, this.appConfig.HBaseClusterUserPassword); HBaseClusterClient = new HBaseClient(HBaseClusterCredentials); //Query HBase for existing tables var tabs = HBaseClusterClient.ListTables(); Context.Logger.Info("HBase Tables (" + tabs.name.Count + "): " + String.Join(", ", tabs.name)); this.PrimaryKey = this.appConfig.PrimaryKey; this.SecondaryKey = this.appConfig.SecondaryKey; this.HBaseTableName = this.appConfig.HBaseTableNamePrefix + this.appConfig.PrimaryKey + this.appConfig.SecondaryKey + this.appConfig.HBaseTableNameSuffix; Context.Logger.Info("HBaseTableName = " + this.HBaseTableName); //Create a HBase table if it not exists if (!tabs.name.Contains(this.HBaseTableName)) { var tableSchema = new TableSchema(); tableSchema.name = this.HBaseTableName; tableSchema.columns.Add(new ColumnSchema() { name = "v" }); HBaseClusterClient.CreateTable(tableSchema); Context.Logger.Info("Created HBase Table: " + this.HBaseTableName); } Context.Logger.Info("HBaseOverwrite: " + this.appConfig.HBaseOverwrite); globalstopwatch = new Stopwatch(); globalstopwatch.Start(); emitstopwatch = new Stopwatch(); emitstopwatch.Start(); }
public HBaseClient(ClusterCredentials credentials, RequestOptions globalRequestOptions = null, ILoadBalancer loadBalancer = null) { _globalRequestOptions = globalRequestOptions ?? RequestOptions.GetDefaultOptions(); _globalRequestOptions.Validate(); if (credentials != null) // gateway mode { _requester = new GatewayWebRequester(credentials); } else // vnet mode { _requester = new VNetWebRequester(loadBalancer); } }
static void Main(string[] args) { while (true) { Random rnd = new Random(); Console.Clear(); string clusterURL = "https://hb12345.azurehdinsight.net"; string userName = "******"; string password = "******"; // Connect to HBase cluster ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password); HBaseClient hbaseClient = new HBaseClient(creds); // Get all stocks Scanner scanSettings = new Scanner() { batch = 10, startRow = Encoding.UTF8.GetBytes("AAA"), endRow = Encoding.UTF8.GetBytes("ZZZ") }; ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings); CellSet stockCells = null; while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null) { foreach (var row in stockCells.rows) { string stock = Encoding.UTF8.GetString(row.key); Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data)); Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1); Cell c = new Cell { column = Encoding.UTF8.GetBytes("Current:Price"), data = Encoding.UTF8.GetBytes(newPrice.ToString()) }; row.values.Insert(2, c); Console.WriteLine(stock + ": " + currentPrice.ToString() + " := " + newPrice.ToString()); } hbaseClient.StoreCells("Stocks", stockCells); } } }
public HBaseReader() { //Get the Hadoop Cluster info and create connection this.ClusterName = ConfigurationManager.AppSettings["ClusterName"]; this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"]; string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"]; SecureString pw = new SecureString(); for (int i = 0; i < HadoopUserPassword.Length; i++) { pw.InsertAt(i, HadoopUserPassword[i]); } Uri clusterUri = new Uri(this.ClusterName); ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw); this.client = new HBaseClient(creds); }
/// <summary> /// Initializes a new instance of the <see cref="HBaseClient"/> class. /// </summary> /// <param name="credentials">The credentials.</param> /// <param name="retryPolicyFactory">The retry policy factory.</param> public HBaseClient(ClusterCredentials credentials, IRetryPolicyFactory retryPolicyFactory, ILoadBalancer loadBalancer = null) { retryPolicyFactory.ArgumentNotNull("retryPolicyFactory"); if (credentials != null) { _requester = new WebRequesterSecure(credentials); } else { _requester = new WebRequesterBasic(); _loadBalancer = loadBalancer; } _retryPolicyFactory = retryPolicyFactory; }
static void Main(string[] args) { bool quit = false; while (!quit) { Console.ResetColor(); Console.WriteLine("Enter a stock code, or enter 'quit' to exit"); // Connect to HBase cluster string clusterURL = "https://hb12345.azurehdinsight.net"; string userName = "******"; string password = "******"; ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password); HBaseClient hbaseClient = new HBaseClient(creds); string input = Console.ReadLine(); if (input.ToLower() == "quit") { quit = true; } else { CellSet cellSet = hbaseClient.GetCells("Stocks", input); var row = cellSet.rows[0]; Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data)); Double closingPrice = Double.Parse(Encoding.UTF8.GetString(row.values[0].data)); if (currentPrice > closingPrice) { Console.ForegroundColor = ConsoleColor.Green; } else if (currentPrice < closingPrice) { Console.ForegroundColor = ConsoleColor.Red; } Console.WriteLine(input + ": " + currentPrice.ToString()); } } }
static void Main(string[] args) { //Connect to HBase var credentials = new ClusterCredentials( new Uri(Properties.Settings.Default.HBaseClusterUrl), Properties.Settings.Default.HBaseClusterUserName, Properties.Settings.Default.HBaseClusterPassword); //Get the client var hbaseClient = new HBaseClient(credentials); char operation = ' '; //Loop until 'q' is selected do { Console.Write("(C)reate table, (d)elete table, or search for (s)tart or (e)nd events. Or (q)uit: "); //Get the character operation = Console.ReadKey().KeyChar; Console.WriteLine(); //Which operation? switch (operation) { case 'c': CreateHBaseTable(hbaseClient); break; case 'd': DeleteHBaseTable(hbaseClient); break; case 's': //Fall through since both Start and Event searches //have the same logic/signature case 'e': DateTime start = GetDateTime("start"); DateTime end = GetDateTime("end"); //Was it start or end we want to look for? string eventType = operation == 's' ? "START" : "END"; //Find sessions for the event type, by start and end times GetSessionsByTime(hbaseClient, eventType, start, end); break; default: break; } } while (operation != 'q'); }
/// <summary> /// Create a new instance of an HBase client. /// </summary> /// <param name="clusterURL"></param> /// <param name="httpName"></param> /// <param name="httpUserPassword"></param> /// <returns></returns> private static HBaseClient CreateHBaseClient(string clusterURL, string httpName, string httpUserPassword) { ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), httpName, httpUserPassword); return new HBaseClient(creds); }
// This function connects to HBase, loads the sentiment dictionary, and starts the thread for writting. public HBaseWriter() { ClusterCredentials credentials = new ClusterCredentials(new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD); client = new HBaseClient(credentials); // create the HBase table if it doesn't exist if (!client.ListTables().name.Contains(HBASETABLENAME)) { TableSchema tableSchema = new TableSchema(); tableSchema.name = HBASETABLENAME; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); Console.WriteLine("Table \"{0}\" is created.", HBASETABLENAME); } // Load sentiment dictionary from a file LoadDictionary(); // Start a thread for writting to HBase writerThread = new Thread(new ThreadStart(WriterThreadFunction)); writerThread.Start(); }
/// <summary> /// Initializes a new instance of the <see cref="HBaseClient"/> class. /// </summary> /// <param name="credentials">The credentials.</param> public HBaseClient(ClusterCredentials credentials) : this(credentials, RequestOptions.GetDefaultOptions()) { }
internal static ClusterCredentials CreateFromList(List<string> lines) { if (lines.Count() != 3) { throw new ArgumentException( string.Format( CultureInfo.InvariantCulture, "Expected the credentials file to have exactly three lines, " + "first containing the cluster URL, second the username, third the password. " + "Given {0} lines!", lines.Count()), "lines"); } var rv = new ClusterCredentials(new Uri(lines[0]), lines[1], lines[2]); return rv; }
public bool StartListening() { Background = Task.Run(async () => { while (true) { var clusterURL = ConfigurationManager.AppSettings["ClusterUrl"]; var hadoopUsername = ConfigurationManager.AppSettings["Username"]; var hadoopUserPassword = ConfigurationManager.AppSettings["Password"]; var hbaseTableName =ConfigurationManager.AppSettings["TableName"]; // Create a new instance of an HBase client. var creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword); var hbaseClient = new HBaseClient(creds); //Scan over rows in a table. Assume the table has integer keys and you want data between keys 25 and 35. var scanSettings = new Scanner() { batch = 10 }; var scannerInfo = await hbaseClient.CreateScannerAsync(hbaseTableName, scanSettings); CellSet next = null; var res = new List<HBaseEntry>(); var rowNum = 0; while ((next = hbaseClient.ScannerGetNext(scannerInfo)) != null) { res.AddRange(from row in next.rows let key = Encoding.UTF8.GetString(row.key) let value = Encoding.UTF8.GetString(row.values[0].data) let parts = key.Split(',') where parts.Length == 3 select new HBaseEntry() { Type = parts[0], Date = parts[1].Replace("0000000", "000"), RoomNumber = parts[2], Reading = value }); } var dateRangedList = res.Where(a => DateTime.Parse(a.Date) >= DateTime.UtcNow.AddDays(-5)).Select(a => a).ToList(); var map = new Dictionary<String, Dictionary<String, Dictionary<String, String>>> (); var tempList = dateRangedList.Where(a => a.Type == "Temperature").ToList(); var engyList = dateRangedList.Where(a => a.Type == "Energy").ToList(); var humList = dateRangedList.Where(a => a.Type == "Humidity").ToList(); var lghtList = dateRangedList.Where(a => a.Type == "Light").ToList(); map.Add("Temperature", GetDataAsMap(tempList)); map.Add("Energy", GetDataAsMap(engyList)); map.Add("Humidity", GetDataAsMap(humList)); map.Add("Light", GetDataAsMap(lghtList)); //get data var ctx = GlobalHost.ConnectionManager.GetHubContext<DashHub>(); ctx.Clients.All.acceptData(JsonConvert.SerializeObject(map)); Thread.Sleep(15000); } }); Background.Wait(); return true; }
/// <summary> /// Initialize the HBase settings and connections /// </summary> public void InitializeHBase() { this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"]; if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl"); } this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"]; if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName"); } this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"]; if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword"); } this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"]; if (String.IsNullOrWhiteSpace(this.HBaseTableName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName"); } //Setup the credentials for HBase cluster this.HBaseClusterCredentials = new ClusterCredentials( new Uri(this.HBaseClusterUrl), this.HBaseClusterUserName, this.HBaseClusterPassword); this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials); //Query HBase for existing tables var tables = this.HBaseClusterClient.ListTables(); Context.Logger.Info("HBase Tables (" + tables.name.Count + "): " + String.Join(", ", tables.name)); //Create a HBase table if it not exists if (!tables.name.Contains(this.HBaseTableName)) { throw new Exception("Cannot find HBase table: " + this.HBaseTableName); } }
/// <summary> /// Initialize the HBase settings and connections /// </summary> public void InitializeHBase() { this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"]; if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl"); } this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"]; if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName"); } this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"]; if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword"); } this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"]; if (String.IsNullOrWhiteSpace(this.HBaseTableName)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName"); } this.HBaseTableColumnFamily = ConfigurationManager.AppSettings["HBaseTableColumnFamily"]; if (String.IsNullOrWhiteSpace(this.HBaseTableColumnFamily)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnFamily"); } //TODO - DO NOT include the ROWKEY field in the column list as it is assumed to be at index 0 in tuple fields //Rest of the tuple fields are mapped with the column names provided var hbaseTableColumnNames = ConfigurationManager.AppSettings["HBaseTableColumnNames"]; if (String.IsNullOrWhiteSpace(hbaseTableColumnNames)) { throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnNames"); } //Setup the credentials for HBase cluster this.HBaseClusterCredentials = new ClusterCredentials( new Uri(this.HBaseClusterUrl), this.HBaseClusterUserName, this.HBaseClusterPassword); this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials); //Query HBase for existing tables var tables = this.HBaseClusterClient.ListTables(); Context.Logger.Info("Existing HBase tables - Count: {0}, Tables: {1}", tables.name.Count, String.Join(", ", tables.name)); //Read the HBaseTable Columns and convert them to byte[] this.HBaseTableColumns = hbaseTableColumnNames.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries). Select(c => c.Trim()).ToList(); //Create a HBase table if it not exists if (!tables.name.Contains(this.HBaseTableName)) { var tableSchema = new TableSchema(); tableSchema.name = this.HBaseTableName; tableSchema.columns.Add(new ColumnSchema() { name = this.HBaseTableColumnFamily }); HBaseClusterClient.CreateTable(tableSchema); Context.Logger.Info("Created HBase table: {0}", this.HBaseTableName); } else { Context.Logger.Info("Found an existing HBase table: {0}", this.HBaseTableName); } }
/// <summary> /// Initializes a new instance of the <see cref="HBaseClient"/> class. /// </summary> /// <param name="credentials">The credentials.</param> public HBaseClient(ClusterCredentials credentials) : this(credentials, new DefaultRetryPolicyFactory()) { }