public HBaseWriter()
 {
     //Get the Hadoop Cluster info and create connection
     this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
     this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
     string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
     this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
     SecureString pw = new SecureString();
     for(int i = 0; i < HadoopUserPassword.Length; i++){
         pw.InsertAt(i, HadoopUserPassword[i]);
     }
     Uri clusterUri = new Uri(this.ClusterName);
     ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
     this.client = new HBaseClient(creds);
     //create table and enable the hbase writer
     if (!client.ListTables().name.Contains(this.HBaseTableName))
     {
         // Create the table
         var tableSchema = new TableSchema();
         tableSchema.name = this.HBaseTableName;
         tableSchema.columns.Add(new ColumnSchema { name = "d" });
         client.CreateTable(tableSchema);
         Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName);
     }
     WriterThread = new Thread(new ThreadStart(WriterThreadFunction));
     WriterThread.Start();
 }
 // The constructor
 public HBaseReader()
 {
     ClusterCredentials creds = new ClusterCredentials(
                     new Uri(CLUSTERNAME),
                     HADOOPUSERNAME,
                     HADOOPUSERPASSWORD);
     client = new HBaseClient(creds);
 }
 public HBaseReader()
 {
     var creds = new ClusterCredentials(
                     new Uri(ConfigurationManager.AppSettings["Cluster"]),
                     ConfigurationManager.AppSettings["User"],
                     ConfigurationManager.AppSettings["Pwd"]);
     client = new HBaseClient(creds);
 }
        /// <summary>
        /// Initializes a new instance of the <see cref="WebRequesterSecure"/> class.
        /// </summary>
        /// <param name="credentials">The credentials.</param>
        /// <param name="contentType">Type of the content.</param>
        public WebRequesterSecure(ClusterCredentials credentials, string contentType = "application/x-protobuf")
        {
            credentials.ArgumentNotNull("credentials");

            _credentials = credentials;
            _contentType = contentType;
            _credentialCache = new CredentialCache();
            InitCache();
        }
        public HBaseReaderClient(string hbaseclusterurl, string hbaseclusterusername, string hbaseclusteruserpassword)
        {
            this.HBaseClusterUrl = hbaseclusterurl;
            this.HBaseClusterUserName = hbaseclusterusername;
            this.HBaseClusterUserPassword = hbaseclusteruserpassword;

            ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true;

            this.HBaseClusterCredentials = new ClusterCredentials(new Uri(HBaseClusterUrl), HBaseClusterUserName, HBaseClusterUserPassword);
            this.HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);
        }
        /// <summary>
        /// 
        /// </summary>
        /// <param name="context"></param>
        /// <param name="tablename"></param>
        public EventHBaseWriter(Context context, Dictionary<string, Object> parms = null)
        {
            this.context = context;

            this.appConfig = new AppConfig();

            Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>();
            inputSchema.Add(Constants.DEFAULT_STREAM_ID, AggregatedTupleFields);

            this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null));

            //Setup the credentials for HBase cluster
            HBaseClusterCredentials = 
                new ClusterCredentials(
                    new Uri(this.appConfig.HBaseClusterUrl), 
                    this.appConfig.HBaseClusterUserName, 
                    this.appConfig.HBaseClusterUserPassword);

            HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);

            //Query HBase for existing tables
            var tabs = HBaseClusterClient.ListTables();
            Context.Logger.Info("HBase Tables (" + tabs.name.Count + "): " + String.Join(", ", tabs.name));

            this.PrimaryKey = this.appConfig.PrimaryKey;
            this.SecondaryKey = this.appConfig.SecondaryKey;

            this.HBaseTableName =
                this.appConfig.HBaseTableNamePrefix +
                this.appConfig.PrimaryKey + this.appConfig.SecondaryKey +
                this.appConfig.HBaseTableNameSuffix;

            Context.Logger.Info("HBaseTableName = " + this.HBaseTableName);

            //Create a HBase table if it not exists
            if (!tabs.name.Contains(this.HBaseTableName))
            {
                var tableSchema = new TableSchema();
                tableSchema.name = this.HBaseTableName;
                tableSchema.columns.Add(new ColumnSchema() { name = "v" });
                HBaseClusterClient.CreateTable(tableSchema);
                Context.Logger.Info("Created HBase Table: " + this.HBaseTableName);
            }

            Context.Logger.Info("HBaseOverwrite: " + this.appConfig.HBaseOverwrite);

            globalstopwatch = new Stopwatch();
            globalstopwatch.Start();

            emitstopwatch = new Stopwatch();
            emitstopwatch.Start();
        }
 public HBaseClient(ClusterCredentials credentials, RequestOptions globalRequestOptions = null, ILoadBalancer loadBalancer = null)
 {
     _globalRequestOptions = globalRequestOptions ?? RequestOptions.GetDefaultOptions();
     _globalRequestOptions.Validate();
     if (credentials != null) // gateway mode
     {
         _requester = new GatewayWebRequester(credentials);
     }
     else // vnet mode
     {
         _requester = new VNetWebRequester(loadBalancer);
     }
 }
        static void Main(string[] args)
        {

            while (true)
            {
                Random rnd = new Random();
                Console.Clear();

                string clusterURL = "https://hb12345.azurehdinsight.net";
                string userName = "******";
                string password = "******";

                // Connect to HBase cluster
                ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL),
                                                                  userName, password);
                HBaseClient hbaseClient = new HBaseClient(creds);

                // Get all stocks
                Scanner scanSettings = new Scanner()
                {
                    batch = 10,
                    startRow = Encoding.UTF8.GetBytes("AAA"),
                    endRow = Encoding.UTF8.GetBytes("ZZZ")
                };

                ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings);
                CellSet stockCells = null;
                while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null)
                {
                    foreach (var row in stockCells.rows)
                    {
                        string stock = Encoding.UTF8.GetString(row.key);
                        Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
                        Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1);
                        Cell c = new Cell
                        {
                            column = Encoding.UTF8.GetBytes("Current:Price"),
                            data =
                           Encoding.UTF8.GetBytes(newPrice.ToString())
                        };
                        row.values.Insert(2, c);
                        Console.WriteLine(stock + ": " + currentPrice.ToString() + " := "
                                                       + newPrice.ToString());
                    }
                    hbaseClient.StoreCells("Stocks", stockCells);
                }
            }

        }
 public HBaseReader()
 {
     //Get the Hadoop Cluster info and create connection
     this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
     this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
     string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
     SecureString pw = new SecureString();
     for (int i = 0; i < HadoopUserPassword.Length; i++)
     {
         pw.InsertAt(i, HadoopUserPassword[i]);
     }
     Uri clusterUri = new Uri(this.ClusterName);
     ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
     this.client = new HBaseClient(creds);
 }
        /// <summary>
        /// Initializes a new instance of the <see cref="HBaseClient"/> class.
        /// </summary>
        /// <param name="credentials">The credentials.</param>
        /// <param name="retryPolicyFactory">The retry policy factory.</param>
        public HBaseClient(ClusterCredentials credentials, IRetryPolicyFactory retryPolicyFactory, ILoadBalancer loadBalancer = null)
        {
            retryPolicyFactory.ArgumentNotNull("retryPolicyFactory");

            if (credentials != null)
            {
                _requester = new WebRequesterSecure(credentials);
            }
            else
            {
                _requester    = new WebRequesterBasic();
                _loadBalancer = loadBalancer;
            }

            _retryPolicyFactory = retryPolicyFactory;
        }
        static void Main(string[] args)
        {

            bool quit = false;
            while (!quit)
            {
                Console.ResetColor();
                Console.WriteLine("Enter a stock code, or enter 'quit' to exit");

                // Connect to HBase cluster
                string clusterURL = "https://hb12345.azurehdinsight.net";
                string userName = "******";
                string password = "******";

                ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password);
                HBaseClient hbaseClient = new HBaseClient(creds);

                string input = Console.ReadLine();
                if (input.ToLower() == "quit")
                {
                    quit = true;
                }
                else
                {
                    CellSet cellSet = hbaseClient.GetCells("Stocks", input);
                    var row = cellSet.rows[0];
                    Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
                    Double closingPrice = Double.Parse(Encoding.UTF8.GetString(row.values[0].data));
                    if (currentPrice > closingPrice)
                    {
                        Console.ForegroundColor = ConsoleColor.Green;
                    }
                    else
                        if (currentPrice < closingPrice)
                    {
                        Console.ForegroundColor = ConsoleColor.Red;
                    }
                    Console.WriteLine(input + ": " + currentPrice.ToString());
                }
            }

        }
        static void Main(string[] args)
        {
            //Connect to HBase
            var credentials = new ClusterCredentials(
                new Uri(Properties.Settings.Default.HBaseClusterUrl),
                Properties.Settings.Default.HBaseClusterUserName,
                Properties.Settings.Default.HBaseClusterPassword);
            //Get the client
            var hbaseClient = new HBaseClient(credentials);

            char operation = ' ';
            //Loop until 'q' is selected
            do {
                Console.Write("(C)reate table, (d)elete table, or search for (s)tart or (e)nd events. Or (q)uit: ");
                //Get the character
                operation = Console.ReadKey().KeyChar;
                Console.WriteLine();
                //Which operation?
                switch (operation)
                {
                    case 'c':
                        CreateHBaseTable(hbaseClient);
                        break;

                    case 'd':
                        DeleteHBaseTable(hbaseClient);
                        break;

                    case 's':
                        //Fall through since both Start and Event searches
                        //have the same logic/signature
                    case 'e':
                        DateTime start = GetDateTime("start");
                        DateTime end = GetDateTime("end");
                        //Was it start or end we want to look for?
                        string eventType = operation == 's' ? "START" : "END";
                        //Find sessions for the event type, by start and end times
                        GetSessionsByTime(hbaseClient, eventType, start, end);
                        break;

                    default:
                        break;
                }
            } while (operation != 'q');
        }
Exemple #13
0
 /// <summary>
 /// Create a new instance of an HBase client.
 /// </summary>
 /// <param name="clusterURL"></param>
 /// <param name="httpName"></param>
 /// <param name="httpUserPassword"></param>
 /// <returns></returns>
 private static HBaseClient CreateHBaseClient(string clusterURL, string httpName, string httpUserPassword)
 {
     ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), httpName, httpUserPassword);
     return new HBaseClient(creds);
 }
        // This function connects to HBase, loads the sentiment dictionary, and starts the thread for writting.
        public HBaseWriter()
        {
            ClusterCredentials credentials = new ClusterCredentials(new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD);
            client = new HBaseClient(credentials);

            // create the HBase table if it doesn't exist
            if (!client.ListTables().name.Contains(HBASETABLENAME))
            {
                TableSchema tableSchema = new TableSchema();
                tableSchema.name = HBASETABLENAME;
                tableSchema.columns.Add(new ColumnSchema { name = "d" });
                client.CreateTable(tableSchema);
                Console.WriteLine("Table \"{0}\" is created.", HBASETABLENAME);
            }

            // Load sentiment dictionary from a file
            LoadDictionary();

            // Start a thread for writting to HBase
            writerThread = new Thread(new ThreadStart(WriterThreadFunction));
            writerThread.Start();
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="HBaseClient"/> class.
 /// </summary>
 /// <param name="credentials">The credentials.</param>
 public HBaseClient(ClusterCredentials credentials)
     : this(credentials, RequestOptions.GetDefaultOptions())
 {
 }
        internal static ClusterCredentials CreateFromList(List<string> lines)
        {
            if (lines.Count() != 3)
            {
                throw new ArgumentException(
                    string.Format(
                        CultureInfo.InvariantCulture,
                        "Expected the credentials file to have exactly three lines, " +
                        "first containing the cluster URL, second the username, third the password. " + "Given {0} lines!",
                        lines.Count()),
                    "lines");
            }

            var rv = new ClusterCredentials(new Uri(lines[0]), lines[1], lines[2]);
            return rv;
        }
Exemple #17
0
        public bool StartListening()
        {
            Background = Task.Run(async () =>
            {
                while (true)
                {
     
                    var clusterURL = ConfigurationManager.AppSettings["ClusterUrl"];
                    var hadoopUsername = ConfigurationManager.AppSettings["Username"];
                    var hadoopUserPassword = ConfigurationManager.AppSettings["Password"];

                    var hbaseTableName =ConfigurationManager.AppSettings["TableName"];

                    // Create a new instance of an HBase client.
                    var creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword);
                    var hbaseClient = new HBaseClient(creds);

                    //Scan over rows in a table. Assume the table has integer keys and you want data between keys 25 and 35. 
                    var scanSettings = new Scanner()
                    {
                        batch = 10
                    };

                    var scannerInfo = await hbaseClient.CreateScannerAsync(hbaseTableName, scanSettings);
                    CellSet next = null;

                    var res = new List<HBaseEntry>();
                    var rowNum = 0;
                    while ((next = hbaseClient.ScannerGetNext(scannerInfo)) != null)
                    {
                        res.AddRange(from row in next.rows
                            let key = Encoding.UTF8.GetString(row.key)
                            let value = Encoding.UTF8.GetString(row.values[0].data)
                            let parts = key.Split(',')
                            where parts.Length == 3
                            select new HBaseEntry()
                            {
                                Type = parts[0], Date = parts[1].Replace("0000000", "000"), RoomNumber = parts[2], Reading = value
                            });

                    }

                    var dateRangedList =
                        res.Where(a => DateTime.Parse(a.Date) >= DateTime.UtcNow.AddDays(-5)).Select(a => a).ToList();

                    var map = new Dictionary<String, Dictionary<String, Dictionary<String, String>>> ();
                    var tempList = dateRangedList.Where(a => a.Type == "Temperature").ToList();
                    var engyList = dateRangedList.Where(a => a.Type == "Energy").ToList();
                    var humList = dateRangedList.Where(a => a.Type == "Humidity").ToList();
                    var lghtList = dateRangedList.Where(a => a.Type == "Light").ToList();

                    map.Add("Temperature", GetDataAsMap(tempList));
                    map.Add("Energy", GetDataAsMap(engyList));
                    map.Add("Humidity", GetDataAsMap(humList));
                    map.Add("Light", GetDataAsMap(lghtList));

                    //get data
                    var ctx = GlobalHost.ConnectionManager.GetHubContext<DashHub>();

                    ctx.Clients.All.acceptData(JsonConvert.SerializeObject(map));
                    Thread.Sleep(15000);
                }
            });
            Background.Wait();

            return true;
        }
        /// <summary>
        /// Initialize the HBase settings and connections
        /// </summary>
        public void InitializeHBase()
        {
            this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"];
            if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl");
            }

            this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"];
            if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName");
            }

            this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"];
            if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword");
            }

            this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
            if (String.IsNullOrWhiteSpace(this.HBaseTableName))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName");
            }

            //Setup the credentials for HBase cluster
            this.HBaseClusterCredentials =
                new ClusterCredentials(
                    new Uri(this.HBaseClusterUrl),
                    this.HBaseClusterUserName,
                    this.HBaseClusterPassword);

            this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials);

            //Query HBase for existing tables
            var tables = this.HBaseClusterClient.ListTables();
            Context.Logger.Info("HBase Tables (" + tables.name.Count + "): " + String.Join(", ", tables.name));


            //Create a HBase table if it not exists
            if (!tables.name.Contains(this.HBaseTableName))
            {
                throw new Exception("Cannot find HBase table: " + this.HBaseTableName);
            }
        }
        /// <summary>
        /// Initialize the HBase settings and connections
        /// </summary>
        public void InitializeHBase()
        {
            this.HBaseClusterUrl = ConfigurationManager.AppSettings["HBaseClusterUrl"];
            if (String.IsNullOrWhiteSpace(this.HBaseClusterUrl))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUrl");
            }

            this.HBaseClusterUserName = ConfigurationManager.AppSettings["HBaseClusterUserName"];
            if (String.IsNullOrWhiteSpace(this.HBaseClusterUserName))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterUserName");
            }

            this.HBaseClusterPassword = ConfigurationManager.AppSettings["HBaseClusterPassword"];
            if (String.IsNullOrWhiteSpace(this.HBaseClusterPassword))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseClusterPassword");
            }

            this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
            if (String.IsNullOrWhiteSpace(this.HBaseTableName))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableName");
            }

            this.HBaseTableColumnFamily = ConfigurationManager.AppSettings["HBaseTableColumnFamily"];
            if (String.IsNullOrWhiteSpace(this.HBaseTableColumnFamily))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnFamily");
            }

            //TODO - DO NOT include the ROWKEY field in the column list as it is assumed to be at index 0 in tuple fields
            //Rest of the tuple fields are mapped with the column names provided
            var hbaseTableColumnNames = ConfigurationManager.AppSettings["HBaseTableColumnNames"];
            if (String.IsNullOrWhiteSpace(hbaseTableColumnNames))
            {
                throw new ArgumentException("A required AppSetting cannot be null or empty", "HBaseTableColumnNames");
            }

            //Setup the credentials for HBase cluster
            this.HBaseClusterCredentials =
                new ClusterCredentials(
                    new Uri(this.HBaseClusterUrl),
                    this.HBaseClusterUserName,
                    this.HBaseClusterPassword);

            this.HBaseClusterClient = new HBaseClient(this.HBaseClusterCredentials);

            //Query HBase for existing tables
            var tables = this.HBaseClusterClient.ListTables();
            Context.Logger.Info("Existing HBase tables - Count: {0}, Tables: {1}", tables.name.Count, String.Join(", ", tables.name));

            //Read the HBaseTable Columns and convert them to byte[]
            this.HBaseTableColumns = hbaseTableColumnNames.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries).
                Select(c => c.Trim()).ToList();

            //Create a HBase table if it not exists
            if (!tables.name.Contains(this.HBaseTableName))
            {
                var tableSchema = new TableSchema();
                tableSchema.name = this.HBaseTableName;
                tableSchema.columns.Add(new ColumnSchema() { name = this.HBaseTableColumnFamily });
                HBaseClusterClient.CreateTable(tableSchema);
                Context.Logger.Info("Created HBase table: {0}", this.HBaseTableName);
            }
            else
            {
                Context.Logger.Info("Found an existing HBase table: {0}", this.HBaseTableName);
            }
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="HBaseClient"/> class.
 /// </summary>
 /// <param name="credentials">The credentials.</param>
 public HBaseClient(ClusterCredentials credentials)
     : this(credentials, new DefaultRetryPolicyFactory())
 {
 }