public void TestFullScan() { var client = new HBaseClient(_credentials); StoreTestData(client); // full range scan var scanSettings = new Scanner { batch = 10 }; ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings); CellSet next; var expectedSet = new HashSet <int>(Enumerable.Range(0, 100)); while ((next = client.ScannerGetNext(scannerInfo)) != null) { Assert.AreEqual(10, next.rows.Count); foreach (CellSet.Row row in next.rows) { int k = BitConverter.ToInt32(row.key, 0); expectedSet.Remove(k); } } Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet)); }
protected override void Context() { _credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"); var client = new HBaseClient(_credentials); // ensure tables from previous tests are cleaned up TableList tables = client.ListTables(); foreach (string name in tables.name) { if (name.StartsWith(TestTablePrefix, StringComparison.Ordinal)) { client.DeleteTable(name); } } // add a table specific to this test _testTableName = TestTablePrefix + _random.Next(10000); _testTableSchema = new TableSchema(); _testTableSchema.name = _testTableName; _testTableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(_testTableSchema); }
public void When_I_Scan_with_a_MultipleColumnPrefixFilter_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); // set this large enough so that we get all records back var prefixes = new List <byte[]> { Encoding.UTF8.GetBytes(ColumnNameA), Encoding.UTF8.GetBytes(ColumnNameB) }; var filter = new MultipleColumnPrefixFilter(prefixes); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void TestLoadBalancerRetriesExhausted() { int numServers = 4; int numFailures = 5; var client = new HBaseClient(numServers); int count = 0; Func <string, Task <int> > f = (endpoint) => { count++; if (count < numFailures) { throw new TimeoutException(count.ToString()); } return(EmitIntAsync(count)); }; try { var output = client.ExecuteAndGetWithVirtualNetworkLoadBalancing(f); } catch (TimeoutException ex) { Assert.AreEqual(ex.Message, numServers.ToString()); return; } Assert.Fail(); }
public void When_I_Scan_with_a_RandomRowFilter_I_get_the_expected_results() { var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); // set this large enough so that we get all records back var filter = new RandomRowFilter(2000.0F); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void TestStoreSingleCell() { const string testKey = "content"; const string testValue = "the force is strong in this column"; var client = new HBaseClient(_credentials); var set = new CellSet(); var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) }; set.rows.Add(row); var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); client.StoreCells(_testTableName, set); CellSet cells = client.GetCells(_testTableName, testKey); Assert.AreEqual(1, cells.rows.Count); Assert.AreEqual(1, cells.rows[0].values.Count); Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data)); }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results() { // grab a substring that is guaranteed to match at least one record. string ss = _allExpectedRecords.First().A.Substring(1, 2); //Debug.WriteLine("The substring value is: " + ss); List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var comparer = new SubstringComparator(ss); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(ColumnNameA), CompareFilter.CompareOp.Equal, comparer); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner); List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); }
public HBaseWriter() { var credentials = CreateFromFile(@"..\..\credentials.txt"); client = new HBaseClient(credentials); if (!client.ListTablesAsync().Result.name.Contains(TABLE_BY_WORDS_NAME)) { // Create the table var tableSchema = new TableSchema(); tableSchema.name = TABLE_BY_WORDS_NAME; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTableAsync(tableSchema).Wait(); Console.WriteLine("Table \"{0}\" created.", TABLE_BY_WORDS_NAME); } // Read current row count cell rowCount = GetRowCount(); // Load sentiment dictionary file LoadDictionary(); writerThread = new Thread(new ThreadStart(WriterThreadFunction)); writerThread.Start(); }
public void TestLoadBalancerRetriesExhausted() { int numServers = 4; int numFailures = 5; var client = new HBaseClient(numServers); int count = 0; Func<string, Task<int>> f = (endpoint) => { count++; if (count < numFailures) { throw new TimeoutException(count.ToString()); } return EmitIntAsync(count); }; try { var output = client.ExecuteAndGetWithVirtualNetworkLoadBalancing(f); } catch (TimeoutException ex) { Assert.AreEqual(ex.Message, numServers.ToString()); return; } Assert.Fail(); }
// This function connects to HBase, loads the sentiment dictionary, and starts the thread for writting. public HBaseWriter() { ClusterCredentials credentials = new ClusterCredentials(new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD); client = new HBaseClient(credentials); // create the HBase table if it doesn't exist if (!client.ListTables().name.Contains(HBASETABLENAME)) { TableSchema tableSchema = new TableSchema(); tableSchema.name = HBASETABLENAME; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); Console.WriteLine("Table \"{0}\" is created.", HBASETABLENAME); } // Load sentiment dictionary from a file LoadDictionary(); // Start a thread for writting to HBase writerThread = new Thread(new ThreadStart(WriterThreadFunction)); writerThread.Start(); }
public void TestLoadBalancerIgnoreBlackListedEndpoints() { int numServers = 10; int numBlackListedServers = 8; var client = new HBaseClient(numServers); var blackListedServers = BuildServersList(numBlackListedServers); int result = 0; int numFailures = 0; Func <string, Task <int> > f = (endpoint) => { var blacklistedEndpointFound = blackListedServers.Find(x => x.Equals(endpoint)); if (blacklistedEndpointFound != null) { numFailures++; throw new TimeoutException(); } result = 100; return(EmitIntAsync(result)); }; var resultInOutput = client.ExecuteAndGetWithVirtualNetworkLoadBalancing <int>(f); Assert.IsTrue(resultInOutput == 100); Assert.IsTrue(numFailures >= 0); }
public HBaseWriter() { //Get the Hadoop Cluster info and create connection this.ClusterName = ConfigurationManager.AppSettings["ClusterName"]; this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"]; string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"]; this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"]; SecureString pw = new SecureString(); for(int i = 0; i < HadoopUserPassword.Length; i++){ pw.InsertAt(i, HadoopUserPassword[i]); } Uri clusterUri = new Uri(this.ClusterName); ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw); this.client = new HBaseClient(creds); //create table and enable the hbase writer if (!client.ListTables().name.Contains(this.HBaseTableName)) { // Create the table var tableSchema = new TableSchema(); tableSchema.name = this.HBaseTableName; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName); } WriterThread = new Thread(new ThreadStart(WriterThreadFunction)); WriterThread.Start(); }
public void TestExecuteAndGetWithVirtualNetworkLoadBalancingOneArg() { int numServers = 4; int numFailures = 3; var client = new HBaseClient(numServers); var arg1Value = "arg1"; int count = 0; Func <string, string, Task <int> > f = (arg1, endpoint) => { Assert.AreEqual(arg1Value, arg1); count++; if (count < numFailures) { throw new TimeoutException(); } return(EmitIntAsync(count)); }; var output = client.ExecuteAndGetWithVirtualNetworkLoadBalancing <string, int>(f, arg1Value); Assert.AreEqual(count, numFailures); Assert.AreEqual(output, numFailures); }
public void TestExecuteWithVirtualNetworkLoadBalancingTwoArgs() { int numServers = 4; int numFailures = 3; var client = new HBaseClient(numServers); var arg1Value = "arg1"; var arg2Value = "arg2"; int count = 0; Func <string, string, string, Task> f = (arg1, arg2, endpoint) => { Assert.AreEqual(arg1Value, arg1); Assert.AreEqual(arg2Value, arg2); count++; if (count < numFailures) { throw new TimeoutException(); } return(NoOpTask()); }; client.ExecuteWithVirtualNetworkLoadBalancing(f, arg1Value, arg2Value); Assert.AreEqual(count, numFailures); }
protected override void Context() { if (!_arrangementCompleted) { // at present, no tables are modified so only arrange the tables once per test pass // and putting the arrangement into a static context. // (this knocked test runs down to ~30 seconds from ~5 minutes). _credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"); var client = new HBaseClient(_credentials); // ensure tables from previous tests are cleaned up TableList tables = client.ListTables(); foreach (string name in tables.name) { string pinnedName = name; if (name.StartsWith(TableNamePrefix, StringComparison.Ordinal)) { client.DeleteTable(pinnedName); } } AddTable(); PopulateTable(); _arrangementCompleted = true; } }
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results() { // B Column should not be returned, so set the value to null. var expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ColumnCountGetFilter(2); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
static void Main(string[] args) { Console.ReadLine(); try { var creds = new ClusterCredentials(new Uri("https://x.azurehdinsight.net"), "", ""); var client = new HBaseClient(creds); var version = client.GetVersionAsync().Result; Console.WriteLine(version); var tableschema = new TableSchema(); tableschema.name = "ReallyBigtable"; tableschema.columns.Add(new ColumnSchema() { name = "d" }); tableschema.columns.Add(new ColumnSchema() { name = "f" }); client.CreateTableAsync(tableschema).Wait(); } catch (Exception ex) { Console.WriteLine(ex.Message.ToString()); } Console.ReadLine(); }
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results() { // B column should not be returned, so set the value to null. List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new ColumnCountGetFilter(2); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results() { List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); Filter f0 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.Equal, BitConverter.GetBytes(1)); Filter f1 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.LessThanOrEqualTo, BitConverter.GetBytes(2)); var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner); List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); }
public void When_I_Scan_with_a_FamilyFilter_I_get_the_expected_results() { // B is in Column family 2 var expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new FamilyFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(ColumnFamilyName1))); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_RowFilter_I_get_the_expected_results() { var example = _allExpectedRecords.First(); var expectedRecords = (from r in _allExpectedRecords where r.RowKey == example.RowKey select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(example.RowKey))); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.NotEqual, BitConverter.GetBytes(1)); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
protected override void Context() { if (!_arrangementCompleted) { // at present, no tables are modified so only arrange the tables once per test pass // and putting the arrangement into a static context. // (this knocked test runs down to ~30 seconds from ~5 minutes). _credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"); var client = new HBaseClient(_credentials); // ensure tables from previous tests are cleaned up TableList tables = client.ListTables(); foreach (string name in tables.name) { string pinnedName = name; if (name.StartsWith(TableNamePrefix, StringComparison.Ordinal)) { client.DeleteTable(pinnedName); } } AddTable(); PopulateTable(); _arrangementCompleted = true; } }
public HBaseWriter() { //Get the Hadoop Cluster info and create connection this.ClusterName = ConfigurationManager.AppSettings["ClusterName"]; this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"]; string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"]; this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"]; SecureString pw = new SecureString(); for (int i = 0; i < HadoopUserPassword.Length; i++) { pw.InsertAt(i, HadoopUserPassword[i]); } Uri clusterUri = new Uri(this.ClusterName); ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw); this.client = new HBaseClient(creds); //create table and enable the hbase writer if (!client.ListTables().name.Contains(this.HBaseTableName)) { // Create the table var tableSchema = new TableSchema(); tableSchema.name = this.HBaseTableName; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName); } WriterThread = new Thread(new ThreadStart(WriterThreadFunction)); WriterThread.Start(); }
public void TestSubsetScan() { var client = new HBaseClient(_credentials); const int startRow = 15; const int endRow = 15 + 13; StoreTestData(client); // subset range scan var scanSettings = new Scanner { batch = 10, startRow = BitConverter.GetBytes(startRow), endRow = BitConverter.GetBytes(endRow) }; ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings); CellSet next; var expectedSet = new HashSet <int>(Enumerable.Range(startRow, endRow - startRow)); while ((next = client.ScannerGetNext(scannerInfo)) != null) { foreach (CellSet.Row row in next.rows) { int k = BitConverter.ToInt32(row.key, 0); expectedSet.Remove(k); } } Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet)); }
public void When_I_Scan_with_a_PageFilter_I_get_the_expected_results() { var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new PageFilter(2); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.Count.ShouldBeGreaterThanOrEqualTo(2); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_KeyOnlyFilter_I_get_the_expected_results() { // a key only filter does not return Column values var expectedRecords = (from r in _allExpectedRecords select new FilterTestRecord(r.RowKey, 0, string.Empty, string.Empty)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new KeyOnlyFilter(); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public FilterTests() { if (!_arrangementCompleted) { // at present, no tables are modified so only arrange the tables once per test pass // and putting the arrangement into a static context. // (this knocked test runs down to ~30 seconds from ~5 minutes). var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); // ensure tables from previous tests are cleaned up var tables = client.ListTablesAsync().Result; foreach (var Name in tables.Name) { var pinnedName = Name; if (Name.StartsWith(TableNamePrefix, StringComparison.Ordinal)) { client.DeleteTableAsync(pinnedName).Wait(); } } AddTable(); PopulateTable(); _arrangementCompleted = true; } }
public void When_I_Scan_with_a_PrefixFilter_I_get_the_expected_results() { FilterTestRecord example = _allExpectedRecords.First(); byte[] rawRowkey = Encoding.UTF8.GetBytes(example.RowKey); const int prefixLength = 4; var prefix = new byte[prefixLength]; Array.Copy(rawRowkey, prefix, prefixLength); List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords let rawKey = Encoding.UTF8.GetBytes(r.RowKey) where rawKey[0] == prefix[0] && rawKey[1] == prefix[1] && rawKey[2] == prefix[2] && rawKey[3] == prefix[3] select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new PrefixFilter(prefix); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner); List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); }
public void When_I_Scan_with_a_InclusiveStopFilter_I_get_the_expected_results() { var example = (from r in _allExpectedRecords where r.LineNumber == 2 select r).Single(); var rawRowKey = Encoding.UTF8.GetBytes(example.RowKey); var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new InclusiveStopFilter(rawRowKey); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ColumnRangeFilter_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0).WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ColumnRangeFilter(Encoding.UTF8.GetBytes(ColumnNameA), true, Encoding.UTF8.GetBytes(ColumnNameB), false); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ColumnPaginationFilter_I_get_the_expected_results() { // only grabbing the LineNumber Column with (1, 1) var expectedRecords = (from r in _allExpectedRecords select r.WithAValue(null).WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ColumnPaginationFilter(1, 1); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_WhileMatchFilter_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 0 select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new WhileMatchFilter(new ValueFilter(CompareFilter.CompareOp.NotEqual, new BinaryComparator(BitConverter.GetBytes(0)))); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ValueFilter_and_a_RegexStringComparator_I_get_the_expected_results() { var expectedRecords = _allExpectedRecords; var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ValueFilter(CompareFilter.CompareOp.Equal, new RegexStringComparator(".*")); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public HBaseReader() { var creds = new ClusterCredentials( new Uri(ConfigurationManager.AppSettings["Cluster"]), ConfigurationManager.AppSettings["User"], ConfigurationManager.AppSettings["Pwd"]); client = new HBaseClient(creds); }
// The constructor public HBaseReader() { ClusterCredentials creds = new ClusterCredentials( new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD); client = new HBaseClient(creds); }
public void When_I_Scan_all_I_get_the_expected_results() { var client = new HBaseClient(_credentials); var scan = new Scanner(); ScannerInformation scanInfo = client.CreateScanner(_tableName, scan); List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); }
public HBaseReaderClient(string hbaseclusterurl, string hbaseclusterusername, string hbaseclusteruserpassword) { this.HBaseClusterUrl = hbaseclusterurl; this.HBaseClusterUserName = hbaseclusterusername; this.HBaseClusterUserPassword = hbaseclusteruserpassword; ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true; this.HBaseClusterCredentials = new ClusterCredentials(new Uri(HBaseClusterUrl), HBaseClusterUserName, HBaseClusterUserPassword); this.HBaseClusterClient = new HBaseClient(HBaseClusterCredentials); }
/// <summary> /// /// </summary> /// <param name="context"></param> /// <param name="tablename"></param> public EventHBaseWriter(Context context, Dictionary<string, Object> parms = null) { this.context = context; this.appConfig = new AppConfig(); Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>(); inputSchema.Add(Constants.DEFAULT_STREAM_ID, AggregatedTupleFields); this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null)); //Setup the credentials for HBase cluster HBaseClusterCredentials = new ClusterCredentials( new Uri(this.appConfig.HBaseClusterUrl), this.appConfig.HBaseClusterUserName, this.appConfig.HBaseClusterUserPassword); HBaseClusterClient = new HBaseClient(HBaseClusterCredentials); //Query HBase for existing tables var tabs = HBaseClusterClient.ListTables(); Context.Logger.Info("HBase Tables (" + tabs.name.Count + "): " + String.Join(", ", tabs.name)); this.PrimaryKey = this.appConfig.PrimaryKey; this.SecondaryKey = this.appConfig.SecondaryKey; this.HBaseTableName = this.appConfig.HBaseTableNamePrefix + this.appConfig.PrimaryKey + this.appConfig.SecondaryKey + this.appConfig.HBaseTableNameSuffix; Context.Logger.Info("HBaseTableName = " + this.HBaseTableName); //Create a HBase table if it not exists if (!tabs.name.Contains(this.HBaseTableName)) { var tableSchema = new TableSchema(); tableSchema.name = this.HBaseTableName; tableSchema.columns.Add(new ColumnSchema() { name = "v" }); HBaseClusterClient.CreateTable(tableSchema); Context.Logger.Info("Created HBase Table: " + this.HBaseTableName); } Context.Logger.Info("HBaseOverwrite: " + this.appConfig.HBaseOverwrite); globalstopwatch = new Stopwatch(); globalstopwatch.Start(); emitstopwatch = new Stopwatch(); emitstopwatch.Start(); }
static void Main(string[] args) { while (true) { Random rnd = new Random(); Console.Clear(); string clusterURL = "https://hb12345.azurehdinsight.net"; string userName = "******"; string password = "******"; // Connect to HBase cluster ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password); HBaseClient hbaseClient = new HBaseClient(creds); // Get all stocks Scanner scanSettings = new Scanner() { batch = 10, startRow = Encoding.UTF8.GetBytes("AAA"), endRow = Encoding.UTF8.GetBytes("ZZZ") }; ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings); CellSet stockCells = null; while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null) { foreach (var row in stockCells.rows) { string stock = Encoding.UTF8.GetString(row.key); Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data)); Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1); Cell c = new Cell { column = Encoding.UTF8.GetBytes("Current:Price"), data = Encoding.UTF8.GetBytes(newPrice.ToString()) }; row.values.Insert(2, c); Console.WriteLine(stock + ": " + currentPrice.ToString() + " := " + newPrice.ToString()); } hbaseClient.StoreCells("Stocks", stockCells); } } }
public HBaseReader() { //Get the Hadoop Cluster info and create connection this.ClusterName = ConfigurationManager.AppSettings["ClusterName"]; this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"]; string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"]; SecureString pw = new SecureString(); for (int i = 0; i < HadoopUserPassword.Length; i++) { pw.InsertAt(i, HadoopUserPassword[i]); } Uri clusterUri = new Uri(this.ClusterName); ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw); this.client = new HBaseClient(creds); }
public HBaseStore() { // Initialize HBase connection var credentials = CreateFromFile(@"credentials.txt"); client = new HBaseClient(credentials); if (!client.ListTables().name.Contains(TABLE_BY_WORDS_NAME)) { // Create the table var tableSchema = new TableSchema(); tableSchema.name = TABLE_BY_WORDS_NAME; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); } }
/// <summary> /// Create a new HBase table /// </summary> /// <param name="table"></param> /// <returns></returns> public static bool CreateTable(string tableName, string[] columnFamilysKeys) { TableSchema tableSchema = new TableSchema(); tableSchema.name = tableName; foreach (var columnFamilySKey in columnFamilysKeys) { tableSchema.columns.Add(new ColumnSchema() { name = columnFamilySKey }); } if (hbaseClient == null) hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword); int millisecondsTimeout = 3000; return hbaseClient.CreateTableAsync(tableSchema).Wait(millisecondsTimeout); }
/// <summary> /// Create a new HBase table /// </summary> /// <param name="hbaseClient">client used to connect to HBase</param> public static void CreateHBaseTable(HBaseClient hbaseClient) { //Define the 'cf family //Set versions to retain to 5 ColumnSchema cf = new ColumnSchema() { name = Properties.Settings.Default.HBaseTableColumnFamily, maxVersions = 5 }; //Define the table TableSchema tableSchema = new TableSchema(); tableSchema.name = Properties.Settings.Default.HBaseTableName; tableSchema.columns.Add(cf); //Create the table hbaseClient.CreateTable(tableSchema); Console.WriteLine("Table created."); }
private HBaseClient GetClient() { _credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"); var options = RequestOptions.GetDefaultOptions(); options.RetryPolicy = RetryPolicy.NoRetry; var client = new HBaseClient(_credentials, options); #region VNet //options.TimeoutMillis = 30000; //options.KeepAlive = false; //options.Port = 8090; //options.AlternativeEndpoint = "/"; //var client = new HBaseClient(null, options, new LoadBalancerRoundRobin(new List<string> { "ip address" })); #endregion return client; }
private static void InitHBase() { try { clusterURL = System.Configuration.ConfigurationManager.AppSettings["HBaseClusterURL"].ToString(); httpName = System.Configuration.ConfigurationManager.AppSettings["HBaseHttpName"].ToString(); httpUserPassword = System.Configuration.ConfigurationManager.AppSettings["HBaseHttpUserPassword"].ToString(); hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword); } catch (Exception e) { Console.Error.WriteLine(e.Message + "\r\n" + e.StackTrace); Console.ReadKey(); } }
public Helper(HBaseClient client, string sampleTableName, string sampleRowKey) { _client = client; _sampleTableName = sampleTableName; _sampleRowKey = sampleRowKey; _serializer = new JsonSerializer { ReferenceLoopHandling = ReferenceLoopHandling.Ignore, ContractResolver = new DefaultContractResolver { DefaultMembersSearchFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance }, TypeNameAssemblyFormat = FormatterAssemblyStyle.Simple, ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor }; _columnNameMapping = new Dictionary<string, byte[]>(); }
static void Main(string[] args) { bool quit = false; while (!quit) { Console.ResetColor(); Console.WriteLine("Enter a stock code, or enter 'quit' to exit"); // Connect to HBase cluster string clusterURL = "https://hb12345.azurehdinsight.net"; string userName = "******"; string password = "******"; ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password); HBaseClient hbaseClient = new HBaseClient(creds); string input = Console.ReadLine(); if (input.ToLower() == "quit") { quit = true; } else { CellSet cellSet = hbaseClient.GetCells("Stocks", input); var row = cellSet.rows[0]; Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data)); Double closingPrice = Double.Parse(Encoding.UTF8.GetString(row.values[0].data)); if (currentPrice > closingPrice) { Console.ForegroundColor = ConsoleColor.Green; } else if (currentPrice < closingPrice) { Console.ForegroundColor = ConsoleColor.Red; } Console.WriteLine(input + ": " + currentPrice.ToString()); } } }
public void When_I_Scan_all_I_get_the_expected_results() { var client = new HBaseClient(_credentials); var scan = new Scanner(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scan, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void TestFullScan() { var client = new HBaseClient(_credentials); StoreTestData(client); // full range scan var scanSettings = new Scanner { batch = 10 }; ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings); CellSet next; var expectedSet = new HashSet<int>(Enumerable.Range(0, 100)); while ((next = client.ScannerGetNext(scannerInfo)) != null) { Assert.AreEqual(10, next.rows.Count); foreach (CellSet.Row row in next.rows) { int k = BitConverter.ToInt32(row.key, 0); expectedSet.Remove(k); } } Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet)); }
public HBaseWriter() { var credentials = CreateFromFile(@"..\..\credentials.txt"); client = new HBaseClient(credentials); if (!client.ListTables().name.Contains(TABLE_BY_WORDS_NAME)) { // Create the table var tableSchema = new TableSchema(); tableSchema.name = TABLE_BY_WORDS_NAME; tableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(tableSchema); Console.WriteLine("Table \"{0}\" created.", TABLE_BY_WORDS_NAME); } // Read current row count cell rowCount = GetRowCount(); // Load sentiment dictionary file LoadDictionary(); writerThread = new Thread(new ThreadStart(WriterThreadFunction)); writerThread.Start(); }
protected override void Context() { _credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"); var client = new HBaseClient(_credentials); // ensure tables from previous tests are cleaned up TableList tables = client.ListTables(); foreach (string name in tables.name) { if (name.StartsWith(TestTablePrefix, StringComparison.Ordinal)) { client.DeleteTable(name); } } // add a table specific to this test _testTableName = TestTablePrefix + _random.Next(10000); _testTableSchema = new TableSchema(); _testTableSchema.name = _testTableName; _testTableSchema.columns.Add(new ColumnSchema { name = "d" }); client.CreateTable(_testTableSchema); }
public void TestGetVersion() { var client = new HBaseClient(_credentials); org.apache.hadoop.hbase.rest.protobuf.generated.Version version = client.GetVersion(); Trace.WriteLine(version); version.jvmVersion.ShouldNotBeNullOrEmpty(); version.jerseyVersion.ShouldNotBeNullOrEmpty(); version.osVersion.ShouldNotBeNullOrEmpty(); version.restVersion.ShouldNotBeNullOrEmpty(); }
public void TestListTables() { var client = new HBaseClient(_credentials); TableList tables = client.ListTables(); List<string> testtables = tables.name.Where(item => item.StartsWith("marlintest", StringComparison.Ordinal)).ToList(); Assert.AreEqual(1, testtables.Count); Assert.AreEqual(_testTableName, testtables[0]); }
public void TestScannerCreation() { var client = new HBaseClient(_credentials); var batchSetting = new Scanner { batch = 2 }; ScannerInformation scannerInfo = client.CreateScanner(_testTableName, batchSetting); Assert.AreEqual(_testTableName, scannerInfo.TableName); Assert.IsTrue( scannerInfo.Location.Authority.StartsWith("headnode", StringComparison.Ordinal), "returned location didn't start with \"headnode\", it was: {0}", scannerInfo.Location); }
public void TestStoreSingleCell() { const string testKey = "content"; const string testValue = "the force is strong in this column"; var client = new HBaseClient(_credentials); var set = new CellSet(); var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) }; set.rows.Add(row); var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); client.StoreCells(_testTableName, set); CellSet cells = client.GetCells(_testTableName, testKey); Assert.AreEqual(1, cells.rows.Count); Assert.AreEqual(1, cells.rows[0].values.Count); Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data)); }
public void TestSubsetScan() { var client = new HBaseClient(_credentials); const int startRow = 15; const int endRow = 15 + 13; StoreTestData(client); // subset range scan var scanSettings = new Scanner { batch = 10, startRow = BitConverter.GetBytes(startRow), endRow = BitConverter.GetBytes(endRow) }; ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings); CellSet next; var expectedSet = new HashSet<int>(Enumerable.Range(startRow, endRow - startRow)); while ((next = client.ScannerGetNext(scannerInfo)) != null) { foreach (CellSet.Row row in next.rows) { int k = BitConverter.ToInt32(row.key, 0); expectedSet.Remove(k); } } Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet)); }
public void TestGetStorageClusterStatus() { var client = new HBaseClient(_credentials); StorageClusterStatus status = client.GetStorageClusterStatus(); // TODO not really a good test Assert.IsTrue(status.requests >= 0, "number of requests is negative"); Assert.IsTrue(status.liveNodes.Count >= 1, "number of live nodes is zero or negative"); Assert.IsTrue(status.liveNodes[0].requests >= 0, "number of requests to the first node is negative"); }
private void StoreTestData(HBaseClient hBaseClient) { // we are going to insert the keys 0 to 100 and then do some range queries on that const string testValue = "the force is strong in this column"; var set = new CellSet(); for (int i = 0; i < 100; i++) { var row = new CellSet.Row { key = BitConverter.GetBytes(i) }; var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); set.rows.Add(row); } hBaseClient.StoreCells(_testTableName, set); }
public void TestTableSchema() { var client = new HBaseClient(_credentials); TableSchema schema = client.GetTableSchema(_testTableName); Assert.AreEqual(_testTableName, schema.name); Assert.AreEqual(_testTableSchema.columns.Count, schema.columns.Count); Assert.AreEqual(_testTableSchema.columns[0].name, schema.columns[0].name); }