public void When_I_Scan_with_a_TimestampsFilter_I_get_the_expected_results() { var expectedRecords = _allExpectedRecords; // scan all and retrieve timestamps var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); ScannerInformation scanAll = null; List <long> timestamps = null; try { scanAll = client.CreateScannerAsync(_tableName, scanner).Result; timestamps = RetrieveTimestamps(scanAll).ToList(); } finally { if (scanAll != null) { client.DeleteScannerAsync(_tableName, scanAll).Wait(); } } Assert.NotNull(timestamps); // timestamps scan scanner = new Scanner(); var filter = new TimestampsFilter(timestamps); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_WhileMatchFilter_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 0 select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new WhileMatchFilter(new ValueFilter(CompareFilter.CompareOp.NotEqual, new BinaryComparator(BitConverter.GetBytes(0)))); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ValueFilter_and_a_RegexStringComparator_I_get_the_expected_results() { var expectedRecords = _allExpectedRecords; var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ValueFilter(CompareFilter.CompareOp.Equal, new RegexStringComparator(".*")); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ColumnRangeFilter_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0).WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ColumnRangeFilter(Encoding.UTF8.GetBytes(ColumnNameA), true, Encoding.UTF8.GetBytes(ColumnNameB), false); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ColumnPaginationFilter_I_get_the_expected_results() { // only grabbing the LineNumber Column with (1, 1) var expectedRecords = (from r in _allExpectedRecords select r.WithAValue(null).WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ColumnPaginationFilter(1, 1); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.NotEqual, BitConverter.GetBytes(1)); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_FamilyFilter_I_get_the_expected_results() { // B is in Column family 2 var expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new FamilyFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(ColumnFamilyName1))); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_KeyOnlyFilter_I_get_the_expected_results() { // a key only filter does not return Column values var expectedRecords = (from r in _allExpectedRecords select new FilterTestRecord(r.RowKey, 0, string.Empty, string.Empty)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new KeyOnlyFilter(); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_InclusiveStopFilter_I_get_the_expected_results() { var example = (from r in _allExpectedRecords where r.LineNumber == 2 select r).Single(); var rawRowKey = Encoding.UTF8.GetBytes(example.RowKey); var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new InclusiveStopFilter(rawRowKey); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_PageFilter_I_get_the_expected_results() { var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new PageFilter(2); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.Count.ShouldBeGreaterThanOrEqualTo(2); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_MultipleColumnPrefixFilter_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); // set this large enough so that we get all records back var prefixes = new List <byte[]> { Encoding.UTF8.GetBytes(ColumnNameA), Encoding.UTF8.GetBytes(ColumnNameB) }; var filter = new MultipleColumnPrefixFilter(prefixes); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_RandomRowFilter_I_get_the_expected_results() { var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); // set this large enough so that we get all records back var filter = new RandomRowFilter(2000.0F); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_RowFilter_I_get_the_expected_results() { var example = _allExpectedRecords.First(); var expectedRecords = (from r in _allExpectedRecords where r.RowKey == example.RowKey select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(example.RowKey))); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results() { // B Column should not be returned, so set the value to null. var expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new ColumnCountGetFilter(2); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
// Query Tweets sentiment data from the HBase table asynchronously public async Task <IEnumerable <Tweet> > QueryTweetsByKeywordAsync(string keyword) { List <Tweet> list = new List <Tweet>(); // Demonstrate Filtering the data from the past 6 hours the row key string timeIndex = (ulong.MaxValue - (ulong)DateTime.UtcNow.Subtract(new TimeSpan(6, 0, 0)).ToBinary()).ToString().PadLeft(20); string startRow = keyword + "_" + timeIndex; string endRow = keyword + "|"; Scanner scanSettings = new Scanner { batch = 100000, startRow = Encoding.UTF8.GetBytes(startRow), endRow = Encoding.UTF8.GetBytes(endRow) }; // Make async scan call ScannerInformation scannerInfo = await client.CreateScannerAsync(data_connnection.HBASETABLENAME, scanSettings); CellSet next; while ((next = await client.ScannerGetNextAsync(scannerInfo)) != null) { foreach (CellSet.Row row in next.rows) { // find the cell with string pattern "tweets:coor" var coordinates = row.values.Find(c => Encoding.UTF8.GetString(c.column) == "tweets:coor"); if (coordinates != null) { string[] lonlat = Encoding.UTF8.GetString(coordinates.data).Split(','); var sentimentField = row.values.Find(c => Encoding.UTF8.GetString(c.column) == "tweets:sentiment"); Int32 sentiment = 0; if (sentimentField != null) { sentiment = Convert.ToInt32(Encoding.UTF8.GetString(sentimentField.data)); } list.Add(new Tweet { Longtitude = Convert.ToDouble(lonlat[0]), Latitude = Convert.ToDouble(lonlat[1]), Sentiment = sentiment }); } if (coordinates != null) { string[] lonlat = Encoding.UTF8.GetString(coordinates.data).Split(','); } } } return(list); }
public async Task <IEnumerable <Tweet> > QueryTweetsByKeywordAsync(string keyword) { var list = new List <Tweet>(); var time_index = (ulong.MaxValue - (ulong)DateTime.UtcNow.Subtract(new TimeSpan(6, 0, 0)).ToBinary()).ToString().PadLeft(20); var startRow = keyword + "_" + time_index; var endRow = keyword + "|"; var scanSettings = new Scanner { batch = 100000, startRow = Encoding.UTF8.GetBytes(startRow), endRow = Encoding.UTF8.GetBytes(endRow) }; ScannerInformation scannerInfo = await client.CreateScannerAsync(tableByWordsName, scanSettings); CellSet next; while ((next = await client.ScannerGetNextAsync(scannerInfo)) != null) { foreach (CellSet.Row row in next.rows) { var coordinates = row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:coor"); if (coordinates != null) { var lonlat = Encoding.UTF8.GetString(coordinates.data).Split(','); var sentimentField = row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:sentiment"); var sentiment = 0; if (sentimentField != null) { sentiment = Convert.ToInt32(Encoding.UTF8.GetString(sentimentField.data)); } list.Add(new Tweet { Longtitude = Convert.ToDouble(lonlat[0]), Latitude = Convert.ToDouble(lonlat[1]), Sentiment = sentiment }); } if (coordinates != null) { var lonlat = Encoding.UTF8.GetString(coordinates.data).Split(','); } } } return(list); }
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results() { var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); Filter f0 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.Equal, BitConverter.GetBytes(1)); Filter f1 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.LessThanOrEqualTo, BitConverter.GetBytes(2)); var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_all_I_get_the_expected_results() { var client = new HBaseClient(_credentials); var scan = new Scanner(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scan, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_all_I_get_the_expected_results() { var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scan = new Scanner(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scan).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results() { // grab a substring that is guaranteed to match at least one record. var ss = _allExpectedRecords.First().A.Substring(1, 2); //Debug.WriteLine("The substring value is: " + ss); var expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var comparer = new SubstringComparator(ss); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(ColumnNameA), CompareFilter.CompareOp.Equal, comparer); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_PrefixFilter_I_get_the_expected_results() { var example = _allExpectedRecords.First(); var rawRowKey = Encoding.UTF8.GetBytes(example.RowKey); const int prefixLength = 4; var prefix = new byte[prefixLength]; Array.Copy(rawRowKey, prefix, prefixLength); var expectedRecords = (from r in _allExpectedRecords let rawKey = ByteString.CopyFromUtf8(r.RowKey) where rawKey[0] == prefix[0] && rawKey[1] == prefix[1] && rawKey[2] == prefix[2] && rawKey[3] == prefix[3] select r).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new PrefixFilter(prefix); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_SingleColumnValueExcludeFilter_and_a_BinaryComparator_with_the_operator_equal_I_get_the_expected_results() { var bValue = (from r in _allExpectedRecords select r.B).First(); // B Column should not be returned, so set the value to null. var expectedRecords = (from r in _allExpectedRecords where r.B == bValue select r.WithBValue(null)).ToList(); var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions()); var scanner = new Scanner(); var filter = new SingleColumnValueExcludeFilter( Encoding.UTF8.GetBytes(ColumnFamilyName2), Encoding.UTF8.GetBytes(ColumnNameB), CompareFilter.CompareOp.Equal, Encoding.UTF8.GetBytes(bValue)); scanner.Filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner).Result; var actualRecords = RetrieveResults(scanInfo).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo).Wait(); } } }
public void When_I_Scan_with_a_TimestampsFilter_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = _allExpectedRecords; // scan all and retrieve timestamps var client = new HBaseClient(_credentials); var scanner = new Scanner(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanAll = null; List<long> timestamps = null; try { scanAll = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; timestamps = RetrieveTimestamps(scanAll, scanOptions).ToList(); } finally { if (scanAll != null) { client.DeleteScannerAsync(_tableName, scanAll, scanOptions).Wait(); } } Assert.IsNotNull(timestamps); // timestamps scan scanner = new Scanner(); var filter = new TimestampsFilter(timestamps); scanner.filter = filter.ToEncodedString(); ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_ValueFilter_and_a_RegexStringComparator_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = _allExpectedRecords; var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new ValueFilter(CompareFilter.CompareOp.Equal, new RegexStringComparator(".*")); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_WhileMatchFilter_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 0 select r.WithBValue(null)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new WhileMatchFilter(new ValueFilter(CompareFilter.CompareOp.NotEqual, new BinaryComparator(BitConverter.GetBytes(0)))); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_ColumnRangeFilter_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0).WithBValue(null)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new ColumnRangeFilter(Encoding.UTF8.GetBytes(ColumnNameA), true, Encoding.UTF8.GetBytes(ColumnNameB), false); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_RandomRowFilter_I_get_the_expected_results() { var client = new HBaseClient(_credentials); var scanner = new Scanner(); // set this large enough so that we get all records back var filter = new RandomRowFilter(2000.0F); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(_allExpectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public bool StartListening() { Background = Task.Run(async () => { while (true) { var clusterURL = ConfigurationManager.AppSettings["ClusterUrl"]; var hadoopUsername = ConfigurationManager.AppSettings["Username"]; var hadoopUserPassword = ConfigurationManager.AppSettings["Password"]; var hbaseTableName =ConfigurationManager.AppSettings["TableName"]; // Create a new instance of an HBase client. var creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword); var hbaseClient = new HBaseClient(creds); //Scan over rows in a table. Assume the table has integer keys and you want data between keys 25 and 35. var scanSettings = new Scanner() { batch = 10 }; var scannerInfo = await hbaseClient.CreateScannerAsync(hbaseTableName, scanSettings); CellSet next = null; var res = new List<HBaseEntry>(); var rowNum = 0; while ((next = hbaseClient.ScannerGetNext(scannerInfo)) != null) { res.AddRange(from row in next.rows let key = Encoding.UTF8.GetString(row.key) let value = Encoding.UTF8.GetString(row.values[0].data) let parts = key.Split(',') where parts.Length == 3 select new HBaseEntry() { Type = parts[0], Date = parts[1].Replace("0000000", "000"), RoomNumber = parts[2], Reading = value }); } var dateRangedList = res.Where(a => DateTime.Parse(a.Date) >= DateTime.UtcNow.AddDays(-5)).Select(a => a).ToList(); var map = new Dictionary<String, Dictionary<String, Dictionary<String, String>>> (); var tempList = dateRangedList.Where(a => a.Type == "Temperature").ToList(); var engyList = dateRangedList.Where(a => a.Type == "Energy").ToList(); var humList = dateRangedList.Where(a => a.Type == "Humidity").ToList(); var lghtList = dateRangedList.Where(a => a.Type == "Light").ToList(); map.Add("Temperature", GetDataAsMap(tempList)); map.Add("Energy", GetDataAsMap(engyList)); map.Add("Humidity", GetDataAsMap(humList)); map.Add("Light", GetDataAsMap(lghtList)); //get data var ctx = GlobalHost.ConnectionManager.GetHubContext<DashHub>(); ctx.Clients.All.acceptData(JsonConvert.SerializeObject(map)); Thread.Sleep(15000); } }); Background.Wait(); return true; }
public void When_I_Scan_with_a_KeyOnlyFilter_I_get_the_expected_results() { // a key only filter does not return column values List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select new FilterTestRecord(r.RowKey, 0, string.Empty, string.Empty)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new KeyOnlyFilter(); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_MultipleColumnPrefixFilter_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); // set this large enough so that we get all records back var prefixes = new List<byte[]> { Encoding.UTF8.GetBytes(ColumnNameA), Encoding.UTF8.GetBytes(ColumnNameB) }; var filter = new MultipleColumnPrefixFilter(prefixes); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_PageFilter_I_get_the_expected_results() { var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new PageFilter(2); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.Count.ShouldBeGreaterThanOrEqualTo(2); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_PrefixFilter_I_get_the_expected_results() { FilterTestRecord example = _allExpectedRecords.First(); byte[] rawRowkey = Encoding.UTF8.GetBytes(example.RowKey); const int prefixLength = 4; var prefix = new byte[prefixLength]; Array.Copy(rawRowkey, prefix, prefixLength); List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords let rawKey = Encoding.UTF8.GetBytes(r.RowKey) where rawKey[0] == prefix[0] && rawKey[1] == prefix[1] && rawKey[2] == prefix[2] && rawKey[3] == prefix[3] select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new PrefixFilter(prefix); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_NullComparator_with_the_operator_not_equal_I_get_the_expected_results() { var expectedRecords = new List<FilterTestRecord>(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var comparer = new NullComparator(); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.Equal, comparer); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public bool StartListening() { Background = Task.Run(async() => { while (true) { var clusterURL = ConfigurationManager.AppSettings["ClusterUrl"]; var hadoopUsername = ConfigurationManager.AppSettings["Username"]; var hadoopUserPassword = ConfigurationManager.AppSettings["Password"]; var hbaseTableName = ConfigurationManager.AppSettings["TableName"]; // Create a new instance of an HBase client. var creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword); var hbaseClient = new HBaseClient(creds); //Scan over rows in a table. Assume the table has integer keys and you want data between keys 25 and 35. var scanSettings = new Scanner() { batch = 10 }; var scannerInfo = await hbaseClient.CreateScannerAsync(hbaseTableName, scanSettings); CellSet next = null; var res = new List <HBaseEntry>(); var rowNum = 0; while ((next = hbaseClient.ScannerGetNext(scannerInfo)) != null) { res.AddRange(from row in next.rows let key = Encoding.UTF8.GetString(row.key) let value = Encoding.UTF8.GetString(row.values[0].data) let parts = key.Split(',') where parts.Length == 3 select new HBaseEntry() { Type = parts[0], Date = parts[1].Replace("0000000", "000"), RoomNumber = parts[2], Reading = value }); } var dateRangedList = res.Where(a => DateTime.Parse(a.Date) >= DateTime.UtcNow.AddDays(-5)).Select(a => a).ToList(); var map = new Dictionary <String, Dictionary <String, Dictionary <String, String> > > (); var tempList = dateRangedList.Where(a => a.Type == "Temperature").ToList(); var engyList = dateRangedList.Where(a => a.Type == "Energy").ToList(); var humList = dateRangedList.Where(a => a.Type == "Humidity").ToList(); var lghtList = dateRangedList.Where(a => a.Type == "Light").ToList(); map.Add("Temperature", GetDataAsMap(tempList)); map.Add("Energy", GetDataAsMap(engyList)); map.Add("Humidity", GetDataAsMap(humList)); map.Add("Light", GetDataAsMap(lghtList)); //get data var ctx = GlobalHost.ConnectionManager.GetHubContext <DashHub>(); ctx.Clients.All.acceptData(JsonConvert.SerializeObject(map)); Thread.Sleep(15000); } }); Background.Wait(); return(true); }
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results() { // grab a substring that is guaranteed to match at least one record. string ss = _allExpectedRecords.First().A.Substring(1, 2); //Debug.WriteLine("The substring value is: " + ss); List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var comparer = new SubstringComparator(ss); var filter = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(ColumnNameA), CompareFilter.CompareOp.Equal, comparer); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_InclusiveStopFilter_I_get_the_expected_results() { FilterTestRecord example = (from r in _allExpectedRecords where r.LineNumber == 2 select r).Single(); byte[] rawRowKey = Encoding.UTF8.GetBytes(example.RowKey); List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new InclusiveStopFilter(rawRowKey); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); Filter f0 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.Equal, BitConverter.GetBytes(1)); Filter f1 = new SingleColumnValueFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), CompareFilter.CompareOp.LessThanOrEqualTo, BitConverter.GetBytes(2)); var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public async Task TestScannerDeletion() { var client = new HBaseClient(_credentials); // full range scan var scanSettings = new Scanner { batch = 10 }; ScannerInformation scannerInfo = await client.CreateScannerAsync(_testTableName, scanSettings, "hbaserest0/"); await client.DeleteScannerAsync(scannerInfo.TableName, scannerInfo.ScannerId, "hbaserest0/"); }
public void When_I_Scan_with_a_DependentColumnFilter_and_a_BinaryComparator_with_the_operator_equal_I_get_the_expected_results() { List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 1 select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new DependentColumnFilter( Encoding.UTF8.GetBytes(ColumnFamilyName1), Encoding.UTF8.GetBytes(LineNumberColumnName), false, CompareFilter.CompareOp.Equal, new BinaryComparator(BitConverter.GetBytes(1))); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_SingleColumnValueExcludeFilter_and_a_BinaryComparator_with_the_operator_equal_I_get_the_expected_results() { string bValue = (from r in _allExpectedRecords select r.B).First(); // B column should not be returned, so set the value to null. List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.B == bValue select r.WithBValue(null)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new SingleColumnValueExcludeFilter( Encoding.UTF8.GetBytes(ColumnFamilyName2), Encoding.UTF8.GetBytes(ColumnNameB), CompareFilter.CompareOp.Equal, Encoding.UTF8.GetBytes(bValue)); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
public void When_I_Scan_with_a_RowFilter_I_get_the_expected_results() { FilterTestRecord example = _allExpectedRecords.First(); List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.RowKey == example.RowKey select r).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(example.RowKey))); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }
static async Task MainAsync(string[] args) { try { string clusterURL = "https://myClusterName.azurehdinsight.net"; string hadoopUsername = "******"; string hadoopUserPassword = "******"; string hbaseTableName = "sampleHbaseTable"; // Create a new instance of an HBase client. ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword); HBaseClient hbaseClient = new HBaseClient(creds); // Retrieve the cluster version. org.apache.hadoop.hbase.rest.protobuf.generated.Version version = await hbaseClient.GetVersionAsync(); Console.WriteLine("The HBase cluster version is " + version); // Create a new HBase table. TableSchema testTableSchema = new TableSchema(); testTableSchema.name = hbaseTableName; testTableSchema.columns.Add(new ColumnSchema() { name = "d" }); testTableSchema.columns.Add(new ColumnSchema() { name = "f" }); hbaseClient.CreateTableAsync(testTableSchema).Wait(); // Insert data into the HBase table. string testKey = "content"; string testValue = "the force is strong in this column"; CellSet cellSet = new CellSet(); CellSet.Row cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) }; cellSet.rows.Add(cellSetRow); Cell value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; cellSetRow.values.Add(value); hbaseClient.StoreCellsAsync(hbaseTableName, cellSet).Wait(); // Retrieve a cell by its key. cellSet = await hbaseClient.GetCellsAsync(hbaseTableName, testKey); Console.WriteLine("The data with the key '" + testKey + "' is: " + Encoding.UTF8.GetString(cellSet.rows[0].values[0].data)); // with the previous insert, it should yield: "the force is strong in this column" //Scan over rows in a table. Assume the table has integer keys and you want data between keys 25 and 35. Scanner scanSettings = new Scanner() { batch = 10, startRow = BitConverter.GetBytes(25), endRow = BitConverter.GetBytes(35) }; ScannerInformation scannerInfo = await hbaseClient.CreateScannerAsync(hbaseTableName, scanSettings, null); CellSet next = null; Console.WriteLine("Scan results"); while ((next = await hbaseClient.ScannerGetNextAsync(scannerInfo, null)) != null) { foreach (CellSet.Row row in next.rows) { Console.WriteLine(row.key + " : " + Encoding.UTF8.GetString(row.values[0].data)); } } } catch (Exception ex) { Console.WriteLine(ex); } Console.WriteLine("Press ENTER to continue ..."); Console.ReadLine(); }
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results() { // B column should not be returned, so set the value to null. List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList(); var client = new HBaseClient(_credentials); var scanner = new Scanner(); var filter = new ColumnCountGetFilter(2); scanner.filter = filter.ToEncodedString(); RequestOptions scanOptions = RequestOptions.GetDefaultOptions(); scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero; ScannerInformation scanInfo = null; try { scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result; List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList(); actualRecords.ShouldContainOnly(expectedRecords); } finally { if (scanInfo != null) { client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait(); } } }