// Query Tweets sentiment data from the HBase table asynchronously
public async Task<IEnumerable<Tweet>> QueryTweetsByKeywordAsync(string keyword)
{
List<Tweet> list = new List<Tweet>();
// Demonstrate Filtering the data from the past 6 hours the row key
string timeIndex = (ulong.MaxValue -
(ulong)DateTime.UtcNow.Subtract(new TimeSpan(6, 0, 0)).ToBinary()).ToString().PadLeft(20);
string startRow = keyword + "_" + timeIndex;
string endRow = keyword + "|";
Scanner scanSettings = new Scanner
{
batch = 100000,
startRow = Encoding.UTF8.GetBytes(startRow),
endRow = Encoding.UTF8.GetBytes(endRow)
};
// Make async scan call
ScannerInformation scannerInfo =
await client.CreateScannerAsync(HBASETABLENAME, scanSettings);
CellSet next;
while ((next = await client.ScannerGetNextAsync(scannerInfo)) != null)
{
foreach (CellSet.Row row in next.rows)
{
// find the cell with string pattern "d:coor"
var coordinates =
row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:coor");
if (coordinates != null)
{
string[] lonlat = Encoding.UTF8.GetString(coordinates.data).Split(',');
var sentimentField =
row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:sentiment");
Int32 sentiment = 0;
if (sentimentField != null)
{
sentiment = Convert.ToInt32(Encoding.UTF8.GetString(sentimentField.data));
}
list.Add(new Tweet
{
Longtitude = Convert.ToDouble(lonlat[0]),
Latitude = Convert.ToDouble(lonlat[1]),
Sentiment = sentiment
});
}
if (coordinates != null)
{
string[] lonlat = Encoding.UTF8.GetString(coordinates.data).Split(',');
}
}
}
return list;
}
public void When_I_Scan_all_I_get_the_expected_results()
{
var client = new HBaseClient(_credentials);
var scan = new Scanner();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scan);
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(_allExpectedRecords);
}
public async Task<IEnumerable<Tweet>> QueryTweetsByKeywordAsync(string keyword)
{
var list = new List<Tweet>();
var time_index = (ulong.MaxValue -
(ulong)DateTime.UtcNow.Subtract(new TimeSpan(6, 0, 0)).ToBinary()).ToString().PadLeft(20);
var startRow = keyword + "_" + time_index;
var endRow = keyword + "|";
var scanSettings = new Scanner {
batch = 100000,
startRow = Encoding.UTF8.GetBytes(startRow),
endRow = Encoding.UTF8.GetBytes(endRow) };
ScannerInformation scannerInfo =
await client.CreateScannerAsync(tableByWordsName, scanSettings);
CellSet next;
while ((next = await client.ScannerGetNextAsync(scannerInfo)) != null)
{
foreach (CellSet.Row row in next.rows)
{
var coordinates =
row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:coor");
if (coordinates != null)
{
var lonlat = Encoding.UTF8.GetString(coordinates.data).Split(',');
var sentimentField =
row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:sentiment");
var sentiment = 0;
if (sentimentField != null)
{
sentiment = Convert.ToInt32(Encoding.UTF8.GetString(sentimentField.data));
}
list.Add(new Tweet {
Longtitude = Convert.ToDouble(lonlat[0]),
Latitude = Convert.ToDouble(lonlat[1]),
Sentiment = sentiment
});
}
if (coordinates != null)
{
var lonlat = Encoding.UTF8.GetString(coordinates.data).Split(',');
}
}
}
return list;
}
static void Main(string[] args)
{
while (true)
{
Random rnd = new Random();
Console.Clear();
string clusterURL = "https://hb12345.azurehdinsight.net";
string userName = "******";
string password = "******";
// Connect to HBase cluster
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL),
userName, password);
HBaseClient hbaseClient = new HBaseClient(creds);
// Get all stocks
Scanner scanSettings = new Scanner()
{
batch = 10,
startRow = Encoding.UTF8.GetBytes("AAA"),
endRow = Encoding.UTF8.GetBytes("ZZZ")
};
ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings);
CellSet stockCells = null;
while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null)
{
foreach (var row in stockCells.rows)
{
string stock = Encoding.UTF8.GetString(row.key);
Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1);
Cell c = new Cell
{
column = Encoding.UTF8.GetBytes("Current:Price"),
data =
Encoding.UTF8.GetBytes(newPrice.ToString())
};
row.values.Insert(2, c);
Console.WriteLine(stock + ": " + currentPrice.ToString() + " := "
+ newPrice.ToString());
}
hbaseClient.StoreCells("Stocks", stockCells);
}
}
}
public IEnumerable<FSTweet> ReadAllTweets()
{
Scanner s = new Scanner()
{
batch = 10
};
ScannerInformation si = client.CreateScanner(HadoopContext.TweetTableName, s);
CellSet next = null;
while((next = client.ScannerGetNext(si)) != null)
{
foreach(CellSet.Row row in next.rows)
{
}
}
return null;
}
public IEnumerable<FSWordRelationship> ReadAllWordRelationships()
{
Scanner s = new Scanner()
{
batch = 10
};
ScannerInformation si = client.CreateScanner(HadoopContext.WordRelationTableName, s);
CellSet next = null;
CellSet readRows = new CellSet();
while ((next = client.ScannerGetNext(si)) != null)
{
foreach (CellSet.Row row in next.rows)
{
//convert row into desired domain type....
readRows.rows.Add(row);
}
}
return null;
}
public void When_I_Scan_all_I_get_the_expected_results()
{
var client = new HBaseClient(_credentials);
var scan = new Scanner();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scan, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(_allExpectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void TestFullScan()
{
var client = new HBaseClient(_credentials);
StoreTestData(client);
// full range scan
var scanSettings = new Scanner { batch = 10 };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
CellSet next;
var expectedSet = new HashSet<int>(Enumerable.Range(0, 100));
while ((next = client.ScannerGetNext(scannerInfo)) != null)
{
Assert.AreEqual(10, next.rows.Count);
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
public override void TestFullScan()
{
var client = CreateClient();
StoreTestData(client);
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
// full range scan
var scanSettings = new Scanner { batch = 10 };
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
CellSet next;
var expectedSet = new HashSet<int>(Enumerable.Range(0, 100));
while ((next = client.ScannerGetNextAsync(scannerInfo, scanOptions).Result) != null)
{
Assert.AreEqual(10, next.rows.Count);
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
public async Task TestScannerDeletion()
{
var client = new HBaseClient(_credentials);
// full range scan
var scanSettings = new Scanner { batch = 10 };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
await client.DeleteScannerAsync(scannerInfo.TableName, scannerInfo.ScannerId);
}
public Dictionary<string, Dictionary<string, double>> ScanHBase(string hbasetablename, string startkey, string endkey)
{
var scannersettings = new Scanner()
{
startRow = Encoding.UTF8.GetBytes(startkey),
endRow = Encoding.UTF8.GetBytes(endkey),
};
var localstopwatch = new Stopwatch();
localstopwatch.Start();
var hbaseresultset = new Dictionary<string, Dictionary<string, double>>();
var scannerInfo = HBaseClusterClient.CreateScanner(hbasetablename, scannersettings);
CellSet readset = null;
while ((readset = HBaseClusterClient.ScannerGetNext(scannerInfo)) != null)
{
foreach (var row in readset.rows)
{
var rowkey = Encoding.UTF8.GetString(row.key);
if (hbaseresultset.ContainsKey(rowkey))
{
foreach (var column in row.values)
{
var columnkey = Encoding.UTF8.GetString(column.column);
var value = BitConverter.ToDouble(column.data, 0);
if (hbaseresultset[rowkey].ContainsKey(columnkey))
{
hbaseresultset[rowkey][columnkey] += value;
}
else
{
hbaseresultset[rowkey].Add(columnkey, value);
}
}
}
else
{
var newresult = new Dictionary<string, double>();
foreach (var column in row.values)
{
var columnkey = Encoding.UTF8.GetString(column.column);
var value = BitConverter.ToDouble(column.data, 0);
if (newresult.ContainsKey(columnkey))
{
newresult[columnkey] += value;
}
else
{
newresult.Add(columnkey, value);
}
}
hbaseresultset.Add(rowkey, newresult);
}
}
}
var overallresult = new Dictionary<string, double>();
LOG.InfoFormat("ScanHBase: {0} - Time Taken = {1} ms", hbasetablename, localstopwatch.ElapsedMilliseconds);
return hbaseresultset;
}
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results()
{
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
Filter f0 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.Equal,
BitConverter.GetBytes(1));
Filter f1 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.LessThanOrEqualTo,
BitConverter.GetBytes(2));
var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void TestScannerCreation()
{
var client = new HBaseClient(_credentials);
var batchSetting = new Scanner { batch = 2 };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, batchSetting);
Assert.AreEqual(_testTableName, scannerInfo.TableName);
Assert.IsTrue(
scannerInfo.Location.Authority.StartsWith("headnode", StringComparison.Ordinal),
"returned location didn't start with \"headnode\", it was: {0}",
scannerInfo.Location);
}
public void When_I_Scan_with_a_WhileMatchFilter_I_get_the_expected_results()
{
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 0 select r.WithBValue(null)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new WhileMatchFilter(new ValueFilter(CompareFilter.CompareOp.NotEqual, new BinaryComparator(BitConverter.GetBytes(0))));
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_PrefixFilter_I_get_the_expected_results()
{
FilterTestRecord example = _allExpectedRecords.First();
byte[] rawRowkey = Encoding.UTF8.GetBytes(example.RowKey);
const int prefixLength = 4;
var prefix = new byte[prefixLength];
Array.Copy(rawRowkey, prefix, prefixLength);
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords
let rawKey = Encoding.UTF8.GetBytes(r.RowKey)
where rawKey[0] == prefix[0] && rawKey[1] == prefix[1] && rawKey[2] == prefix[2] && rawKey[3] == prefix[3]
select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new PrefixFilter(prefix);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_PageFilter_I_get_the_expected_results()
{
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new PageFilter(2);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.Count.ShouldBeGreaterThanOrEqualTo(2);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_MultipleColumnPrefixFilter_I_get_the_expected_results()
{
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
// set this large enough so that we get all records back
var prefixes = new List<byte[]> { Encoding.UTF8.GetBytes(ColumnNameA), Encoding.UTF8.GetBytes(ColumnNameB) };
var filter = new MultipleColumnPrefixFilter(prefixes);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_KeyOnlyFilter_I_get_the_expected_results()
{
// a key only filter does not return column values
List<FilterTestRecord> expectedRecords =
(from r in _allExpectedRecords select new FilterTestRecord(r.RowKey, 0, string.Empty, string.Empty)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new KeyOnlyFilter();
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_InclusiveStopFilter_I_get_the_expected_results()
{
FilterTestRecord example = (from r in _allExpectedRecords where r.LineNumber == 2 select r).Single();
byte[] rawRowKey = Encoding.UTF8.GetBytes(example.RowKey);
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new InclusiveStopFilter(rawRowKey);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_RandomRowFilter_I_get_the_expected_results()
{
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
// set this large enough so that we get all records back
var filter = new RandomRowFilter(2000.0F);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(_allExpectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void TestScannerCreation()
{
var client = CreateClient();
var scanSettings = new Scanner { batch = 2 };
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
Assert.AreEqual(testTableName, scannerInfo.TableName);
Assert.IsNotNull(scannerInfo.ScannerId);
Assert.IsFalse(scannerInfo.ScannerId.StartsWith("/"), "scanner id starts with a slash");
Assert.IsNotNull(scannerInfo.ResponseHeaderCollection);
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_RowFilter_I_get_the_expected_results()
{
FilterTestRecord example = _allExpectedRecords.First();
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.RowKey == example.RowKey select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(example.RowKey)));
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_ValueFilter_and_a_RegexStringComparator_I_get_the_expected_results()
{
List<FilterTestRecord> expectedRecords = _allExpectedRecords;
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new ValueFilter(CompareFilter.CompareOp.Equal, new RegexStringComparator(".*"));
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_ColumnRangeFilter_I_get_the_expected_results()
{
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0).WithBValue(null)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new ColumnRangeFilter(Encoding.UTF8.GetBytes(ColumnNameA), true, Encoding.UTF8.GetBytes(ColumnNameB), false);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void TestSubsetScan()
{
var client = new HBaseClient(_credentials);
const int startRow = 15;
const int endRow = 15 + 13;
StoreTestData(client);
// subset range scan
var scanSettings = new Scanner { batch = 10, startRow = BitConverter.GetBytes(startRow), endRow = BitConverter.GetBytes(endRow) };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
CellSet next;
var expectedSet = new HashSet<int>(Enumerable.Range(startRow, endRow - startRow));
while ((next = client.ScannerGetNext(scannerInfo)) != null)
{
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results()
{
// B column should not be returned, so set the value to null.
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new ColumnCountGetFilter(2);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public IEnumerable<FSWordRelationship> FindRelationsWithWord(string word)
{
Scanner s = new Scanner()
{
batch = 1000
};
ScannerInformation si = client.CreateScanner(HadoopContext.HBaseWordRelationTableName, s);
CellSet next = null;
CellSet readRows = new CellSet();
List<FSWordRelationship> relations = new List<FSWordRelationship>();
while ((next = client.ScannerGetNext(si)) != null)
{
foreach (CellSet.Row row in next.rows)
{
//convert row into desired domain type....
var w1 = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordOne");
string wordOne = Encoding.UTF8.GetString(w1.data);
var w1Id = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordOneId");
int wordOneId = Convert.ToInt32(Encoding.UTF8.GetString(w1Id.data));
var w2 = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordTwo");
string wordTwo = Encoding.UTF8.GetString(w2.data);
var w2Id = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordTwoId");
int wordTwoId = Convert.ToInt32(Encoding.UTF8.GetString(w2Id.data));
var rS = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:RScore");
var rScore = Convert.ToDouble(Encoding.UTF8.GetString(rS.data));
var id = Encoding.UTF8.GetString(row.key);
if (word.CompareTo(wordOne) == 0 || word.CompareTo(wordTwo) == 0
&& !(String.IsNullOrWhiteSpace(wordOne) || String.IsNullOrWhiteSpace(wordTwo)))
{
FSWordRelationship rel = new FSWordRelationship(id, wordOne, wordOneId, wordTwo, wordTwoId, rScore);
relations.Add(rel);
}
}
}
return relations;
}
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_NullComparator_with_the_operator_not_equal_I_get_the_expected_results()
{
var expectedRecords = new List<FilterTestRecord>();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var comparer = new NullComparator();
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.Equal,
comparer);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_SingleColumnValueExcludeFilter_and_a_BinaryComparator_with_the_operator_equal_I_get_the_expected_results()
{
string bValue = (from r in _allExpectedRecords select r.B).First();
// B column should not be returned, so set the value to null.
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.B == bValue select r.WithBValue(null)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new SingleColumnValueExcludeFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName2),
Encoding.UTF8.GetBytes(ColumnNameB),
CompareFilter.CompareOp.Equal,
Encoding.UTF8.GetBytes(bValue));
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_DependentColumnFilter_and_a_BinaryComparator_with_the_operator_equal_I_get_the_expected_results()
{
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 1 select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new DependentColumnFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
false,
CompareFilter.CompareOp.Equal,
new BinaryComparator(BitConverter.GetBytes(1)));
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
