public void TestFullScan()
{
var client = new HBaseClient(_credentials);
StoreTestData(client);
// full range scan
var scanSettings = new Scanner {
batch = 10
};
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
CellSet next;
var expectedSet = new HashSet <int>(Enumerable.Range(0, 100));
while ((next = client.ScannerGetNext(scannerInfo)) != null)
{
Assert.AreEqual(10, next.rows.Count);
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
protected override void Context()
{
_credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt");
var client = new HBaseClient(_credentials);
// ensure tables from previous tests are cleaned up
TableList tables = client.ListTables();
foreach (string name in tables.name)
{
if (name.StartsWith(TestTablePrefix, StringComparison.Ordinal))
{
client.DeleteTable(name);
}
}
// add a table specific to this test
_testTableName = TestTablePrefix + _random.Next(10000);
_testTableSchema = new TableSchema();
_testTableSchema.name = _testTableName;
_testTableSchema.columns.Add(new ColumnSchema {
name = "d"
});
client.CreateTable(_testTableSchema);
}
public void When_I_Scan_with_a_MultipleColumnPrefixFilter_I_get_the_expected_results()
{
var expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
// set this large enough so that we get all records back
var prefixes = new List <byte[]> {
Encoding.UTF8.GetBytes(ColumnNameA), Encoding.UTF8.GetBytes(ColumnNameB)
};
var filter = new MultipleColumnPrefixFilter(prefixes);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void TestLoadBalancerRetriesExhausted()
{
int numServers = 4;
int numFailures = 5;
var client = new HBaseClient(numServers);
int count = 0;
Func <string, Task <int> > f = (endpoint) =>
{
count++;
if (count < numFailures)
{
throw new TimeoutException(count.ToString());
}
return(EmitIntAsync(count));
};
try
{
var output = client.ExecuteAndGetWithVirtualNetworkLoadBalancing(f);
}
catch (TimeoutException ex)
{
Assert.AreEqual(ex.Message, numServers.ToString());
return;
}
Assert.Fail();
}
public void When_I_Scan_with_a_RandomRowFilter_I_get_the_expected_results()
{
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
// set this large enough so that we get all records back
var filter = new RandomRowFilter(2000.0F);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(_allExpectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void TestStoreSingleCell()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = new HBaseClient(_credentials);
var set = new CellSet();
var row = new CellSet.Row {
key = Encoding.UTF8.GetBytes(testKey)
};
set.rows.Add(row);
var value = new Cell {
column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue)
};
row.values.Add(value);
client.StoreCells(_testTableName, set);
CellSet cells = client.GetCells(_testTableName, testKey);
Assert.AreEqual(1, cells.rows.Count);
Assert.AreEqual(1, cells.rows[0].values.Count);
Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data));
}
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_SubstringComparator_with_the_operator_equal_I_get_the_expected_results()
{
// grab a substring that is guaranteed to match at least one record.
string ss = _allExpectedRecords.First().A.Substring(1, 2);
//Debug.WriteLine("The substring value is: " + ss);
List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.A.Contains(ss) select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var comparer = new SubstringComparator(ss);
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(ColumnNameA),
CompareFilter.CompareOp.Equal,
comparer);
scanner.filter = filter.ToEncodedString();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner);
List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
public HBaseWriter()
{
var credentials = CreateFromFile(@"..\..\credentials.txt");
client = new HBaseClient(credentials);
if (!client.ListTablesAsync().Result.name.Contains(TABLE_BY_WORDS_NAME))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = TABLE_BY_WORDS_NAME;
tableSchema.columns.Add(new ColumnSchema {
name = "d"
});
client.CreateTableAsync(tableSchema).Wait();
Console.WriteLine("Table \"{0}\" created.", TABLE_BY_WORDS_NAME);
}
// Read current row count cell
rowCount = GetRowCount();
// Load sentiment dictionary file
LoadDictionary();
writerThread = new Thread(new ThreadStart(WriterThreadFunction));
writerThread.Start();
}
public void TestLoadBalancerRetriesExhausted()
{
int numServers = 4;
int numFailures = 5;
var client = new HBaseClient(numServers);
int count = 0;
Func<string, Task<int>> f = (endpoint) =>
{
count++;
if (count < numFailures)
{
throw new TimeoutException(count.ToString());
}
return EmitIntAsync(count);
};
try
{
var output = client.ExecuteAndGetWithVirtualNetworkLoadBalancing(f);
}
catch (TimeoutException ex)
{
Assert.AreEqual(ex.Message, numServers.ToString());
return;
}
Assert.Fail();
}
// This function connects to HBase, loads the sentiment dictionary, and starts the thread for writting.
public HBaseWriter()
{
ClusterCredentials credentials = new ClusterCredentials(new Uri(CLUSTERNAME), HADOOPUSERNAME, HADOOPUSERPASSWORD);
client = new HBaseClient(credentials);
// create the HBase table if it doesn't exist
if (!client.ListTables().name.Contains(HBASETABLENAME))
{
TableSchema tableSchema = new TableSchema();
tableSchema.name = HBASETABLENAME;
tableSchema.columns.Add(new ColumnSchema {
name = "d"
});
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" is created.", HBASETABLENAME);
}
// Load sentiment dictionary from a file
LoadDictionary();
// Start a thread for writting to HBase
writerThread = new Thread(new ThreadStart(WriterThreadFunction));
writerThread.Start();
}
public void TestLoadBalancerIgnoreBlackListedEndpoints()
{
int numServers = 10;
int numBlackListedServers = 8;
var client = new HBaseClient(numServers);
var blackListedServers = BuildServersList(numBlackListedServers);
int result = 0;
int numFailures = 0;
Func <string, Task <int> > f = (endpoint) =>
{
var blacklistedEndpointFound = blackListedServers.Find(x => x.Equals(endpoint));
if (blacklistedEndpointFound != null)
{
numFailures++;
throw new TimeoutException();
}
result = 100;
return(EmitIntAsync(result));
};
var resultInOutput = client.ExecuteAndGetWithVirtualNetworkLoadBalancing <int>(f);
Assert.IsTrue(resultInOutput == 100);
Assert.IsTrue(numFailures >= 0);
}
public HBaseWriter()
{
//Get the Hadoop Cluster info and create connection
this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
SecureString pw = new SecureString();
for(int i = 0; i < HadoopUserPassword.Length; i++){
pw.InsertAt(i, HadoopUserPassword[i]);
}
Uri clusterUri = new Uri(this.ClusterName);
ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
this.client = new HBaseClient(creds);
//create table and enable the hbase writer
if (!client.ListTables().name.Contains(this.HBaseTableName))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName);
}
WriterThread = new Thread(new ThreadStart(WriterThreadFunction));
WriterThread.Start();
}
public void TestExecuteAndGetWithVirtualNetworkLoadBalancingOneArg()
{
int numServers = 4;
int numFailures = 3;
var client = new HBaseClient(numServers);
var arg1Value = "arg1";
int count = 0;
Func <string, string, Task <int> > f = (arg1, endpoint) =>
{
Assert.AreEqual(arg1Value, arg1);
count++;
if (count < numFailures)
{
throw new TimeoutException();
}
return(EmitIntAsync(count));
};
var output = client.ExecuteAndGetWithVirtualNetworkLoadBalancing <string, int>(f, arg1Value);
Assert.AreEqual(count, numFailures);
Assert.AreEqual(output, numFailures);
}
public void TestExecuteWithVirtualNetworkLoadBalancingTwoArgs()
{
int numServers = 4;
int numFailures = 3;
var client = new HBaseClient(numServers);
var arg1Value = "arg1";
var arg2Value = "arg2";
int count = 0;
Func <string, string, string, Task> f = (arg1, arg2, endpoint) =>
{
Assert.AreEqual(arg1Value, arg1);
Assert.AreEqual(arg2Value, arg2);
count++;
if (count < numFailures)
{
throw new TimeoutException();
}
return(NoOpTask());
};
client.ExecuteWithVirtualNetworkLoadBalancing(f, arg1Value, arg2Value);
Assert.AreEqual(count, numFailures);
}
protected override void Context()
{
if (!_arrangementCompleted)
{
// at present, no tables are modified so only arrange the tables once per test pass
// and putting the arrangement into a static context.
// (this knocked test runs down to ~30 seconds from ~5 minutes).
_credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt");
var client = new HBaseClient(_credentials);
// ensure tables from previous tests are cleaned up
TableList tables = client.ListTables();
foreach (string name in tables.name)
{
string pinnedName = name;
if (name.StartsWith(TableNamePrefix, StringComparison.Ordinal))
{
client.DeleteTable(pinnedName);
}
}
AddTable();
PopulateTable();
_arrangementCompleted = true;
}
}
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results()
{
// B Column should not be returned, so set the value to null.
var expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new ColumnCountGetFilter(2);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
static void Main(string[] args)
{
Console.ReadLine();
try
{
var creds = new ClusterCredentials(new Uri("https://x.azurehdinsight.net"), "", "");
var client = new HBaseClient(creds);
var version = client.GetVersionAsync().Result;
Console.WriteLine(version);
var tableschema = new TableSchema();
tableschema.name = "ReallyBigtable";
tableschema.columns.Add(new ColumnSchema()
{
name = "d"
});
tableschema.columns.Add(new ColumnSchema()
{
name = "f"
});
client.CreateTableAsync(tableschema).Wait();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
Console.ReadLine();
}
public void When_I_Scan_with_a_ColumnCountGetFilter_I_get_the_expected_results()
{
// B column should not be returned, so set the value to null.
List<FilterTestRecord> expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new ColumnCountGetFilter(2);
scanner.filter = filter.ToEncodedString();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void When_I_Scan_with_a_FilterList_with_OR_logic_I_get_the_expected_results()
{
List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
Filter f0 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.Equal,
BitConverter.GetBytes(1));
Filter f1 = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.LessThanOrEqualTo,
BitConverter.GetBytes(2));
var filter = new FilterList(FilterList.Operator.MustPassOne, f0, f1);
scanner.filter = filter.ToEncodedString();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner);
List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
public void When_I_Scan_with_a_FamilyFilter_I_get_the_expected_results()
{
// B is in Column family 2
var expectedRecords = (from r in _allExpectedRecords select r.WithBValue(null)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new FamilyFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(ColumnFamilyName1)));
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_RowFilter_I_get_the_expected_results()
{
var example = _allExpectedRecords.First();
var expectedRecords = (from r in _allExpectedRecords where r.RowKey == example.RowKey select r).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new RowFilter(CompareFilter.CompareOp.Equal, new BinaryComparator(Encoding.UTF8.GetBytes(example.RowKey)));
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_SingleColumnValueFilter_and_a_BinaryComparator_with_the_operator_not_equal_I_get_the_expected_results()
{
var expectedRecords = (from r in _allExpectedRecords where r.LineNumber != 1 select r).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new SingleColumnValueFilter(
Encoding.UTF8.GetBytes(ColumnFamilyName1),
Encoding.UTF8.GetBytes(LineNumberColumnName),
CompareFilter.CompareOp.NotEqual,
BitConverter.GetBytes(1));
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
protected override void Context()
{
if (!_arrangementCompleted)
{
// at present, no tables are modified so only arrange the tables once per test pass
// and putting the arrangement into a static context.
// (this knocked test runs down to ~30 seconds from ~5 minutes).
_credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt");
var client = new HBaseClient(_credentials);
// ensure tables from previous tests are cleaned up
TableList tables = client.ListTables();
foreach (string name in tables.name)
{
string pinnedName = name;
if (name.StartsWith(TableNamePrefix, StringComparison.Ordinal))
{
client.DeleteTable(pinnedName);
}
}
AddTable();
PopulateTable();
_arrangementCompleted = true;
}
}
public HBaseWriter()
{
//Get the Hadoop Cluster info and create connection
this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
this.HBaseTableName = ConfigurationManager.AppSettings["HBaseTableName"];
SecureString pw = new SecureString();
for (int i = 0; i < HadoopUserPassword.Length; i++)
{
pw.InsertAt(i, HadoopUserPassword[i]);
}
Uri clusterUri = new Uri(this.ClusterName);
ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
this.client = new HBaseClient(creds);
//create table and enable the hbase writer
if (!client.ListTables().name.Contains(this.HBaseTableName))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema {
name = "d"
});
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" created.", this.HBaseTableName);
}
WriterThread = new Thread(new ThreadStart(WriterThreadFunction));
WriterThread.Start();
}
public void TestSubsetScan()
{
var client = new HBaseClient(_credentials);
const int startRow = 15;
const int endRow = 15 + 13;
StoreTestData(client);
// subset range scan
var scanSettings = new Scanner {
batch = 10, startRow = BitConverter.GetBytes(startRow), endRow = BitConverter.GetBytes(endRow)
};
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
CellSet next;
var expectedSet = new HashSet <int>(Enumerable.Range(startRow, endRow - startRow));
while ((next = client.ScannerGetNext(scannerInfo)) != null)
{
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
public void When_I_Scan_with_a_PageFilter_I_get_the_expected_results()
{
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new PageFilter(2);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.Count.ShouldBeGreaterThanOrEqualTo(2);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_KeyOnlyFilter_I_get_the_expected_results()
{
// a key only filter does not return Column values
var expectedRecords =
(from r in _allExpectedRecords select new FilterTestRecord(r.RowKey, 0, string.Empty, string.Empty)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new KeyOnlyFilter();
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public FilterTests()
{
if (!_arrangementCompleted)
{
// at present, no tables are modified so only arrange the tables once per test pass
// and putting the arrangement into a static context.
// (this knocked test runs down to ~30 seconds from ~5 minutes).
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
// ensure tables from previous tests are cleaned up
var tables = client.ListTablesAsync().Result;
foreach (var Name in tables.Name)
{
var pinnedName = Name;
if (Name.StartsWith(TableNamePrefix, StringComparison.Ordinal))
{
client.DeleteTableAsync(pinnedName).Wait();
}
}
AddTable();
PopulateTable();
_arrangementCompleted = true;
}
}
public void When_I_Scan_with_a_PrefixFilter_I_get_the_expected_results()
{
FilterTestRecord example = _allExpectedRecords.First();
byte[] rawRowkey = Encoding.UTF8.GetBytes(example.RowKey);
const int prefixLength = 4;
var prefix = new byte[prefixLength];
Array.Copy(rawRowkey, prefix, prefixLength);
List <FilterTestRecord> expectedRecords = (from r in _allExpectedRecords
let rawKey = Encoding.UTF8.GetBytes(r.RowKey)
where rawKey[0] == prefix[0] && rawKey[1] == prefix[1] && rawKey[2] == prefix[2] && rawKey[3] == prefix[3]
select r).ToList();
var client = new HBaseClient(_credentials);
var scanner = new Scanner();
var filter = new PrefixFilter(prefix);
scanner.filter = filter.ToEncodedString();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scanner);
List <FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
public void When_I_Scan_with_a_InclusiveStopFilter_I_get_the_expected_results()
{
var example = (from r in _allExpectedRecords where r.LineNumber == 2 select r).Single();
var rawRowKey = Encoding.UTF8.GetBytes(example.RowKey);
var expectedRecords = (from r in _allExpectedRecords where r.LineNumber <= 2 select r).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new InclusiveStopFilter(rawRowKey);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_ColumnRangeFilter_I_get_the_expected_results()
{
var expectedRecords = (from r in _allExpectedRecords select r.WithLineNumberValue(0).WithBValue(null)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new ColumnRangeFilter(Encoding.UTF8.GetBytes(ColumnNameA), true, Encoding.UTF8.GetBytes(ColumnNameB), false);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_ColumnPaginationFilter_I_get_the_expected_results()
{
// only grabbing the LineNumber Column with (1, 1)
var expectedRecords = (from r in _allExpectedRecords select r.WithAValue(null).WithBValue(null)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new ColumnPaginationFilter(1, 1);
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_WhileMatchFilter_I_get_the_expected_results()
{
var expectedRecords = (from r in _allExpectedRecords where r.LineNumber == 0 select r.WithBValue(null)).ToList();
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new WhileMatchFilter(new ValueFilter(CompareFilter.CompareOp.NotEqual, new BinaryComparator(BitConverter.GetBytes(0))));
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public void When_I_Scan_with_a_ValueFilter_and_a_RegexStringComparator_I_get_the_expected_results()
{
var expectedRecords = _allExpectedRecords;
var client = new HBaseClient(RequestOptionsFactory.GetDefaultOptions());
var scanner = new Scanner();
var filter = new ValueFilter(CompareFilter.CompareOp.Equal, new RegexStringComparator(".*"));
scanner.Filter = filter.ToEncodedString();
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scanner).Result;
var actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(expectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo).Wait();
}
}
}
public HBaseReader()
{
var creds = new ClusterCredentials(
new Uri(ConfigurationManager.AppSettings["Cluster"]),
ConfigurationManager.AppSettings["User"],
ConfigurationManager.AppSettings["Pwd"]);
client = new HBaseClient(creds);
}
// The constructor
public HBaseReader()
{
ClusterCredentials creds = new ClusterCredentials(
new Uri(CLUSTERNAME),
HADOOPUSERNAME,
HADOOPUSERPASSWORD);
client = new HBaseClient(creds);
}
public void When_I_Scan_all_I_get_the_expected_results()
{
var client = new HBaseClient(_credentials);
var scan = new Scanner();
ScannerInformation scanInfo = client.CreateScanner(_tableName, scan);
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo).ToList();
actualRecords.ShouldContainOnly(_allExpectedRecords);
}
public HBaseReaderClient(string hbaseclusterurl, string hbaseclusterusername, string hbaseclusteruserpassword)
{
this.HBaseClusterUrl = hbaseclusterurl;
this.HBaseClusterUserName = hbaseclusterusername;
this.HBaseClusterUserPassword = hbaseclusteruserpassword;
ServicePointManager.ServerCertificateValidationCallback += (sender, certificate, chain, errors) => true;
this.HBaseClusterCredentials = new ClusterCredentials(new Uri(HBaseClusterUrl), HBaseClusterUserName, HBaseClusterUserPassword);
this.HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);
}
/// <summary>
///
/// </summary>
/// <param name="context"></param>
/// <param name="tablename"></param>
public EventHBaseWriter(Context context, Dictionary<string, Object> parms = null)
{
this.context = context;
this.appConfig = new AppConfig();
Dictionary<string, List<Type>> inputSchema = new Dictionary<string, List<Type>>();
inputSchema.Add(Constants.DEFAULT_STREAM_ID, AggregatedTupleFields);
this.context.DeclareComponentSchema(new ComponentStreamSchema(inputSchema, null));
//Setup the credentials for HBase cluster
HBaseClusterCredentials =
new ClusterCredentials(
new Uri(this.appConfig.HBaseClusterUrl),
this.appConfig.HBaseClusterUserName,
this.appConfig.HBaseClusterUserPassword);
HBaseClusterClient = new HBaseClient(HBaseClusterCredentials);
//Query HBase for existing tables
var tabs = HBaseClusterClient.ListTables();
Context.Logger.Info("HBase Tables (" + tabs.name.Count + "): " + String.Join(", ", tabs.name));
this.PrimaryKey = this.appConfig.PrimaryKey;
this.SecondaryKey = this.appConfig.SecondaryKey;
this.HBaseTableName =
this.appConfig.HBaseTableNamePrefix +
this.appConfig.PrimaryKey + this.appConfig.SecondaryKey +
this.appConfig.HBaseTableNameSuffix;
Context.Logger.Info("HBaseTableName = " + this.HBaseTableName);
//Create a HBase table if it not exists
if (!tabs.name.Contains(this.HBaseTableName))
{
var tableSchema = new TableSchema();
tableSchema.name = this.HBaseTableName;
tableSchema.columns.Add(new ColumnSchema() { name = "v" });
HBaseClusterClient.CreateTable(tableSchema);
Context.Logger.Info("Created HBase Table: " + this.HBaseTableName);
}
Context.Logger.Info("HBaseOverwrite: " + this.appConfig.HBaseOverwrite);
globalstopwatch = new Stopwatch();
globalstopwatch.Start();
emitstopwatch = new Stopwatch();
emitstopwatch.Start();
}
static void Main(string[] args)
{
while (true)
{
Random rnd = new Random();
Console.Clear();
string clusterURL = "https://hb12345.azurehdinsight.net";
string userName = "******";
string password = "******";
// Connect to HBase cluster
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL),
userName, password);
HBaseClient hbaseClient = new HBaseClient(creds);
// Get all stocks
Scanner scanSettings = new Scanner()
{
batch = 10,
startRow = Encoding.UTF8.GetBytes("AAA"),
endRow = Encoding.UTF8.GetBytes("ZZZ")
};
ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings);
CellSet stockCells = null;
while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null)
{
foreach (var row in stockCells.rows)
{
string stock = Encoding.UTF8.GetString(row.key);
Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1);
Cell c = new Cell
{
column = Encoding.UTF8.GetBytes("Current:Price"),
data =
Encoding.UTF8.GetBytes(newPrice.ToString())
};
row.values.Insert(2, c);
Console.WriteLine(stock + ": " + currentPrice.ToString() + " := "
+ newPrice.ToString());
}
hbaseClient.StoreCells("Stocks", stockCells);
}
}
}
public HBaseReader()
{
//Get the Hadoop Cluster info and create connection
this.ClusterName = ConfigurationManager.AppSettings["ClusterName"];
this.HadoopUserName = ConfigurationManager.AppSettings["HadoopUserName"];
string HadoopUserPassword = ConfigurationManager.AppSettings["HadoopUserPassword"];
SecureString pw = new SecureString();
for (int i = 0; i < HadoopUserPassword.Length; i++)
{
pw.InsertAt(i, HadoopUserPassword[i]);
}
Uri clusterUri = new Uri(this.ClusterName);
ClusterCredentials creds = new ClusterCredentials(clusterUri, this.HadoopUserName, pw);
this.client = new HBaseClient(creds);
}
public HBaseStore()
{
// Initialize HBase connection
var credentials = CreateFromFile(@"credentials.txt");
client = new HBaseClient(credentials);
if (!client.ListTables().name.Contains(TABLE_BY_WORDS_NAME))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = TABLE_BY_WORDS_NAME;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
}
}
/// <summary>
/// Create a new HBase table
/// </summary>
/// <param name="table"></param>
/// <returns></returns>
public static bool CreateTable(string tableName, string[] columnFamilysKeys)
{
TableSchema tableSchema = new TableSchema();
tableSchema.name = tableName;
foreach (var columnFamilySKey in columnFamilysKeys)
{
tableSchema.columns.Add(new ColumnSchema() { name = columnFamilySKey });
}
if (hbaseClient == null)
hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword);
int millisecondsTimeout = 3000;
return hbaseClient.CreateTableAsync(tableSchema).Wait(millisecondsTimeout);
}
/// <summary>
/// Create a new HBase table
/// </summary>
/// <param name="hbaseClient">client used to connect to HBase</param>
public static void CreateHBaseTable(HBaseClient hbaseClient)
{
//Define the 'cf family
//Set versions to retain to 5
ColumnSchema cf = new ColumnSchema() {
name = Properties.Settings.Default.HBaseTableColumnFamily,
maxVersions = 5
};
//Define the table
TableSchema tableSchema = new TableSchema();
tableSchema.name = Properties.Settings.Default.HBaseTableName;
tableSchema.columns.Add(cf);
//Create the table
hbaseClient.CreateTable(tableSchema);
Console.WriteLine("Table created.");
}
private HBaseClient GetClient()
{
_credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt");
var options = RequestOptions.GetDefaultOptions();
options.RetryPolicy = RetryPolicy.NoRetry;
var client = new HBaseClient(_credentials, options);
#region VNet
//options.TimeoutMillis = 30000;
//options.KeepAlive = false;
//options.Port = 8090;
//options.AlternativeEndpoint = "/";
//var client = new HBaseClient(null, options, new LoadBalancerRoundRobin(new List<string> { "ip address" }));
#endregion
return client;
}
private static void InitHBase()
{
try
{
clusterURL = System.Configuration.ConfigurationManager.AppSettings["HBaseClusterURL"].ToString();
httpName = System.Configuration.ConfigurationManager.AppSettings["HBaseHttpName"].ToString();
httpUserPassword = System.Configuration.ConfigurationManager.AppSettings["HBaseHttpUserPassword"].ToString();
hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword);
}
catch (Exception e)
{
Console.Error.WriteLine(e.Message + "\r\n" + e.StackTrace);
Console.ReadKey();
}
}
public Helper(HBaseClient client, string sampleTableName, string sampleRowKey)
{
_client = client;
_sampleTableName = sampleTableName;
_sampleRowKey = sampleRowKey;
_serializer = new JsonSerializer
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
ContractResolver =
new DefaultContractResolver { DefaultMembersSearchFlags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance },
TypeNameAssemblyFormat = FormatterAssemblyStyle.Simple,
ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor
};
_columnNameMapping = new Dictionary<string, byte[]>();
}
static void Main(string[] args)
{
bool quit = false;
while (!quit)
{
Console.ResetColor();
Console.WriteLine("Enter a stock code, or enter 'quit' to exit");
// Connect to HBase cluster
string clusterURL = "https://hb12345.azurehdinsight.net";
string userName = "******";
string password = "******";
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), userName, password);
HBaseClient hbaseClient = new HBaseClient(creds);
string input = Console.ReadLine();
if (input.ToLower() == "quit")
{
quit = true;
}
else
{
CellSet cellSet = hbaseClient.GetCells("Stocks", input);
var row = cellSet.rows[0];
Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
Double closingPrice = Double.Parse(Encoding.UTF8.GetString(row.values[0].data));
if (currentPrice > closingPrice)
{
Console.ForegroundColor = ConsoleColor.Green;
}
else
if (currentPrice < closingPrice)
{
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine(input + ": " + currentPrice.ToString());
}
}
}
public void When_I_Scan_all_I_get_the_expected_results()
{
var client = new HBaseClient(_credentials);
var scan = new Scanner();
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scanInfo = null;
try
{
scanInfo = client.CreateScannerAsync(_tableName, scan, scanOptions).Result;
List<FilterTestRecord> actualRecords = RetrieveResults(scanInfo, scanOptions).ToList();
actualRecords.ShouldContainOnly(_allExpectedRecords);
}
finally
{
if (scanInfo != null)
{
client.DeleteScannerAsync(_tableName, scanInfo, scanOptions).Wait();
}
}
}
public void TestFullScan()
{
var client = new HBaseClient(_credentials);
StoreTestData(client);
// full range scan
var scanSettings = new Scanner { batch = 10 };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
CellSet next;
var expectedSet = new HashSet<int>(Enumerable.Range(0, 100));
while ((next = client.ScannerGetNext(scannerInfo)) != null)
{
Assert.AreEqual(10, next.rows.Count);
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
public HBaseWriter()
{
var credentials = CreateFromFile(@"..\..\credentials.txt");
client = new HBaseClient(credentials);
if (!client.ListTables().name.Contains(TABLE_BY_WORDS_NAME))
{
// Create the table
var tableSchema = new TableSchema();
tableSchema.name = TABLE_BY_WORDS_NAME;
tableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(tableSchema);
Console.WriteLine("Table \"{0}\" created.", TABLE_BY_WORDS_NAME);
}
// Read current row count cell
rowCount = GetRowCount();
// Load sentiment dictionary file
LoadDictionary();
writerThread = new Thread(new ThreadStart(WriterThreadFunction));
writerThread.Start();
}
protected override void Context()
{
_credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt");
var client = new HBaseClient(_credentials);
// ensure tables from previous tests are cleaned up
TableList tables = client.ListTables();
foreach (string name in tables.name)
{
if (name.StartsWith(TestTablePrefix, StringComparison.Ordinal))
{
client.DeleteTable(name);
}
}
// add a table specific to this test
_testTableName = TestTablePrefix + _random.Next(10000);
_testTableSchema = new TableSchema();
_testTableSchema.name = _testTableName;
_testTableSchema.columns.Add(new ColumnSchema { name = "d" });
client.CreateTable(_testTableSchema);
}
public void TestGetVersion()
{
var client = new HBaseClient(_credentials);
org.apache.hadoop.hbase.rest.protobuf.generated.Version version = client.GetVersion();
Trace.WriteLine(version);
version.jvmVersion.ShouldNotBeNullOrEmpty();
version.jerseyVersion.ShouldNotBeNullOrEmpty();
version.osVersion.ShouldNotBeNullOrEmpty();
version.restVersion.ShouldNotBeNullOrEmpty();
}
public void TestListTables()
{
var client = new HBaseClient(_credentials);
TableList tables = client.ListTables();
List<string> testtables = tables.name.Where(item => item.StartsWith("marlintest", StringComparison.Ordinal)).ToList();
Assert.AreEqual(1, testtables.Count);
Assert.AreEqual(_testTableName, testtables[0]);
}
public void TestScannerCreation()
{
var client = new HBaseClient(_credentials);
var batchSetting = new Scanner { batch = 2 };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, batchSetting);
Assert.AreEqual(_testTableName, scannerInfo.TableName);
Assert.IsTrue(
scannerInfo.Location.Authority.StartsWith("headnode", StringComparison.Ordinal),
"returned location didn't start with \"headnode\", it was: {0}",
scannerInfo.Location);
}
public void TestStoreSingleCell()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = new HBaseClient(_credentials);
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCells(_testTableName, set);
CellSet cells = client.GetCells(_testTableName, testKey);
Assert.AreEqual(1, cells.rows.Count);
Assert.AreEqual(1, cells.rows[0].values.Count);
Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data));
}
public void TestSubsetScan()
{
var client = new HBaseClient(_credentials);
const int startRow = 15;
const int endRow = 15 + 13;
StoreTestData(client);
// subset range scan
var scanSettings = new Scanner { batch = 10, startRow = BitConverter.GetBytes(startRow), endRow = BitConverter.GetBytes(endRow) };
ScannerInformation scannerInfo = client.CreateScanner(_testTableName, scanSettings);
CellSet next;
var expectedSet = new HashSet<int>(Enumerable.Range(startRow, endRow - startRow));
while ((next = client.ScannerGetNext(scannerInfo)) != null)
{
foreach (CellSet.Row row in next.rows)
{
int k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.AreEqual(0, expectedSet.Count, "The expected set wasn't empty! Items left {0}!", string.Join(",", expectedSet));
}
public void TestGetStorageClusterStatus()
{
var client = new HBaseClient(_credentials);
StorageClusterStatus status = client.GetStorageClusterStatus();
// TODO not really a good test
Assert.IsTrue(status.requests >= 0, "number of requests is negative");
Assert.IsTrue(status.liveNodes.Count >= 1, "number of live nodes is zero or negative");
Assert.IsTrue(status.liveNodes[0].requests >= 0, "number of requests to the first node is negative");
}
private void StoreTestData(HBaseClient hBaseClient)
{
// we are going to insert the keys 0 to 100 and then do some range queries on that
const string testValue = "the force is strong in this column";
var set = new CellSet();
for (int i = 0; i < 100; i++)
{
var row = new CellSet.Row { key = BitConverter.GetBytes(i) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
set.rows.Add(row);
}
hBaseClient.StoreCells(_testTableName, set);
}
public void TestTableSchema()
{
var client = new HBaseClient(_credentials);
TableSchema schema = client.GetTableSchema(_testTableName);
Assert.AreEqual(_testTableName, schema.name);
Assert.AreEqual(_testTableSchema.columns.Count, schema.columns.Count);
Assert.AreEqual(_testTableSchema.columns[0].name, schema.columns[0].name);
}
