public void TestScannerCreation()
{
var client = CreateClient();
var scanSettings = new Scanner {
batch = 2
};
var scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
Assert.Equal(testTableName, scannerInfo.TableName);
Assert.NotNull(scannerInfo.ScannerId);
Assert.False(scannerInfo.ScannerId.StartsWith("/"), "scanner id starts with a slash");
Assert.NotNull(scannerInfo.ResponseHeaderCollection);
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
public void TestScannerDeletion()
{
var client = CreateClient();
// full range scan
var scanSettings = new Scanner {
batch = 10
};
var scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
Assert.Equal(testTableName, scannerInfo.TableName);
Assert.NotNull(scannerInfo.ScannerId);
Assert.False(scannerInfo.ScannerId.StartsWith("/"), "scanner id starts with a slash");
Assert.NotNull(scannerInfo.ResponseHeaderCollection);
// delete the scanner
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
// try to fetch data use the deleted scanner
scanOptions.RetryPolicy =
Policy.Handle <HttpRequestException>().WaitAndRetry(1, x => TimeSpan.FromSeconds(1));
client.ScannerGetNextAsync(scannerInfo, scanOptions).Wait();
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
public override void TestScannerDeletion()
{
var client = CreateClient();
// full range scan
var scanSettings = new Scanner {
batch = 10
};
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
Assert.AreEqual(testTableName, scannerInfo.TableName);
Assert.IsNotNull(scannerInfo.ScannerId);
Assert.IsFalse(scannerInfo.ScannerId.StartsWith("/"), "scanner id starts with a slash");
Assert.IsNotNull(scannerInfo.ResponseHeaderCollection);
// delete the scanner
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
// try to fetch data use the deleted scanner
scanOptions.RetryPolicy = RetryPolicy.NoRetry;
client.ScannerGetNextAsync(scannerInfo, scanOptions).Wait();
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
private long GetRowCount()
{
try
{
// Overwrite default retry policy (Exponential) to NoRetry.
// 404 NOT FOUND is falsely treated as exception and will be stuck here
RequestOptions options = RequestOptions.GetDefaultOptions();
options.RetryPolicy = RetryPolicy.NoRetry;
var cellSet = client.GetCellsAsync(TABLE_BY_WORDS_NAME, COUNT_ROW_KEY, null, null, options).Result;
if (cellSet.rows.Count != 0)
{
var countCol = cellSet.rows[0].values.Find(cell => Encoding.UTF8.GetString(cell.column) == COUNT_COLUMN_NAME);
if (countCol != null)
{
return(Convert.ToInt64(Encoding.UTF8.GetString(countCol.data)));
}
}
}
catch (Exception ex)
{
if (ex.InnerException.Message.Equals("The remote server returned an error: (404) Not Found.", StringComparison.OrdinalIgnoreCase))
{
return(0);
}
else
{
throw ex;
}
}
return(0);
}
public override IHBaseClient CreateClient()
{
var options = RequestOptions.GetDefaultOptions();
options.RetryPolicy = Policy.Handle <HttpRequestException>().WaitAndRetry(1, x => TimeSpan.FromSeconds(1));
options.TimeoutMillis = 30000;
options.KeepAlive = false;
return(new HBaseClient(ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"), options));
}
public override IHBaseClient CreateClient()
{
var options = RequestOptions.GetDefaultOptions();
options.RetryPolicy = RetryPolicy.NoRetry;
options.TimeoutMillis = 30000;
options.KeepAlive = false;
return(new HBaseClient(ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt"), options));
}
/// <inheritdoc />
public virtual async Task MakeRequest(
Func <TApiService, CancellationToken, Task> operation,
RequestOptions?options = null)
{
options ??= RequestOptions.GetDefaultOptions();
await CreatePolicy(options)
.ExecuteAsync(ct => operation(_apiService, ct), options.CancellationToken)
.ConfigureAwait(false);
}
/// <inheritdoc />
public virtual async Task <TResult> MakeRequest <TResult>(
Func <TApiService, CancellationToken, Task <TResult> > operation,
RequestOptions?options = null)
{
options ??= RequestOptions.GetDefaultOptions();
return(await CreatePolicy(options)
.AsAsyncPolicy <TResult>()
.ExecuteAsync(ct => operation(_apiService, ct), options.CancellationToken)
.ConfigureAwait(false));
}
public override IHBaseClient CreateClient()
{
var regionServerIPs = new List <string>();
// TODO automatically retrieve IPs from Ambari REST APIs
regionServerIPs.Add("10.17.0.7");
regionServerIPs.Add("10.17.0.4");
var options = RequestOptions.GetDefaultOptions();
options.Port = 8090;
options.AlternativeEndpoint = "/";
return(new HBaseClient(null, options, new LoadBalancerRoundRobin(regionServerIPs)));
}
public override void TestSubsetScan()
{
var client = CreateClient();
const int startRow = 15;
const int endRow = 15 + 13;
StoreTestData(client);
// subset range scan
var scanSettings = new Scanner
{
batch = 10,
startRow = BitConverter.GetBytes(startRow),
endRow = BitConverter.GetBytes(endRow)
};
var scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
CellSet next;
var expectedSet = new HashSet <int>(Enumerable.Range(startRow, endRow - startRow));
while ((next = client.ScannerGetNextAsync(scannerInfo, scanOptions).Result) != null)
{
foreach (var row in next.rows)
{
var k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.Empty(expectedSet);
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
public override IHBaseClient CreateClient()
{
var regionServerHostNames = new List <string>();
// TODO add different behaviour for linux vs. windows
regionServerHostNames.Add("wn0-");
regionServerHostNames.Add("wn1-");
var options = RequestOptions.GetDefaultOptions();
options.RetryPolicy = RetryPolicy.NoRetry;
options.TimeoutMillis = 30000;
options.KeepAlive = false;
options.Port = 8090;
options.AlternativeEndpoint = "/";
return(new HBaseClient(null, options, new LoadBalancerRoundRobin(regionServerHostNames)));
}
private HBaseClient GetClient()
{
_credentials = ClusterCredentialsFactory.CreateFromFile(@".\credentials.txt");
var options = RequestOptions.GetDefaultOptions();
options.RetryPolicy = RetryPolicy.NoRetry;
var client = new HBaseClient(_credentials, options);
#region VNet
//options.TimeoutMillis = 30000;
//options.KeepAlive = false;
//options.Port = 8090;
//options.AlternativeEndpoint = "/";
//var client = new HBaseClient(null, options, new LoadBalancerRoundRobin(new List<string> { "ip address" }));
#endregion
return(client);
}
public override void TestFullScan()
{
var client = CreateClient();
StoreTestData(client);
var scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = Constants.RestEndpointBaseZero;
// full range scan
var scanSettings = new Scanner {
batch = 10
};
ScannerInformation scannerInfo = null;
try
{
scannerInfo = client.CreateScannerAsync(testTableName, scanSettings, scanOptions).Result;
CellSet next;
var expectedSet = new HashSet <int>(Enumerable.Range(0, 100));
while ((next = client.ScannerGetNextAsync(scannerInfo, scanOptions).Result) != null)
{
Assert.Equal(10, next.rows.Count);
foreach (var row in next.rows)
{
var k = BitConverter.ToInt32(row.key, 0);
expectedSet.Remove(k);
}
}
Assert.Empty(expectedSet);
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(testTableName, scannerInfo, scanOptions).Wait();
}
}
}
public async Task <IEnumerable <Tweet> > QueryTweetsByKeywordAsync(string keyword)
{
var list = new List <Tweet>();
var time_index = (ulong.MaxValue -
(ulong)DateTime.UtcNow.Subtract(new TimeSpan(6, 0, 0)).ToBinary()).ToString().PadLeft(20);
var startRow = keyword + "_" + time_index;
var endRow = keyword + "|";
RequestOptions scanOptions = RequestOptions.GetDefaultOptions();
scanOptions.AlternativeEndpoint = "hbaserest0/";
var scanSettings = new Scanner {
batch = 100000,
startRow = Encoding.UTF8.GetBytes(startRow),
endRow = Encoding.UTF8.GetBytes(endRow)
};
ScannerInformation scannerInfo = null;
try
{
scannerInfo = await client.CreateScannerAsync(tableByWordsName, scanSettings, scanOptions);
CellSet next;
while ((next = await client.ScannerGetNextAsync(scannerInfo, scanOptions)) != null)
{
foreach (CellSet.Row row in next.rows)
{
var coordinates =
row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:coor");
if (coordinates != null)
{
var lonlat = Encoding.UTF8.GetString(coordinates.data).Split(',');
var sentimentField =
row.values.Find(c => Encoding.UTF8.GetString(c.column) == "d:sentiment");
var sentiment = 0;
if (sentimentField != null)
{
sentiment = Convert.ToInt32(Encoding.UTF8.GetString(sentimentField.data));
}
list.Add(new Tweet
{
Longtitude = Convert.ToDouble(lonlat[0]),
Latitude = Convert.ToDouble(lonlat[1]),
Sentiment = sentiment
});
}
if (coordinates != null)
{
var lonlat = Encoding.UTF8.GetString(coordinates.data).Split(',');
}
}
}
return(list);
}
finally
{
if (scannerInfo != null)
{
client.DeleteScannerAsync(tableByWordsName, scannerInfo, scanOptions).Wait();
}
}
}