public void SaveMyDto(Dto dto)
{
CellSet cellSet = new CellSet();
CellSet.Row cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(_sampleRowKey) };
cellSet.rows.Add(cellSetRow);
Cell value1 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field1"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field1)) };
Cell value2 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field2"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field2)) };
Cell value3 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field3"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field3)) };
Cell value4 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field4"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field4)) };
Cell value5 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field5"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field5)) };
// Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.NestedData))};
byte[] nestedDataBlob;
using (var memoryStream = new MemoryStream())
{
using (var writer = new BsonWriter(memoryStream))
{
_serializer.Serialize(writer, dto.NestedData);
nestedDataBlob = memoryStream.ToArray();
}
}
Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = nestedDataBlob };
cellSetRow.values.AddRange(new List<Cell>() { value1, value2, value3, value4, value5, value6 });
_client.StoreCells(_sampleTableName, cellSet);
}
static void Main(string[] args)
{
while (true)
{
Random rnd = new Random();
Console.Clear();
string clusterURL = "https://hb12345.azurehdinsight.net";
string userName = "******";
string password = "******";
// Connect to HBase cluster
ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL),
userName, password);
HBaseClient hbaseClient = new HBaseClient(creds);
// Get all stocks
Scanner scanSettings = new Scanner()
{
batch = 10,
startRow = Encoding.UTF8.GetBytes("AAA"),
endRow = Encoding.UTF8.GetBytes("ZZZ")
};
ScannerInformation stockScanner = hbaseClient.CreateScanner("Stocks", scanSettings);
CellSet stockCells = null;
while ((stockCells = hbaseClient.ScannerGetNext(stockScanner)) != null)
{
foreach (var row in stockCells.rows)
{
string stock = Encoding.UTF8.GetString(row.key);
Double currentPrice = Double.Parse(Encoding.UTF8.GetString(row.values[1].data));
Double newPrice = currentPrice + (rnd.NextDouble() * (1 - -1) + -1);
Cell c = new Cell
{
column = Encoding.UTF8.GetBytes("Current:Price"),
data =
Encoding.UTF8.GetBytes(newPrice.ToString())
};
row.values.Insert(2, c);
Console.WriteLine(stock + ": " + currentPrice.ToString() + " := "
+ newPrice.ToString());
}
hbaseClient.StoreCells("Stocks", stockCells);
}
}
}
public void StoreTestData(IHBaseClient hbaseClient)
{
// we are going to insert the keys 0 to 100 and then do some range queries on that
const string testValue = "the force is strong in this column";
var set = new CellSet();
for (int i = 0; i < 100; i++)
{
var row = new CellSet.Row { key = BitConverter.GetBytes(i) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
set.rows.Add(row);
}
hbaseClient.StoreCellsAsync(testTableName, set).Wait();
}
public void TestCellsMultiVersionGet()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = CreateClient();
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCellsAsync(testTableName, set).Wait();
client.StoreCellsAsync(testTableName, set).Wait();
CellSet cell = client.GetCellsAsync(testTableName, testKey, "d:starwars", "3").Result;
Assert.AreEqual(2, cell.rows[0].values.Count);
}
public void TestCellsDeletion()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = CreateClient();
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCellsAsync(testTableName, set).Wait();
CellSet cell = client.GetCellsAsync(testTableName, testKey).Result;
// make sure the cell is in the table
Assert.AreEqual(Encoding.UTF8.GetString(cell.rows[0].key), testKey);
// delete cell
client.DeleteCellsAsync(testTableName, testKey).Wait();
// get cell again, 404 exception expected
client.GetCellsAsync(testTableName, testKey).Wait();
}
private void CreateRowCountCell(CellSet set, long count)
{
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(COUNT_ROW_KEY) };
var value = new Cell
{
column = Encoding.UTF8.GetBytes(COUNT_COLUMN_NAME),
data = Encoding.UTF8.GetBytes(count.ToString())
};
row.values.Add(value);
set.rows.Add(row);
}
private static void CreateTweetCells(CellSet set, ITweet tweet)
{
var key = tweet.IdStr;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:created_at"), data = Encoding.UTF8.GetBytes(tweet.CreatedAt.ToLongTimeString()) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:text"), data = Encoding.UTF8.GetBytes(tweet.Text) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString();
value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
if (tweet.Place != null)
{
value = new Cell { column = Encoding.UTF8.GetBytes("d:place_fullname"), data = Encoding.UTF8.GetBytes(tweet.Place.FullName) };
row.values.Add(value);
}
set.rows.Add(row);
}
':', ';', '<', '=', '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~' }; //ascii 58--64 + misc.
private void CreateTweetByWordsCells(CellSet set, ITweet tweet)
{
var words = tweet.Text.ToLower().Split(_punctuationChars);
int sentimentScore = CalcSentimentScore(words);
var word_pairs = words.Take(words.Length - 1)
.Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1]));
var all_words = words.Concat(word_pairs).ToList();
foreach (var word in all_words)
{
var time_index = (ulong.MaxValue -
(ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr;
var key = word + "_" + time_index;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell {
column = Encoding.UTF8.GetBytes("d:id_str"),
data = Encoding.UTF8.GetBytes(tweet.IdStr) };
row.values.Add(value);
value = new Cell {
column = Encoding.UTF8.GetBytes("d:lang"),
data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," +
tweet.Coordinates.Latitude.ToString();
value = new Cell {
column = Encoding.UTF8.GetBytes("d:coor"),
data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
value = new Cell {
column = Encoding.UTF8.GetBytes("d:sentiment"),
data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) };
row.values.Add(value);
set.rows.Add(row);
}
}
private static void Main(string[] args)
{
const string clusterURL = "http://localhost:5555";
const string hadoopUsername = "******";
const string hadoopUserPassword = "******";
var serializer = new JsonDotNetSerializer();
// Create a new instance of an HBase client.
var hbaseClient = new HBaseClient(new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword));
var itemMapper = new ItemMapper(serializer);
hbaseClient.DeleteTable("Items");
var tableSchema = itemMapper.CreateTableSchema("Items");
hbaseClient.CreateTable(tableSchema);
//var perfCheck = new PerformanceChecksManager(hbaseClient, "Items", itemMapper, new ItemsGenerator(),50,100);
//perfCheck.RunPerformanceLoad();
//Generating item.
var itemsGenerator = new ItemsGenerator();
var item = itemsGenerator.CreateItem("mycode");
//Generating cells relevant to the generate item and saving the entity to Hbae.
var itemCellSet = itemMapper.GetCells(item, "en-US");
var stausCell = itemCellSet.rows.Single().values.Single(cell => Encoding.UTF8.GetString(cell.column).Equals("CF1:Status"));
stausCell.data = new byte[0];
hbaseClient.StoreCells("Items", itemCellSet);
//making sure item stored well.
var originalItemCells = hbaseClient.GetCells("Items", item.Code);
var originalItemFetchedFromDb = itemMapper.GetDto(originalItemCells, "en-US");
if (!string.IsNullOrEmpty(originalItemFetchedFromDb.Status))
throw new Exception();
//Describing the conditional update expression.
var cellToCheck = new Cell { column = Encoding.UTF8.GetBytes("CF1:Status"), data = new byte[0] };
//manipulating original item.
stausCell = itemCellSet.rows.Single().values.Single(cell => Encoding.UTF8.GetString(cell.column).Equals("CF1:Status"));
var newStatusValue = Encoding.UTF8.GetBytes("new");
stausCell.data = newStatusValue;
//Testing new functionality...
hbaseClient.CheckAndPutCells("Items", itemCellSet, cellToCheck);
//Thread.Sleep(1);
itemCellSet = hbaseClient.GetCells("Items", item.Code);
var itemFromDb = itemMapper.GetDto(itemCellSet, "en-US");
if (!itemFromDb.Status.Equals(Encoding.UTF8.GetString(newStatusValue)))
throw new Exception();
#region Old-Helper
//string clusterURL = "http://localhost:5555";
//string hadoopUsername = "******";
//string hadoopUserPassword = "******";
//string hbaseTableName = "MyCoolTable";
//// Create a new instance of an HBase client.
//ClusterCredentials creds = new ClusterCredentials(new Uri(clusterURL), hadoopUsername, hadoopUserPassword);
//HBaseClient hbaseClient = new HBaseClient(creds);
//var hbaseHelper = new Helper(hbaseClient, "ThisIsJustATableForShirly1", "DummyKey1ForShirly1");
//hbaseHelper.CreateTable();
//var myClass = new Dto()
//{
// Field1 = "1",
// Field2 = "2",
// Field3 = "3",
// Field4 = "4",
// Field5 = "5",
// NestedData = new NestedDto() { Field1 = "n1", Field2 = "n2" }
//};
//hbaseHelper.SaveMyDto(myClass);
//var dto = hbaseHelper.GetMyDto();
#endregion
}
private CellSet.Row GetCellSet(string key, string columnName, string value, long timestamp)
{
CellSet.Row row = new CellSet.Row() { key = Encoding.UTF8.GetBytes(key) };
Cell c1 = new Cell() { column = BuildCellColumn(ColumnFamilyName1, columnName ), row = row.key };
if (value != null)
{
c1.data = Encoding.UTF8.GetBytes(value);
}
if (timestamp > 0)
{
c1.timestamp = timestamp;
}
row.values.Add(c1);
return row;
}
private void CreateTweetByWordsCells(CellSet set, TweetIndexItem indexItem)
{
var word = indexItem.Word;
var time_index = (ulong.MaxValue -
(ulong)indexItem.CreatedAt).ToString().PadLeft(20) + indexItem.IdStr;
var key = word + "_" + time_index;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell
{
column = Encoding.UTF8.GetBytes("d:id_str"),
data = Encoding.UTF8.GetBytes(indexItem.IdStr)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:lang"),
data = Encoding.UTF8.GetBytes(indexItem.Language)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:coor"),
data = Encoding.UTF8.GetBytes(indexItem.Coordinates)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:sentiment"),
data = Encoding.UTF8.GetBytes(indexItem.SentimentScore.ToString())
};
row.values.Add(value);
set.rows.Add(row);
}
/// <summary>
/// Create a write set for HBase based on cached tuples
/// We delay all the writes to batch the aggregations and writes for them
/// </summary>
/// <returns></returns>
public void WriteToHBase()
{
Context.Logger.Info("WriteToHBase - Start - Writing cached rows into HBase. Rows to write: {0}", cachedTuples.Count);
if (cachedTuples.Count > 0)
{
var writeSet = new CellSet();
foreach (var cachedTuple in cachedTuples)
{
var values = cachedTuple.Value.GetValues();
if(this.HBaseTableColumns.Count < (values.Count - 1))
{
throw new Exception(String.Format(
"Count of HBaseTableColumns is less than fields received. HBaseTableColumns.Count: {0}, Values.Count (without rowkey): {1}",
this.HBaseTableColumns.Count, values.Count - 1)
);
}
//Use the first value as rowkey and add the remaining as a list
var tablerow = new CellSet.Row { key = ToBytes(values[0]) };
//Skip the first value and read the remaining
for (int i = 1; i < values.Count; i++)
{
var rowcell = new Cell
{
//Based on our assumption that ColumnNames do NOT contain rowkey field
column = ToBytes(this.HBaseTableColumnFamily + ":" + this.HBaseTableColumns[i-1]),
data = ToBytes(values[i])
};
tablerow.values.Add(rowcell);
}
writeSet.rows.Add(tablerow);
}
try
{
//Use the StoreCells API to write the cellset into HBase Table
HBaseClusterClient.StoreCells(this.HBaseTableName, writeSet);
}
catch
{
Context.Logger.Error("HBase StoreCells Failed");
foreach(var row in writeSet.rows)
{
Context.Logger.Info("Failed RowKey: {0}, Values (bytes): {1}", Encoding.UTF8.GetString(row.key),
String.Join(", ", row.values.Select(v => Encoding.UTF8.GetString(v.column) + " = " + v.data.LongLength)));
}
throw;
}
Context.Logger.Info("WriteToHBase - End - Stored cells into HBase. Rows written: {0}", writeSet.rows.Count);
}
else
{
Context.Logger.Info("WriteToHBase - End - No cells to write.");
}
}
/// <summary>
/// Create a write set for HBase when the emit period completes
/// We delay all the writes to batch the aggregations and writes for them
/// </summary>
/// <returns></returns>
public override bool EmitAggregations()
{
try
{
if (emitstopwatch.Elapsed > this.appConfig.AggregationWindow)
{
local_hbasescan_count = 0;
var writeset = new CellSet();
var emitime = DateTime.UtcNow.Floor(this.appConfig.AggregationWindow).Subtract(this.appConfig.EmitWindow);
var keystoremove = new List<DateTime>();
var dtkeys = aggregatedCounts.Keys.Where(dt => dt < emitime).OrderBy(dt => dt).ToList();
var hbaseresultsets = new Dictionary<string, Dictionary<string, Dictionary<string, double>>>();
if (dtkeys.Count > 0)
{
var startdt = dtkeys[0];
var enddt = dtkeys[dtkeys.Count-1];
foreach (var dtkey in dtkeys)
{
foreach (var pkey in aggregatedCounts[dtkey].Keys)
{
if (!this.appConfig.HBaseOverwrite && !hbaseresultsets.ContainsKey(pkey))
{
hbaseresultsets.Add(pkey,
ScanHBase(
pkey + Utilities.KEY_DELIMITER + startdt.ToString(Utilities.DATE_TIME_FORMAT),
pkey + Utilities.KEY_DELIMITER + enddt.ToString(Utilities.DATE_TIME_FORMAT)));
}
var rowkey = pkey + Utilities.KEY_DELIMITER + dtkey.ToString(Utilities.DATE_TIME_FORMAT);
var tablerow = new CellSet.Row { key = Encoding.UTF8.GetBytes(rowkey) };
foreach (var skey in aggregatedCounts[dtkey][pkey].Keys)
{
var columnkey = "v:" + skey;
double previousdata = 0;
if (hbaseresultsets.ContainsKey(pkey) &&
hbaseresultsets[pkey].ContainsKey(rowkey) &&
hbaseresultsets[pkey][rowkey].ContainsKey(columnkey))
{
previousdata = hbaseresultsets[pkey][rowkey][columnkey];
}
var rowcell = new Cell
{
column = Encoding.UTF8.GetBytes(columnkey),
data = BitConverter.GetBytes(previousdata + aggregatedCounts[dtkey][pkey][skey])
};
tablerow.values.Add(rowcell);
global_emit_count++;
last_emit_count++;
current_cache_size--;
}
writeset.rows.Add(tablerow);
}
keystoremove.Add(dtkey);
}
}
if (writeset != null && writeset.rows.Count > 0)
{
Context.Logger.Info("HBaseTableName: {0} - Rows to write: {1}, First Rowkey = {2}",
this.HBaseTableName, writeset.rows.Count, Encoding.UTF8.GetString(writeset.rows[0].key));
var localstopwatch = new Stopwatch();
localstopwatch.Start();
//Use the StoreCells API to write the cellset into HBase Table
HBaseClusterClient.StoreCells(this.HBaseTableName, writeset);
Context.Logger.Info("HBase: Table = {0}, Rows Written = {1}, Write Time = {2} secs, Time since last write = {3} secs",
this.HBaseTableName, writeset.rows.Count, localstopwatch.Elapsed.TotalSeconds, emitstopwatch.Elapsed.TotalSeconds);
foreach (var key in keystoremove)
{
aggregatedCounts.Remove(key);
}
last_emit_in_secs = emitstopwatch.Elapsed.TotalSeconds;
emitstopwatch.Restart();
}
if (!this.appConfig.HBaseOverwrite)
{
Context.Logger.Info("ScanHBase: Last Window Scan Count = {0}, Table = {1}", local_hbasescan_count, this.HBaseTableName);
}
}
if (last_emit_count > 0)
{
return true;
}
else
{
return false;
}
}
catch (Exception ex)
{
last_error_count++;
global_error_count++;
Context.Logger.Error(ex.ToString());
return false;
}
}
// Popular a CellSet object to be written into HBase
private void CreateTweetByWordsCells(CellSet set, ITweet tweet)
{
// Split the Tweet into words
string[] words = tweet.Text.ToLower().Split(_punctuationChars);
// Calculate sentiment score base on the words
int sentimentScore = CalcSentimentScore(words);
var word_pairs = words.Take(words.Length - 1)
.Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1]));
var all_words = words.Concat(word_pairs).ToList();
// For each word in the Tweet add a row to the HBase table
foreach (string word in all_words)
{
string time_index = (ulong.MaxValue - (ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr;
string key = word + "_" + time_index;
// Create a row
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
// Add columns to the row, including Tweet identifier, language, coordinator(if available), and sentiment
var value = new Cell { column = Encoding.UTF8.GetBytes("d:id_str"), data = Encoding.UTF8.GetBytes(tweet.IdStr) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString();
value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
value = new Cell { column = Encoding.UTF8.GetBytes("d:sentiment"), data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) };
row.values.Add(value);
set.rows.Add(row);
}
}
private void PopulateTable()
{
var client = new HBaseClient(_credentials);
var cellSet = new CellSet();
string id = Guid.NewGuid().ToString("N");
for (int lineNumber = 0; lineNumber < 10; ++lineNumber)
{
string rowKey = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", id, lineNumber);
// add to expected records
var rec = new FilterTestRecord(rowKey, lineNumber, Guid.NewGuid().ToString("N"), Guid.NewGuid().ToString("D"));
_allExpectedRecords.Add(rec);
// add to row
var row = new CellSet.Row { key = _encoding.GetBytes(rec.RowKey) };
var lineColumnValue = new Cell
{
column = BuildCellColumn(ColumnFamilyName1, LineNumberColumnName),
data = BitConverter.GetBytes(rec.LineNumber)
};
row.values.Add(lineColumnValue);
var paragraphColumnValue = new Cell { column = BuildCellColumn(ColumnFamilyName1, ColumnNameA), data = _encoding.GetBytes(rec.A) };
row.values.Add(paragraphColumnValue);
var columnValueB = new Cell { column = BuildCellColumn(ColumnFamilyName2, ColumnNameB), data = Encoding.UTF8.GetBytes(rec.B) };
row.values.Add(columnValueB);
cellSet.rows.Add(row);
}
client.StoreCellsAsync(_tableName, cellSet).Wait();
}
public void TestStoreSingleCell()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = new HBaseClient(_credentials);
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCells(_testTableName, set);
CellSet cells = client.GetCells(_testTableName, testKey);
Assert.AreEqual(1, cells.rows.Count);
Assert.AreEqual(1, cells.rows[0].values.Count);
Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data));
}
private Cell GetCell(string key, string columnName, string value = null, long timestamp = 0)
{
Cell cell = new Cell() { column = BuildCellColumn(ColumnFamilyName1, columnName) , row = Encoding.UTF8.GetBytes(key) };
if (value != null)
{
cell.data = Encoding.UTF8.GetBytes(value);
}
if (timestamp > 0)
{
cell.timestamp = timestamp;
}
return cell;
}
