public CellSet GetCells(ItemDto item, string culture)
{
var cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(item.Code) };
var cells = new List<Cell>();
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Status"), data = Encoding.UTF8.GetBytes(item.Status) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:MH" + item.MerchandiseHierarchy.Code), data = _serializer.SerializeToBson(item.MerchandiseHierarchy) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:CreatedDate"), data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:UpdatedDate"), data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Long_" + culture), data = Encoding.UTF8.GetBytes(item.Description) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Short_" + culture), data = Encoding.UTF8.GetBytes(item.ShortDescription) });
foreach (var group in item.Groups)
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:G" + group.Code), data = _serializer.SerializeToBson(group) });
foreach (var identifier in item.Identifiers)
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:I" + identifier.Type), data = Encoding.UTF8.GetBytes(identifier.Value) });
foreach (var attributes in item.Attributes)
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:A" + culture + "_" + attributes.Id), data = _serializer.SerializeToBson(attributes.Value) });
cellSetRow.values.AddRange(cells);
var cellSet = new CellSet();
cellSet.rows.Add(cellSetRow);
return cellSet;
}
public void WriteTweet(FSTweet tweet)
{
CellSet.Row row = TableDomainMappers.TweetToRow(tweet);
CellSet set = new CellSet();
set.rows.Add(row);
HadoopContext.HBaseClient.StoreCells(HadoopContext.TweetTableName, set);
}
public void WriteWordRelation(FSWordRelationship relation)
{
CellSet.Row row = TableDomainMappers.WordRelationToRow(relation);
CellSet set = new CellSet();
set.rows.Add(row);
HadoopContext.HBaseClient.StoreCells(HadoopContext.WordRelationTableName, set);
}
public void SaveMyDto(Dto dto)
{
CellSet cellSet = new CellSet();
CellSet.Row cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(_sampleRowKey) };
cellSet.rows.Add(cellSetRow);
Cell value1 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field1"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field1)) };
Cell value2 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field2"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field2)) };
Cell value3 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field3"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field3)) };
Cell value4 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field4"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field4)) };
Cell value5 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field5"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field5)) };
// Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.NestedData))};
byte[] nestedDataBlob;
using (var memoryStream = new MemoryStream())
{
using (var writer = new BsonWriter(memoryStream))
{
_serializer.Serialize(writer, dto.NestedData);
nestedDataBlob = memoryStream.ToArray();
}
}
Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = nestedDataBlob };
cellSetRow.values.AddRange(new List<Cell>() { value1, value2, value3, value4, value5, value6 });
_client.StoreCells(_sampleTableName, cellSet);
}
public void UpdateRowCount(long rowCount)
{
var set = new CellSet();
CreateRowCountCell(set, rowCount);
client.StoreCells(TABLE_BY_WORDS_NAME, set);
}
// Popular a CellSet object to be written into HBase
private void CreateTweetByWordsCells(CellSet set, TweetSentimentData tweet)
{
// Create a row with a key
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) };
// Add columns to the row
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:Text"),
data = Encoding.UTF8.GetBytes(tweet.Text) });
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:CreatedOn"),
data = Encoding.UTF8.GetBytes(tweet.CreatedOn.ToString()) });
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:ReplyToId"),
data = Encoding.UTF8.GetBytes(tweet.ReplyToId) });
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:Sentiment"),
data = Encoding.UTF8.GetBytes(tweet.Sentiment.ToString()) });
if (tweet.Coordinates != null)
{
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:Coordinates"),
data = Encoding.UTF8.GetBytes(tweet.Coordinates) });
}
set.rows.Add(row);
}
public void WriteConversation(FSConversation convo)
{
CellSet.Row row = TableDomainMappers.ConversationToRow(convo);
CellSet set = new CellSet();
set.rows.Add(row);
HadoopContext.HBaseClient.StoreCells(HadoopContext.ConversationTableName, set);
Thread.Sleep(100);
}
public void WriteTweets(IEnumerable<FSTweet> tweets)
{
CellSet set = new CellSet();
foreach(FSTweet t in tweets)
{
set.rows.Add(TableDomainMappers.TweetToRow(t));
}
HadoopContext.HBaseClient.StoreCells(HadoopContext.TweetTableName, set);
}
public void WriteWordRelations(IEnumerable<FSWordRelationship> relations)
{
CellSet set = new CellSet();
foreach(FSWordRelationship relation in relations)
{
set.rows.Add(TableDomainMappers.WordRelationToRow(relation));
}
HadoopContext.HBaseClient.StoreCells(HadoopContext.WordRelationTableName, set);
}
public void WriteIndexItems(List<TweetIndexItem> items)
{
var set = new CellSet();
foreach(var item in items)
{
CreateTweetByWordsCells(set, item);
}
client.StoreCells(TABLE_BY_WORDS_NAME, set);
}
private void CreateRowCountCell(CellSet set, long count)
{
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(COUNT_ROW_KEY) };
var value = new Cell
{
column = Encoding.UTF8.GetBytes(COUNT_COLUMN_NAME),
data = Encoding.UTF8.GetBytes(count.ToString())
};
row.values.Add(value);
set.rows.Add(row);
}
public void StoreTestData(IHBaseClient hbaseClient)
{
// we are going to insert the keys 0 to 100 and then do some range queries on that
const string testValue = "the force is strong in this column";
var set = new CellSet();
for (int i = 0; i < 100; i++)
{
var row = new CellSet.Row { key = BitConverter.GetBytes(i) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
set.rows.Add(row);
}
hbaseClient.StoreCellsAsync(testTableName, set).Wait();
}
public void TestCellsMultiVersionGet()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = CreateClient();
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCellsAsync(testTableName, set).Wait();
client.StoreCellsAsync(testTableName, set).Wait();
CellSet cell = client.GetCellsAsync(testTableName, testKey, "d:starwars", "3").Result;
Assert.AreEqual(2, cell.rows[0].values.Count);
}
public IEnumerable<FSWordRelationship> ReadAllWordRelationships()
{
Scanner s = new Scanner()
{
batch = 10
};
ScannerInformation si = client.CreateScanner(HadoopContext.WordRelationTableName, s);
CellSet next = null;
CellSet readRows = new CellSet();
while ((next = client.ScannerGetNext(si)) != null)
{
foreach (CellSet.Row row in next.rows)
{
//convert row into desired domain type....
readRows.rows.Add(row);
}
}
return null;
}
public void TestCellsDeletion()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = CreateClient();
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCellsAsync(testTableName, set).Wait();
CellSet cell = client.GetCellsAsync(testTableName, testKey).Result;
// make sure the cell is in the table
Assert.AreEqual(Encoding.UTF8.GetString(cell.rows[0].key), testKey);
// delete cell
client.DeleteCellsAsync(testTableName, testKey).Wait();
// get cell again, 404 exception expected
client.GetCellsAsync(testTableName, testKey).Wait();
}
public ItemDto GetDto(CellSet cellSet, string culture)
{
var itemRow = cellSet.rows.First();
//Retrieve a cell by its key.
var columnNameMapping = itemRow.values.ToDictionary(o => Encoding.UTF8.GetString(o.column), o => o.data);
var item = new ItemDto();
item.Code = Encoding.UTF8.GetString(itemRow.key);
item.Status = Encoding.UTF8.GetString(columnNameMapping["CF1:Status"]);
var hierarchyColumnKey = columnNameMapping.Keys.Where(o => o.StartsWith("CF1:MH"));
item.MerchandiseHierarchy = _serializer.DeseralizeFromBson<HierarchyDto>(columnNameMapping[hierarchyColumnKey.First()]);
item.CreatedDate = DateTime.Parse(Encoding.UTF8.GetString(columnNameMapping["CF1:CreatedDate"]));
item.LastUpdateDate = DateTime.Parse(Encoding.UTF8.GetString(columnNameMapping["CF1:UpdatedDate"]));
item.Description = Encoding.UTF8.GetString(columnNameMapping["CF1:Long_" + culture]);
item.ShortDescription = Encoding.UTF8.GetString(columnNameMapping["CF1:Short_" + culture]);
var groupColumnKeys = columnNameMapping.Keys.Where(o => o.StartsWith("CF1:G"));
item.Groups = new List<HierarchyDto>();
foreach (var groupKey in groupColumnKeys)
item.Groups.Add(_serializer.DeseralizeFromBson<HierarchyDto>(columnNameMapping[groupKey]));
var identifiersColumnKeys = columnNameMapping.Keys.Where(o => o.StartsWith("CF1:I"));
item.Identifiers = new List<Identifier>();
foreach (var identifierKey in identifiersColumnKeys)
item.Identifiers.Add(new Identifier{ Value= Encoding.UTF8.GetString(columnNameMapping[identifierKey]),Type= identifierKey.Substring(5)});
var attributesColumnKeys = columnNameMapping.Keys.Where(o => o.StartsWith("CF1:A" + culture));
item.Attributes = new List<AttributeDto>();
foreach (var attributeKey in attributesColumnKeys)
item.Attributes.Add(new AttributeDto { Id = attributeKey.Substring(11), Value = _serializer.DeseralizeFromBson<AttributeValue>(columnNameMapping[attributeKey]) });
return item;
}
':', ';', '<', '=', '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~' }; //ascii 58--64 + misc.
private void CreateTweetByWordsCells(CellSet set, ITweet tweet)
{
var words = tweet.Text.ToLower().Split(_punctuationChars);
int sentimentScore = CalcSentimentScore(words);
var word_pairs = words.Take(words.Length - 1)
.Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1]));
var all_words = words.Concat(word_pairs).ToList();
foreach (var word in all_words)
{
var time_index = (ulong.MaxValue -
(ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr;
var key = word + "_" + time_index;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell {
column = Encoding.UTF8.GetBytes("d:id_str"),
data = Encoding.UTF8.GetBytes(tweet.IdStr) };
row.values.Add(value);
value = new Cell {
column = Encoding.UTF8.GetBytes("d:lang"),
data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," +
tweet.Coordinates.Latitude.ToString();
value = new Cell {
column = Encoding.UTF8.GetBytes("d:coor"),
data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
value = new Cell {
column = Encoding.UTF8.GetBytes("d:sentiment"),
data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) };
row.values.Add(value);
set.rows.Add(row);
}
}
private void CreateTweetByWordsCells(CellSet set, TweetIndexItem indexItem)
{
var word = indexItem.Word;
var time_index = (ulong.MaxValue -
(ulong)indexItem.CreatedAt).ToString().PadLeft(20) + indexItem.IdStr;
var key = word + "_" + time_index;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell
{
column = Encoding.UTF8.GetBytes("d:id_str"),
data = Encoding.UTF8.GetBytes(indexItem.IdStr)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:lang"),
data = Encoding.UTF8.GetBytes(indexItem.Language)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:coor"),
data = Encoding.UTF8.GetBytes(indexItem.Coordinates)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:sentiment"),
data = Encoding.UTF8.GetBytes(indexItem.SentimentScore.ToString())
};
row.values.Add(value);
set.rows.Add(row);
}
/// <summary>
/// Create a write set for HBase based on cached tuples
/// We delay all the writes to batch the aggregations and writes for them
/// </summary>
/// <returns></returns>
public void WriteToHBase()
{
Context.Logger.Info("WriteToHBase - Start - Writing cached rows into HBase. Rows to write: {0}", cachedTuples.Count);
if (cachedTuples.Count > 0)
{
var writeSet = new CellSet();
foreach (var cachedTuple in cachedTuples)
{
var values = cachedTuple.Value.GetValues();
if(this.HBaseTableColumns.Count < (values.Count - 1))
{
throw new Exception(String.Format(
"Count of HBaseTableColumns is less than fields received. HBaseTableColumns.Count: {0}, Values.Count (without rowkey): {1}",
this.HBaseTableColumns.Count, values.Count - 1)
);
}
//Use the first value as rowkey and add the remaining as a list
var tablerow = new CellSet.Row { key = ToBytes(values[0]) };
//Skip the first value and read the remaining
for (int i = 1; i < values.Count; i++)
{
var rowcell = new Cell
{
//Based on our assumption that ColumnNames do NOT contain rowkey field
column = ToBytes(this.HBaseTableColumnFamily + ":" + this.HBaseTableColumns[i-1]),
data = ToBytes(values[i])
};
tablerow.values.Add(rowcell);
}
writeSet.rows.Add(tablerow);
}
try
{
//Use the StoreCells API to write the cellset into HBase Table
HBaseClusterClient.StoreCells(this.HBaseTableName, writeSet);
}
catch
{
Context.Logger.Error("HBase StoreCells Failed");
foreach(var row in writeSet.rows)
{
Context.Logger.Info("Failed RowKey: {0}, Values (bytes): {1}", Encoding.UTF8.GetString(row.key),
String.Join(", ", row.values.Select(v => Encoding.UTF8.GetString(v.column) + " = " + v.data.LongLength)));
}
throw;
}
Context.Logger.Info("WriteToHBase - End - Stored cells into HBase. Rows written: {0}", writeSet.rows.Count);
}
else
{
Context.Logger.Info("WriteToHBase - End - No cells to write.");
}
}
public IEnumerable<FSWordRelationship> FindRelationsWithWord(string word)
{
Scanner s = new Scanner()
{
batch = 1000
};
ScannerInformation si = client.CreateScanner(HadoopContext.HBaseWordRelationTableName, s);
CellSet next = null;
CellSet readRows = new CellSet();
List<FSWordRelationship> relations = new List<FSWordRelationship>();
while ((next = client.ScannerGetNext(si)) != null)
{
foreach (CellSet.Row row in next.rows)
{
//convert row into desired domain type....
var w1 = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordOne");
string wordOne = Encoding.UTF8.GetString(w1.data);
var w1Id = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordOneId");
int wordOneId = Convert.ToInt32(Encoding.UTF8.GetString(w1Id.data));
var w2 = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordTwo");
string wordTwo = Encoding.UTF8.GetString(w2.data);
var w2Id = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:WordTwoId");
int wordTwoId = Convert.ToInt32(Encoding.UTF8.GetString(w2Id.data));
var rS = row.values.Find(w => Encoding.UTF8.GetString(w.column) == "d:RScore");
var rScore = Convert.ToDouble(Encoding.UTF8.GetString(rS.data));
var id = Encoding.UTF8.GetString(row.key);
if (word.CompareTo(wordOne) == 0 || word.CompareTo(wordTwo) == 0
&& !(String.IsNullOrWhiteSpace(wordOne) || String.IsNullOrWhiteSpace(wordTwo)))
{
FSWordRelationship rel = new FSWordRelationship(id, wordOne, wordOneId, wordTwo, wordTwoId, rScore);
relations.Add(rel);
}
}
}
return relations;
}
private void PopulateTable()
{
var client = new HBaseClient(_credentials);
var cellSet = new CellSet();
string id = Guid.NewGuid().ToString("N");
for (int lineNumber = 0; lineNumber < 10; ++lineNumber)
{
string rowKey = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", id, lineNumber);
// add to expected records
var rec = new FilterTestRecord(rowKey, lineNumber, Guid.NewGuid().ToString("N"), Guid.NewGuid().ToString("D"));
_allExpectedRecords.Add(rec);
// add to row
var row = new CellSet.Row { key = _encoding.GetBytes(rec.RowKey) };
var lineColumnValue = new Cell
{
column = BuildCellColumn(ColumnFamilyName1, LineNumberColumnName),
data = BitConverter.GetBytes(rec.LineNumber)
};
row.values.Add(lineColumnValue);
var paragraphColumnValue = new Cell { column = BuildCellColumn(ColumnFamilyName1, ColumnNameA), data = _encoding.GetBytes(rec.A) };
row.values.Add(paragraphColumnValue);
var columnValueB = new Cell { column = BuildCellColumn(ColumnFamilyName2, ColumnNameB), data = Encoding.UTF8.GetBytes(rec.B) };
row.values.Add(columnValueB);
cellSet.rows.Add(row);
}
client.StoreCellsAsync(_tableName, cellSet).Wait();
}
// Popular a CellSet object to be written into HBase
private void CreateTweetByWordsCells(CellSet set, ITweet tweet)
{
// Split the Tweet into words
string[] words = tweet.Text.ToLower().Split(_punctuationChars);
// Calculate sentiment score base on the words
int sentimentScore = CalcSentimentScore(words);
var word_pairs = words.Take(words.Length - 1)
.Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1]));
var all_words = words.Concat(word_pairs).ToList();
// For each word in the Tweet add a row to the HBase table
foreach (string word in all_words)
{
string time_index = (ulong.MaxValue - (ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr;
string key = word + "_" + time_index;
// Create a row
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
// Add columns to the row, including Tweet identifier, language, coordinator(if available), and sentiment
var value = new Cell { column = Encoding.UTF8.GetBytes("d:id_str"), data = Encoding.UTF8.GetBytes(tweet.IdStr) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString();
value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
value = new Cell { column = Encoding.UTF8.GetBytes("d:sentiment"), data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) };
row.values.Add(value);
set.rows.Add(row);
}
}
public void TestStoreSingleCell()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = new HBaseClient(_credentials);
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCells(_testTableName, set);
CellSet cells = client.GetCells(_testTableName, testKey);
Assert.AreEqual(1, cells.rows.Count);
Assert.AreEqual(1, cells.rows[0].values.Count);
Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data));
}
// Write a Tweet (CellSet) to HBase
public void WriterThreadFunction()
{
try
{
while (threadRunning)
{
if (queue.Count > 0)
{
CellSet set = new CellSet();
lock (queue)
{
do
{
ITweet tweet = queue.Dequeue();
CreateTweetByWordsCells(set, tweet);
} while (queue.Count > 0);
}
// Write the Tweet by words cell set to the HBase table
client.StoreCells(HBASETABLENAME, set);
Console.WriteLine("\tRows written: {0}", set.rows.Count);
}
Thread.Sleep(100);
}
}
catch (Exception ex)
{
Console.WriteLine("Exception: " + ex.Message);
}
}
/// <summary>
/// Create a write set for HBase when the emit period completes
/// We delay all the writes to batch the aggregations and writes for them
/// </summary>
/// <returns></returns>
public override bool EmitAggregations()
{
try
{
if (emitstopwatch.Elapsed > this.appConfig.AggregationWindow)
{
local_hbasescan_count = 0;
var writeset = new CellSet();
var emitime = DateTime.UtcNow.Floor(this.appConfig.AggregationWindow).Subtract(this.appConfig.EmitWindow);
var keystoremove = new List<DateTime>();
var dtkeys = aggregatedCounts.Keys.Where(dt => dt < emitime).OrderBy(dt => dt).ToList();
var hbaseresultsets = new Dictionary<string, Dictionary<string, Dictionary<string, double>>>();
if (dtkeys.Count > 0)
{
var startdt = dtkeys[0];
var enddt = dtkeys[dtkeys.Count-1];
foreach (var dtkey in dtkeys)
{
foreach (var pkey in aggregatedCounts[dtkey].Keys)
{
if (!this.appConfig.HBaseOverwrite && !hbaseresultsets.ContainsKey(pkey))
{
hbaseresultsets.Add(pkey,
ScanHBase(
pkey + Utilities.KEY_DELIMITER + startdt.ToString(Utilities.DATE_TIME_FORMAT),
pkey + Utilities.KEY_DELIMITER + enddt.ToString(Utilities.DATE_TIME_FORMAT)));
}
var rowkey = pkey + Utilities.KEY_DELIMITER + dtkey.ToString(Utilities.DATE_TIME_FORMAT);
var tablerow = new CellSet.Row { key = Encoding.UTF8.GetBytes(rowkey) };
foreach (var skey in aggregatedCounts[dtkey][pkey].Keys)
{
var columnkey = "v:" + skey;
double previousdata = 0;
if (hbaseresultsets.ContainsKey(pkey) &&
hbaseresultsets[pkey].ContainsKey(rowkey) &&
hbaseresultsets[pkey][rowkey].ContainsKey(columnkey))
{
previousdata = hbaseresultsets[pkey][rowkey][columnkey];
}
var rowcell = new Cell
{
column = Encoding.UTF8.GetBytes(columnkey),
data = BitConverter.GetBytes(previousdata + aggregatedCounts[dtkey][pkey][skey])
};
tablerow.values.Add(rowcell);
global_emit_count++;
last_emit_count++;
current_cache_size--;
}
writeset.rows.Add(tablerow);
}
keystoremove.Add(dtkey);
}
}
if (writeset != null && writeset.rows.Count > 0)
{
Context.Logger.Info("HBaseTableName: {0} - Rows to write: {1}, First Rowkey = {2}",
this.HBaseTableName, writeset.rows.Count, Encoding.UTF8.GetString(writeset.rows[0].key));
var localstopwatch = new Stopwatch();
localstopwatch.Start();
//Use the StoreCells API to write the cellset into HBase Table
HBaseClusterClient.StoreCells(this.HBaseTableName, writeset);
Context.Logger.Info("HBase: Table = {0}, Rows Written = {1}, Write Time = {2} secs, Time since last write = {3} secs",
this.HBaseTableName, writeset.rows.Count, localstopwatch.Elapsed.TotalSeconds, emitstopwatch.Elapsed.TotalSeconds);
foreach (var key in keystoremove)
{
aggregatedCounts.Remove(key);
}
last_emit_in_secs = emitstopwatch.Elapsed.TotalSeconds;
emitstopwatch.Restart();
}
if (!this.appConfig.HBaseOverwrite)
{
Context.Logger.Info("ScanHBase: Last Window Scan Count = {0}, Table = {1}", local_hbasescan_count, this.HBaseTableName);
}
}
if (last_emit_count > 0)
{
return true;
}
else
{
return false;
}
}
catch (Exception ex)
{
last_error_count++;
global_error_count++;
Context.Logger.Error(ex.ToString());
return false;
}
}
/// <summary>
/// batch insert data to hbase
/// </summary>
/// <param name="tableName"></param>
/// <param name="cellSet"></param>
/// <param name="num"></param>
/// <returns></returns>
public static async Task HbaseBatchInsertAsync(string tableName, CellSet cellSet)
{
if (hbaseClient == null)
hbaseClient = CreateHBaseClient(clusterURL, httpName, httpUserPassword);
await hbaseClient.StoreCellsAsync(tableName, cellSet);
}
// Write a Tweet (CellSet) to HBase
public void WriterThreadFunction()
{
while (ThreadRunning)
{
if (WriteQueue.Count > 0)
{
CellSet set = new CellSet();
lock (WriteQueue)
{
do
{
TweetSentimentData tweet = WriteQueue.Dequeue();
CreateTweetByWordsCells(set, tweet);
} while (WriteQueue.Count > 0);
}
// Write the Tweet by words cell set to the HBase table
client.StoreCells(this.HBaseTableName, set);
Console.WriteLine("\tRows written: {0}", set.rows.Count);
}
}
}
public void WriterThreadFunction()
{
while(threadRunning)
{
try
{
if (queue.Count > 0)
{
var set = new CellSet();
lock (queue)
{
do
{
var tweet = queue.Dequeue();
CreateTweetByWordsCells(set, tweet);
} while (queue.Count > 0);
}
// Update count of rows as part of the same batch
CreateRowCountCell(set, rowCount + set.rows.Count);
client.StoreCells(TABLE_BY_WORDS_NAME, set);
rowCount += set.rows.Count;
Console.WriteLine("===== {0} rows written =====", set.rows.Count);
}
Thread.Sleep(100);
}
catch (Exception ex)
{
Console.WriteLine("Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace);
}
}
}
private static void CreateTweetCells(CellSet set, ITweet tweet)
{
var key = tweet.IdStr;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:created_at"), data = Encoding.UTF8.GetBytes(tweet.CreatedAt.ToLongTimeString()) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:text"), data = Encoding.UTF8.GetBytes(tweet.Text) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString();
value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
if (tweet.Place != null)
{
value = new Cell { column = Encoding.UTF8.GetBytes("d:place_fullname"), data = Encoding.UTF8.GetBytes(tweet.Place.FullName) };
row.values.Add(value);
}
set.rows.Add(row);
}
/// <summary>
/// batch insert data to hbase
/// </summary>
/// <param name="tableName"></param>
/// <param name="cellSetRows"></param>
/// <param name="num"></param>
public static async Task HbaseBatchInsertAsync(string tableName, List<CellSet.Row> cellSetRows)
{
CellSet cellSet = new CellSet();
foreach (var cellSetRow in cellSetRows)
{
cellSet.rows.Add(cellSetRow);
}
await HbaseBatchInsertAsync(tableName, cellSet);
}
