public void SaveMyDto(Dto dto)
{
CellSet cellSet = new CellSet();
CellSet.Row cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(_sampleRowKey) };
cellSet.rows.Add(cellSetRow);
Cell value1 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field1"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field1)) };
Cell value2 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field2"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field2)) };
Cell value3 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field3"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field3)) };
Cell value4 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field4"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field4)) };
Cell value5 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field5"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field5)) };
// Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.NestedData))};
byte[] nestedDataBlob;
using (var memoryStream = new MemoryStream())
{
using (var writer = new BsonWriter(memoryStream))
{
_serializer.Serialize(writer, dto.NestedData);
nestedDataBlob = memoryStream.ToArray();
}
}
Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = nestedDataBlob };
cellSetRow.values.AddRange(new List<Cell>() { value1, value2, value3, value4, value5, value6 });
_client.StoreCells(_sampleTableName, cellSet);
}
public CellSet GetCells(ItemDto item, string culture)
{
var cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(item.Code) };
var cells = new List<Cell>();
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Status"), data = Encoding.UTF8.GetBytes(item.Status) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:MH" + item.MerchandiseHierarchy.Code), data = _serializer.SerializeToBson(item.MerchandiseHierarchy) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:CreatedDate"), data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:UpdatedDate"), data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Long_" + culture), data = Encoding.UTF8.GetBytes(item.Description) });
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Short_" + culture), data = Encoding.UTF8.GetBytes(item.ShortDescription) });
foreach (var group in item.Groups)
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:G" + group.Code), data = _serializer.SerializeToBson(group) });
foreach (var identifier in item.Identifiers)
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:I" + identifier.Type), data = Encoding.UTF8.GetBytes(identifier.Value) });
foreach (var attributes in item.Attributes)
cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:A" + culture + "_" + attributes.Id), data = _serializer.SerializeToBson(attributes.Value) });
cellSetRow.values.AddRange(cells);
var cellSet = new CellSet();
cellSet.rows.Add(cellSetRow);
return cellSet;
}
// Popular a CellSet object to be written into HBase
private void CreateTweetByWordsCells(CellSet set, TweetSentimentData tweet)
{
// Create a row with a key
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) };
// Add columns to the row
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:Text"),
data = Encoding.UTF8.GetBytes(tweet.Text) });
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:CreatedOn"),
data = Encoding.UTF8.GetBytes(tweet.CreatedOn.ToString()) });
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:ReplyToId"),
data = Encoding.UTF8.GetBytes(tweet.ReplyToId) });
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:Sentiment"),
data = Encoding.UTF8.GetBytes(tweet.Sentiment.ToString()) });
if (tweet.Coordinates != null)
{
row.values.Add(
new Cell { column = Encoding.UTF8.GetBytes("d:Coordinates"),
data = Encoding.UTF8.GetBytes(tweet.Coordinates) });
}
set.rows.Add(row);
}
public static void CreateSampleTable(string connectionString, string tableName, IEnumerable<IReadOnlyDictionary<string, object>> documents)
{
var client = CreateClient(connectionString);
client.CreateTable(new TableSchema()
{
name = tableName,
columns = { new ColumnSchema() { name = "data" } }
});
foreach (var document in documents)
{
var row = new CellSet.Row
{
key = Encoding.UTF8.GetBytes(document[RowIdPropertyName].ToString())
};
foreach (var property in document)
{
if (property.Key == RowIdPropertyName)
continue;
row.values.Add(
new Cell {
column = Encoding.UTF8.GetBytes("data:" + property.Key),
data = property.Value == null ? null
: Encoding.UTF8.GetBytes(property.Value.ToString())
});
}
client.StoreCells(tableName, new CellSet { rows = { row } });
}
}
private void CreateRowCountCell(CellSet set, long count)
{
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(COUNT_ROW_KEY) };
var value = new Cell
{
column = Encoding.UTF8.GetBytes(COUNT_COLUMN_NAME),
data = Encoding.UTF8.GetBytes(count.ToString())
};
row.values.Add(value);
set.rows.Add(row);
}
public void StoreTestData(IHBaseClient hbaseClient)
{
// we are going to insert the keys 0 to 100 and then do some range queries on that
const string testValue = "the force is strong in this column";
var set = new CellSet();
for (int i = 0; i < 100; i++)
{
var row = new CellSet.Row { key = BitConverter.GetBytes(i) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
set.rows.Add(row);
}
hbaseClient.StoreCellsAsync(testTableName, set).Wait();
}
public void TestCellsMultiVersionGet()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = CreateClient();
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCellsAsync(testTableName, set).Wait();
client.StoreCellsAsync(testTableName, set).Wait();
CellSet cell = client.GetCellsAsync(testTableName, testKey, "d:starwars", "3").Result;
Assert.AreEqual(2, cell.rows[0].values.Count);
}
public void TestCellsDeletion()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = CreateClient();
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCellsAsync(testTableName, set).Wait();
CellSet cell = client.GetCellsAsync(testTableName, testKey).Result;
// make sure the cell is in the table
Assert.AreEqual(Encoding.UTF8.GetString(cell.rows[0].key), testKey);
// delete cell
client.DeleteCellsAsync(testTableName, testKey).Wait();
// get cell again, 404 exception expected
client.GetCellsAsync(testTableName, testKey).Wait();
}
public static CellSet.Row TweetToRow(FSTweet tweet)
{
// Create a row with a key
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) };
// Add columns to the row
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:Text"),
data = Encoding.UTF8.GetBytes(tweet.Text)
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:CreatedOn"),
data = Encoding.UTF8.GetBytes(tweet.CreatedOn.ToString())
});
if (tweet.ReplyToId != null)
{
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:ReplyToId"),
data = Encoding.UTF8.GetBytes(tweet.ReplyToId)
});
}
if (tweet.Coordinates != null)
{
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:Coordinates"),
data = Encoding.UTF8.GetBytes(tweet.Coordinates)
});
}
return row;
}
public static CellSet.Row AnnotatedTweetToRow(FSAnnotatedTweet tweet)
{
// Create a row with a key
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) };
// Add columns to the row
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:TweetId"),
data = Encoding.UTF8.GetBytes(tweet.TweetId)
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:AnnotatedBy"),
data = Encoding.UTF8.GetBytes(tweet.AnnotatedBy)
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:AnnotatedOn"),
data = Encoding.UTF8.GetBytes(tweet.AnnotatedOn.ToString())
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:SentenceType"),
data = Encoding.UTF8.GetBytes(tweet.SentenceType.ToString())
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:Sentiment"),
data = Encoding.UTF8.GetBytes(tweet.Sentiment.ToString())
});
return row;
}
private static void CreateTweetCells(CellSet set, ITweet tweet)
{
var key = tweet.IdStr;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell { column = Encoding.UTF8.GetBytes("d:created_at"), data = Encoding.UTF8.GetBytes(tweet.CreatedAt.ToLongTimeString()) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:text"), data = Encoding.UTF8.GetBytes(tweet.Text) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString();
value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
if (tweet.Place != null)
{
value = new Cell { column = Encoding.UTF8.GetBytes("d:place_fullname"), data = Encoding.UTF8.GetBytes(tweet.Place.FullName) };
row.values.Add(value);
}
set.rows.Add(row);
}
':', ';', '<', '=', '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~' }; //ascii 58--64 + misc.
private void CreateTweetByWordsCells(CellSet set, ITweet tweet)
{
var words = tweet.Text.ToLower().Split(_punctuationChars);
int sentimentScore = CalcSentimentScore(words);
var word_pairs = words.Take(words.Length - 1)
.Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1]));
var all_words = words.Concat(word_pairs).ToList();
foreach (var word in all_words)
{
var time_index = (ulong.MaxValue -
(ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr;
var key = word + "_" + time_index;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell {
column = Encoding.UTF8.GetBytes("d:id_str"),
data = Encoding.UTF8.GetBytes(tweet.IdStr) };
row.values.Add(value);
value = new Cell {
column = Encoding.UTF8.GetBytes("d:lang"),
data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," +
tweet.Coordinates.Latitude.ToString();
value = new Cell {
column = Encoding.UTF8.GetBytes("d:coor"),
data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
value = new Cell {
column = Encoding.UTF8.GetBytes("d:sentiment"),
data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) };
row.values.Add(value);
set.rows.Add(row);
}
}
public static CellSet.Row ConversationToRow(FSConversation convo)
{
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(convo.Id) };
// Add columns to the row
for (int i = 0; i < convo.Tweets.Count(); i++ )
{
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:TweetId_"+i.ToString()),
data = Encoding.UTF8.GetBytes(convo.Tweets.ElementAt(i).Id)
});
}
return row;
}
private void PopulateTable()
{
var client = new HBaseClient(_credentials);
var cellSet = new CellSet();
string id = Guid.NewGuid().ToString("N");
for (int lineNumber = 0; lineNumber < 10; ++lineNumber)
{
string rowKey = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", id, lineNumber);
// add to expected records
var rec = new FilterTestRecord(rowKey, lineNumber, Guid.NewGuid().ToString("N"), Guid.NewGuid().ToString("D"));
_allExpectedRecords.Add(rec);
// add to row
var row = new CellSet.Row { key = _encoding.GetBytes(rec.RowKey) };
var lineColumnValue = new Cell
{
column = BuildCellColumn(ColumnFamilyName1, LineNumberColumnName),
data = BitConverter.GetBytes(rec.LineNumber)
};
row.values.Add(lineColumnValue);
var paragraphColumnValue = new Cell { column = BuildCellColumn(ColumnFamilyName1, ColumnNameA), data = _encoding.GetBytes(rec.A) };
row.values.Add(paragraphColumnValue);
var columnValueB = new Cell { column = BuildCellColumn(ColumnFamilyName2, ColumnNameB), data = Encoding.UTF8.GetBytes(rec.B) };
row.values.Add(columnValueB);
cellSet.rows.Add(row);
}
client.StoreCellsAsync(_tableName, cellSet).Wait();
}
private void CreateTweetByWordsCells(CellSet set, TweetIndexItem indexItem)
{
var word = indexItem.Word;
var time_index = (ulong.MaxValue -
(ulong)indexItem.CreatedAt).ToString().PadLeft(20) + indexItem.IdStr;
var key = word + "_" + time_index;
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
var value = new Cell
{
column = Encoding.UTF8.GetBytes("d:id_str"),
data = Encoding.UTF8.GetBytes(indexItem.IdStr)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:lang"),
data = Encoding.UTF8.GetBytes(indexItem.Language)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:coor"),
data = Encoding.UTF8.GetBytes(indexItem.Coordinates)
};
row.values.Add(value);
value = new Cell
{
column = Encoding.UTF8.GetBytes("d:sentiment"),
data = Encoding.UTF8.GetBytes(indexItem.SentimentScore.ToString())
};
row.values.Add(value);
set.rows.Add(row);
}
private void StoreStatelessScannerTestData(IHBaseClient hBaseClient)
{
const string testValue = "I will fulfill our destiny. ";
var set = new CellSet();
for (int i = 1; i <= 3; i++)
{
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes("testrow" + i) };
row.values.Add(new Cell { column = Encoding.UTF8.GetBytes("a:1"), data = Encoding.UTF8.GetBytes(testValue + "a" + i) });
set.rows.Add(row);
row = new CellSet.Row { key = Encoding.UTF8.GetBytes("testrow" + i) };
row.values.Add(new Cell { column = Encoding.UTF8.GetBytes("b:1"), data = Encoding.UTF8.GetBytes(testValue + "b" + i) });
set.rows.Add(row);
}
hBaseClient.StoreCells(_testTableName, set);
}
/// <summary>
/// Create a write set for HBase when the emit period completes
/// We delay all the writes to batch the aggregations and writes for them
/// </summary>
/// <returns></returns>
public override bool EmitAggregations()
{
try
{
if (emitstopwatch.Elapsed > this.appConfig.AggregationWindow)
{
local_hbasescan_count = 0;
var writeset = new CellSet();
var emitime = DateTime.UtcNow.Floor(this.appConfig.AggregationWindow).Subtract(this.appConfig.EmitWindow);
var keystoremove = new List<DateTime>();
var dtkeys = aggregatedCounts.Keys.Where(dt => dt < emitime).OrderBy(dt => dt).ToList();
var hbaseresultsets = new Dictionary<string, Dictionary<string, Dictionary<string, double>>>();
if (dtkeys.Count > 0)
{
var startdt = dtkeys[0];
var enddt = dtkeys[dtkeys.Count-1];
foreach (var dtkey in dtkeys)
{
foreach (var pkey in aggregatedCounts[dtkey].Keys)
{
if (!this.appConfig.HBaseOverwrite && !hbaseresultsets.ContainsKey(pkey))
{
hbaseresultsets.Add(pkey,
ScanHBase(
pkey + Utilities.KEY_DELIMITER + startdt.ToString(Utilities.DATE_TIME_FORMAT),
pkey + Utilities.KEY_DELIMITER + enddt.ToString(Utilities.DATE_TIME_FORMAT)));
}
var rowkey = pkey + Utilities.KEY_DELIMITER + dtkey.ToString(Utilities.DATE_TIME_FORMAT);
var tablerow = new CellSet.Row { key = Encoding.UTF8.GetBytes(rowkey) };
foreach (var skey in aggregatedCounts[dtkey][pkey].Keys)
{
var columnkey = "v:" + skey;
double previousdata = 0;
if (hbaseresultsets.ContainsKey(pkey) &&
hbaseresultsets[pkey].ContainsKey(rowkey) &&
hbaseresultsets[pkey][rowkey].ContainsKey(columnkey))
{
previousdata = hbaseresultsets[pkey][rowkey][columnkey];
}
var rowcell = new Cell
{
column = Encoding.UTF8.GetBytes(columnkey),
data = BitConverter.GetBytes(previousdata + aggregatedCounts[dtkey][pkey][skey])
};
tablerow.values.Add(rowcell);
global_emit_count++;
last_emit_count++;
current_cache_size--;
}
writeset.rows.Add(tablerow);
}
keystoremove.Add(dtkey);
}
}
if (writeset != null && writeset.rows.Count > 0)
{
Context.Logger.Info("HBaseTableName: {0} - Rows to write: {1}, First Rowkey = {2}",
this.HBaseTableName, writeset.rows.Count, Encoding.UTF8.GetString(writeset.rows[0].key));
var localstopwatch = new Stopwatch();
localstopwatch.Start();
//Use the StoreCells API to write the cellset into HBase Table
HBaseClusterClient.StoreCells(this.HBaseTableName, writeset);
Context.Logger.Info("HBase: Table = {0}, Rows Written = {1}, Write Time = {2} secs, Time since last write = {3} secs",
this.HBaseTableName, writeset.rows.Count, localstopwatch.Elapsed.TotalSeconds, emitstopwatch.Elapsed.TotalSeconds);
foreach (var key in keystoremove)
{
aggregatedCounts.Remove(key);
}
last_emit_in_secs = emitstopwatch.Elapsed.TotalSeconds;
emitstopwatch.Restart();
}
if (!this.appConfig.HBaseOverwrite)
{
Context.Logger.Info("ScanHBase: Last Window Scan Count = {0}, Table = {1}", local_hbasescan_count, this.HBaseTableName);
}
}
if (last_emit_count > 0)
{
return true;
}
else
{
return false;
}
}
catch (Exception ex)
{
last_error_count++;
global_error_count++;
Context.Logger.Error(ex.ToString());
return false;
}
}
// Popular a CellSet object to be written into HBase
private void CreateTweetByWordsCells(CellSet set, ITweet tweet)
{
// Split the Tweet into words
string[] words = tweet.Text.ToLower().Split(_punctuationChars);
// Calculate sentiment score base on the words
int sentimentScore = CalcSentimentScore(words);
var word_pairs = words.Take(words.Length - 1)
.Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1]));
var all_words = words.Concat(word_pairs).ToList();
// For each word in the Tweet add a row to the HBase table
foreach (string word in all_words)
{
string time_index = (ulong.MaxValue - (ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr;
string key = word + "_" + time_index;
// Create a row
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) };
// Add columns to the row, including Tweet identifier, language, coordinator(if available), and sentiment
var value = new Cell { column = Encoding.UTF8.GetBytes("d:id_str"), data = Encoding.UTF8.GetBytes(tweet.IdStr) };
row.values.Add(value);
value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) };
row.values.Add(value);
if (tweet.Coordinates != null)
{
var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString();
value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) };
row.values.Add(value);
}
value = new Cell { column = Encoding.UTF8.GetBytes("d:sentiment"), data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) };
row.values.Add(value);
set.rows.Add(row);
}
}
public async Task When_I_CheckAndDeleteCells_With_TimeStamp_And_Cells_To_Delete_I_Can_add_with_higher_timestamp()
{
var client = GetClient();
client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1A", 10))).Wait();
client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c2", "1A", 10))).Wait();
// Deletes all the ColumnFamily with timestamp less than 10
CellSet.Row rowToDelete = new CellSet.Row() { key = Encoding.UTF8.GetBytes("3") };
//rowToDelete.values.Add(GetCell(rowToDelete.key, column = BuildCellColumn(ColumnFamilyName1, "c1"), data= "1A", timestamp = 10 });
rowToDelete.values.Add(GetCell("3", "c1", "1A", 10));
rowToDelete.values.Add(GetCell("3", "c2", "1A", 10));
bool deleted = await client.CheckAndDeleteAsync(_tableName, GetCell("3", "c1", "1A", 10), rowToDelete);
deleted.ShouldEqual(true);
CellSet retrievedCells;
try
{
// All cells are deleted so this should fail
retrievedCells = client.GetCellsAsync(_tableName, "3").Result;
throw new AssertFailedException("expecting Get '3' to fail as all cells are removed");
}
catch (Exception ex)
{
if (ex is AggregateException)
{
((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound);
}
else
{
throw ex;
}
}
client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1B", 11))).Wait();
try
{
retrievedCells = client.GetCellsAsync(_tableName, "3").Result;
retrievedCells.rows[0].values.Count.ShouldEqual(1);
Encoding.UTF8.GetString(retrievedCells.rows[0].values[0].column).ShouldBeEqualOrdinalIgnoreCase("c1");
}
catch (Exception ex)
{
if (ex is AggregateException)
{
((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound);
}
}
}
public CellSet.Row ToRowOfCellSet(BsonDocument pv)
{
CellSet.Row cellSetRow = new CellSet.Row() { key = Encoding.UTF8.GetBytes(pv.GetValue(ROW_KEY).AsString) };
try
{
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DMAC.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(DMAC).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + MAC.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(MAC, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + IP.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(IP).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DATETIME_POINT.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(DATETIME_POINT, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + CLIENT_AGENT.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(CLIENT_AGENT, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_METHOD.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_METHOD).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_URI.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_URI).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_VERSION.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_VERSION).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_HOST.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_HOST, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DAY_ID.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(DAY_ID).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + REFERER.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(REFERER).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + CLIENT_OS.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(CLIENT_OS).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + MOBILE_BRAND.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(MOBILE_BRAND).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + CLIENT_BROWSER.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(CLIENT_BROWSER).AsString)});
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + IN_DB_DATETIME.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(IN_DB_DATETIME).AsInt64 + "") });
}
catch (Exception e)
{
Trace.TraceError(" pv covert to hbase model failed:" + pv + "\r\n" + e.Message + "\n" + e.StackTrace);
CommonUtil.LogException(pv.GetValue(DMAC).AsString, e);
return null;
}
return cellSetRow;
}
public CellSet.Row ToRowOfCellSet(BsonDocument uv)
{
CellSet.Row cellSetRow = new CellSet.Row() { key = Encoding.UTF8.GetBytes(uv.GetValue(ROW_KEY).AsString) };
try
{
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DMAC.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(DMAC).AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + MAC.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(MAC, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + IP.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(IP, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + STARTTIME.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(STARTTIME, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + ENDTIME.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(ENDTIME, "").AsString) });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + WIFIUP.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue(WIFIUP).ToString()) + "") });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + WIFIDOWN.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue(WIFIDOWN).ToString()) + "") });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + UP3G.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue("3Gup").ToString()) + "") });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DOWN3G.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue("3Gdown").ToString()) + "") });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DAY_ID.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(DAY_ID).AsInt32 + "") });
cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + INDB_DATETIME.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(INDB_DATETIME).AsInt64 + "") });
}
catch (Exception e)
{
Trace.TraceError("uv covert to hbase model failed:" + uv + "\r\n" + e.Message + "\n" + e.StackTrace);
CommonUtil.LogException(uv.GetValue(DMAC).AsString, e);
return null;
}
return cellSetRow;
}
public async Task When_I_CheckAndDeleteCells_With_ColumnFamily_Deletes_All_cells()
{
var client = GetClient();
client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1A", 10))).Wait();
client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c2", "1A", 10))).Wait();
// Deletes all the ColumnFamily with timestamp less than 10
CellSet.Row rowToDelete = new CellSet.Row() { key = Encoding.UTF8.GetBytes("3") };
rowToDelete.values.Add(new Cell() { row = rowToDelete.key, column = Encoding.UTF8.GetBytes(ColumnFamilyName1), timestamp = 10 });
bool deleted = await client.CheckAndDeleteAsync(_tableName, GetCell("3", "c1", "1A", 10), rowToDelete);
deleted.ShouldEqual(true);
CellSet retrievedCells;
try
{
// All cells are deleted so this should fail
retrievedCells = client.GetCellsAsync(_tableName, "3").Result;
throw new AssertFailedException("expecting Get '3' to fail as all cells are removed");
}
catch (Exception ex)
{
if (ex is AggregateException)
{
((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound);
}
}
client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1B", 11))).Wait();
try
{
retrievedCells = client.GetCellsAsync(_tableName, "3").Result;
retrievedCells.rows[0].values.Count.ShouldEqual(1);
}
catch (Exception ex)
{
if (ex is AggregateException)
{
((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound);
}
}
}
/// <summary>
/// Create a write set for HBase based on cached tuples
/// We delay all the writes to batch the aggregations and writes for them
/// </summary>
/// <returns></returns>
public void WriteToHBase()
{
Context.Logger.Info("WriteToHBase - Start - Writing cached rows into HBase. Rows to write: {0}", cachedTuples.Count);
if (cachedTuples.Count > 0)
{
var writeSet = new CellSet();
foreach (var cachedTuple in cachedTuples)
{
var values = cachedTuple.Value.GetValues();
if(this.HBaseTableColumns.Count < (values.Count - 1))
{
throw new Exception(String.Format(
"Count of HBaseTableColumns is less than fields received. HBaseTableColumns.Count: {0}, Values.Count (without rowkey): {1}",
this.HBaseTableColumns.Count, values.Count - 1)
);
}
//Use the first value as rowkey and add the remaining as a list
var tablerow = new CellSet.Row { key = ToBytes(values[0]) };
//Skip the first value and read the remaining
for (int i = 1; i < values.Count; i++)
{
var rowcell = new Cell
{
//Based on our assumption that ColumnNames do NOT contain rowkey field
column = ToBytes(this.HBaseTableColumnFamily + ":" + this.HBaseTableColumns[i-1]),
data = ToBytes(values[i])
};
tablerow.values.Add(rowcell);
}
writeSet.rows.Add(tablerow);
}
try
{
//Use the StoreCells API to write the cellset into HBase Table
HBaseClusterClient.StoreCells(this.HBaseTableName, writeSet);
}
catch
{
Context.Logger.Error("HBase StoreCells Failed");
foreach(var row in writeSet.rows)
{
Context.Logger.Info("Failed RowKey: {0}, Values (bytes): {1}", Encoding.UTF8.GetString(row.key),
String.Join(", ", row.values.Select(v => Encoding.UTF8.GetString(v.column) + " = " + v.data.LongLength)));
}
throw;
}
Context.Logger.Info("WriteToHBase - End - Stored cells into HBase. Rows written: {0}", writeSet.rows.Count);
}
else
{
Context.Logger.Info("WriteToHBase - End - No cells to write.");
}
}
private CellSet.Row GetCellSet(string key, string columnName, string value, long timestamp)
{
CellSet.Row row = new CellSet.Row() { key = Encoding.UTF8.GetBytes(key) };
Cell c1 = new Cell() { column = BuildCellColumn(ColumnFamilyName1, columnName ), row = row.key };
if (value != null)
{
c1.data = Encoding.UTF8.GetBytes(value);
}
if (timestamp > 0)
{
c1.timestamp = timestamp;
}
row.values.Add(c1);
return row;
}
public void TestStoreSingleCell()
{
const string testKey = "content";
const string testValue = "the force is strong in this column";
var client = new HBaseClient(_credentials);
var set = new CellSet();
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) };
set.rows.Add(row);
var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) };
row.values.Add(value);
client.StoreCells(_testTableName, set);
CellSet cells = client.GetCells(_testTableName, testKey);
Assert.AreEqual(1, cells.rows.Count);
Assert.AreEqual(1, cells.rows[0].values.Count);
Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data));
}
public static CellSet.Row WordRelationToRow(FSWordRelationship relation)
{
var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(relation.Id) };
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:WordOne" ),
data = Encoding.UTF8.GetBytes(relation.WordOne)
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:WordOneId"),
data = Encoding.UTF8.GetBytes(relation.WordOneId.ToString())
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:WordTwo"),
data = Encoding.UTF8.GetBytes(relation.WordTwo)
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:WordTwoId"),
data = Encoding.UTF8.GetBytes(relation.WordTwoId.ToString())
});
row.values.Add(
new Cell
{
column = Encoding.UTF8.GetBytes("d:RScore"),
data = Encoding.UTF8.GetBytes(relation.rScore.ToString())
});
return row;
}