public void SaveMyDto(Dto dto) { CellSet cellSet = new CellSet(); CellSet.Row cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(_sampleRowKey) }; cellSet.rows.Add(cellSetRow); Cell value1 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field1"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field1)) }; Cell value2 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field2"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field2)) }; Cell value3 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field3"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field3)) }; Cell value4 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field4"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field4)) }; Cell value5 = new Cell { column = Encoding.UTF8.GetBytes("CF1:field5"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.Field5)) }; // Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(dto.NestedData))}; byte[] nestedDataBlob; using (var memoryStream = new MemoryStream()) { using (var writer = new BsonWriter(memoryStream)) { _serializer.Serialize(writer, dto.NestedData); nestedDataBlob = memoryStream.ToArray(); } } Cell value6 = new Cell { column = Encoding.UTF8.GetBytes("CF1:NestedData"), data = nestedDataBlob }; cellSetRow.values.AddRange(new List<Cell>() { value1, value2, value3, value4, value5, value6 }); _client.StoreCells(_sampleTableName, cellSet); }
public CellSet GetCells(ItemDto item, string culture) { var cellSetRow = new CellSet.Row { key = Encoding.UTF8.GetBytes(item.Code) }; var cells = new List<Cell>(); cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Status"), data = Encoding.UTF8.GetBytes(item.Status) }); cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:MH" + item.MerchandiseHierarchy.Code), data = _serializer.SerializeToBson(item.MerchandiseHierarchy) }); cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:CreatedDate"), data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()) }); cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:UpdatedDate"), data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()) }); cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Long_" + culture), data = Encoding.UTF8.GetBytes(item.Description) }); cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:Short_" + culture), data = Encoding.UTF8.GetBytes(item.ShortDescription) }); foreach (var group in item.Groups) cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:G" + group.Code), data = _serializer.SerializeToBson(group) }); foreach (var identifier in item.Identifiers) cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:I" + identifier.Type), data = Encoding.UTF8.GetBytes(identifier.Value) }); foreach (var attributes in item.Attributes) cells.Add(new Cell { column = Encoding.UTF8.GetBytes("CF1:A" + culture + "_" + attributes.Id), data = _serializer.SerializeToBson(attributes.Value) }); cellSetRow.values.AddRange(cells); var cellSet = new CellSet(); cellSet.rows.Add(cellSetRow); return cellSet; }
// Popular a CellSet object to be written into HBase private void CreateTweetByWordsCells(CellSet set, TweetSentimentData tweet) { // Create a row with a key var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) }; // Add columns to the row row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:Text"), data = Encoding.UTF8.GetBytes(tweet.Text) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:CreatedOn"), data = Encoding.UTF8.GetBytes(tweet.CreatedOn.ToString()) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:ReplyToId"), data = Encoding.UTF8.GetBytes(tweet.ReplyToId) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:Sentiment"), data = Encoding.UTF8.GetBytes(tweet.Sentiment.ToString()) }); if (tweet.Coordinates != null) { row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:Coordinates"), data = Encoding.UTF8.GetBytes(tweet.Coordinates) }); } set.rows.Add(row); }
public static void CreateSampleTable(string connectionString, string tableName, IEnumerable<IReadOnlyDictionary<string, object>> documents) { var client = CreateClient(connectionString); client.CreateTable(new TableSchema() { name = tableName, columns = { new ColumnSchema() { name = "data" } } }); foreach (var document in documents) { var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(document[RowIdPropertyName].ToString()) }; foreach (var property in document) { if (property.Key == RowIdPropertyName) continue; row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("data:" + property.Key), data = property.Value == null ? null : Encoding.UTF8.GetBytes(property.Value.ToString()) }); } client.StoreCells(tableName, new CellSet { rows = { row } }); } }
private void CreateRowCountCell(CellSet set, long count) { var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(COUNT_ROW_KEY) }; var value = new Cell { column = Encoding.UTF8.GetBytes(COUNT_COLUMN_NAME), data = Encoding.UTF8.GetBytes(count.ToString()) }; row.values.Add(value); set.rows.Add(row); }
public void StoreTestData(IHBaseClient hbaseClient) { // we are going to insert the keys 0 to 100 and then do some range queries on that const string testValue = "the force is strong in this column"; var set = new CellSet(); for (int i = 0; i < 100; i++) { var row = new CellSet.Row { key = BitConverter.GetBytes(i) }; var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); set.rows.Add(row); } hbaseClient.StoreCellsAsync(testTableName, set).Wait(); }
public void TestCellsMultiVersionGet() { const string testKey = "content"; const string testValue = "the force is strong in this column"; var client = CreateClient(); var set = new CellSet(); var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) }; set.rows.Add(row); var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); client.StoreCellsAsync(testTableName, set).Wait(); client.StoreCellsAsync(testTableName, set).Wait(); CellSet cell = client.GetCellsAsync(testTableName, testKey, "d:starwars", "3").Result; Assert.AreEqual(2, cell.rows[0].values.Count); }
public void TestCellsDeletion() { const string testKey = "content"; const string testValue = "the force is strong in this column"; var client = CreateClient(); var set = new CellSet(); var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) }; set.rows.Add(row); var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); client.StoreCellsAsync(testTableName, set).Wait(); CellSet cell = client.GetCellsAsync(testTableName, testKey).Result; // make sure the cell is in the table Assert.AreEqual(Encoding.UTF8.GetString(cell.rows[0].key), testKey); // delete cell client.DeleteCellsAsync(testTableName, testKey).Wait(); // get cell again, 404 exception expected client.GetCellsAsync(testTableName, testKey).Wait(); }
public static CellSet.Row TweetToRow(FSTweet tweet) { // Create a row with a key var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) }; // Add columns to the row row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:Text"), data = Encoding.UTF8.GetBytes(tweet.Text) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:CreatedOn"), data = Encoding.UTF8.GetBytes(tweet.CreatedOn.ToString()) }); if (tweet.ReplyToId != null) { row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:ReplyToId"), data = Encoding.UTF8.GetBytes(tweet.ReplyToId) }); } if (tweet.Coordinates != null) { row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:Coordinates"), data = Encoding.UTF8.GetBytes(tweet.Coordinates) }); } return row; }
public static CellSet.Row AnnotatedTweetToRow(FSAnnotatedTweet tweet) { // Create a row with a key var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(tweet.Id) }; // Add columns to the row row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:TweetId"), data = Encoding.UTF8.GetBytes(tweet.TweetId) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:AnnotatedBy"), data = Encoding.UTF8.GetBytes(tweet.AnnotatedBy) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:AnnotatedOn"), data = Encoding.UTF8.GetBytes(tweet.AnnotatedOn.ToString()) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:SentenceType"), data = Encoding.UTF8.GetBytes(tweet.SentenceType.ToString()) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:Sentiment"), data = Encoding.UTF8.GetBytes(tweet.Sentiment.ToString()) }); return row; }
private static void CreateTweetCells(CellSet set, ITweet tweet) { var key = tweet.IdStr; var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) }; var value = new Cell { column = Encoding.UTF8.GetBytes("d:created_at"), data = Encoding.UTF8.GetBytes(tweet.CreatedAt.ToLongTimeString()) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:text"), data = Encoding.UTF8.GetBytes(tweet.Text) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) }; row.values.Add(value); if (tweet.Coordinates != null) { var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString(); value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) }; row.values.Add(value); } if (tweet.Place != null) { value = new Cell { column = Encoding.UTF8.GetBytes("d:place_fullname"), data = Encoding.UTF8.GetBytes(tweet.Place.FullName) }; row.values.Add(value); } set.rows.Add(row); }
':', ';', '<', '=', '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~' }; //ascii 58--64 + misc. private void CreateTweetByWordsCells(CellSet set, ITweet tweet) { var words = tweet.Text.ToLower().Split(_punctuationChars); int sentimentScore = CalcSentimentScore(words); var word_pairs = words.Take(words.Length - 1) .Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1])); var all_words = words.Concat(word_pairs).ToList(); foreach (var word in all_words) { var time_index = (ulong.MaxValue - (ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr; var key = word + "_" + time_index; var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) }; var value = new Cell { column = Encoding.UTF8.GetBytes("d:id_str"), data = Encoding.UTF8.GetBytes(tweet.IdStr) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) }; row.values.Add(value); if (tweet.Coordinates != null) { var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString(); value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) }; row.values.Add(value); } value = new Cell { column = Encoding.UTF8.GetBytes("d:sentiment"), data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) }; row.values.Add(value); set.rows.Add(row); } }
public static CellSet.Row ConversationToRow(FSConversation convo) { var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(convo.Id) }; // Add columns to the row for (int i = 0; i < convo.Tweets.Count(); i++ ) { row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:TweetId_"+i.ToString()), data = Encoding.UTF8.GetBytes(convo.Tweets.ElementAt(i).Id) }); } return row; }
private void PopulateTable() { var client = new HBaseClient(_credentials); var cellSet = new CellSet(); string id = Guid.NewGuid().ToString("N"); for (int lineNumber = 0; lineNumber < 10; ++lineNumber) { string rowKey = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", id, lineNumber); // add to expected records var rec = new FilterTestRecord(rowKey, lineNumber, Guid.NewGuid().ToString("N"), Guid.NewGuid().ToString("D")); _allExpectedRecords.Add(rec); // add to row var row = new CellSet.Row { key = _encoding.GetBytes(rec.RowKey) }; var lineColumnValue = new Cell { column = BuildCellColumn(ColumnFamilyName1, LineNumberColumnName), data = BitConverter.GetBytes(rec.LineNumber) }; row.values.Add(lineColumnValue); var paragraphColumnValue = new Cell { column = BuildCellColumn(ColumnFamilyName1, ColumnNameA), data = _encoding.GetBytes(rec.A) }; row.values.Add(paragraphColumnValue); var columnValueB = new Cell { column = BuildCellColumn(ColumnFamilyName2, ColumnNameB), data = Encoding.UTF8.GetBytes(rec.B) }; row.values.Add(columnValueB); cellSet.rows.Add(row); } client.StoreCellsAsync(_tableName, cellSet).Wait(); }
private void CreateTweetByWordsCells(CellSet set, TweetIndexItem indexItem) { var word = indexItem.Word; var time_index = (ulong.MaxValue - (ulong)indexItem.CreatedAt).ToString().PadLeft(20) + indexItem.IdStr; var key = word + "_" + time_index; var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) }; var value = new Cell { column = Encoding.UTF8.GetBytes("d:id_str"), data = Encoding.UTF8.GetBytes(indexItem.IdStr) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(indexItem.Language) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(indexItem.Coordinates) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:sentiment"), data = Encoding.UTF8.GetBytes(indexItem.SentimentScore.ToString()) }; row.values.Add(value); set.rows.Add(row); }
private void StoreStatelessScannerTestData(IHBaseClient hBaseClient) { const string testValue = "I will fulfill our destiny. "; var set = new CellSet(); for (int i = 1; i <= 3; i++) { var row = new CellSet.Row { key = Encoding.UTF8.GetBytes("testrow" + i) }; row.values.Add(new Cell { column = Encoding.UTF8.GetBytes("a:1"), data = Encoding.UTF8.GetBytes(testValue + "a" + i) }); set.rows.Add(row); row = new CellSet.Row { key = Encoding.UTF8.GetBytes("testrow" + i) }; row.values.Add(new Cell { column = Encoding.UTF8.GetBytes("b:1"), data = Encoding.UTF8.GetBytes(testValue + "b" + i) }); set.rows.Add(row); } hBaseClient.StoreCells(_testTableName, set); }
/// <summary> /// Create a write set for HBase when the emit period completes /// We delay all the writes to batch the aggregations and writes for them /// </summary> /// <returns></returns> public override bool EmitAggregations() { try { if (emitstopwatch.Elapsed > this.appConfig.AggregationWindow) { local_hbasescan_count = 0; var writeset = new CellSet(); var emitime = DateTime.UtcNow.Floor(this.appConfig.AggregationWindow).Subtract(this.appConfig.EmitWindow); var keystoremove = new List<DateTime>(); var dtkeys = aggregatedCounts.Keys.Where(dt => dt < emitime).OrderBy(dt => dt).ToList(); var hbaseresultsets = new Dictionary<string, Dictionary<string, Dictionary<string, double>>>(); if (dtkeys.Count > 0) { var startdt = dtkeys[0]; var enddt = dtkeys[dtkeys.Count-1]; foreach (var dtkey in dtkeys) { foreach (var pkey in aggregatedCounts[dtkey].Keys) { if (!this.appConfig.HBaseOverwrite && !hbaseresultsets.ContainsKey(pkey)) { hbaseresultsets.Add(pkey, ScanHBase( pkey + Utilities.KEY_DELIMITER + startdt.ToString(Utilities.DATE_TIME_FORMAT), pkey + Utilities.KEY_DELIMITER + enddt.ToString(Utilities.DATE_TIME_FORMAT))); } var rowkey = pkey + Utilities.KEY_DELIMITER + dtkey.ToString(Utilities.DATE_TIME_FORMAT); var tablerow = new CellSet.Row { key = Encoding.UTF8.GetBytes(rowkey) }; foreach (var skey in aggregatedCounts[dtkey][pkey].Keys) { var columnkey = "v:" + skey; double previousdata = 0; if (hbaseresultsets.ContainsKey(pkey) && hbaseresultsets[pkey].ContainsKey(rowkey) && hbaseresultsets[pkey][rowkey].ContainsKey(columnkey)) { previousdata = hbaseresultsets[pkey][rowkey][columnkey]; } var rowcell = new Cell { column = Encoding.UTF8.GetBytes(columnkey), data = BitConverter.GetBytes(previousdata + aggregatedCounts[dtkey][pkey][skey]) }; tablerow.values.Add(rowcell); global_emit_count++; last_emit_count++; current_cache_size--; } writeset.rows.Add(tablerow); } keystoremove.Add(dtkey); } } if (writeset != null && writeset.rows.Count > 0) { Context.Logger.Info("HBaseTableName: {0} - Rows to write: {1}, First Rowkey = {2}", this.HBaseTableName, writeset.rows.Count, Encoding.UTF8.GetString(writeset.rows[0].key)); var localstopwatch = new Stopwatch(); localstopwatch.Start(); //Use the StoreCells API to write the cellset into HBase Table HBaseClusterClient.StoreCells(this.HBaseTableName, writeset); Context.Logger.Info("HBase: Table = {0}, Rows Written = {1}, Write Time = {2} secs, Time since last write = {3} secs", this.HBaseTableName, writeset.rows.Count, localstopwatch.Elapsed.TotalSeconds, emitstopwatch.Elapsed.TotalSeconds); foreach (var key in keystoremove) { aggregatedCounts.Remove(key); } last_emit_in_secs = emitstopwatch.Elapsed.TotalSeconds; emitstopwatch.Restart(); } if (!this.appConfig.HBaseOverwrite) { Context.Logger.Info("ScanHBase: Last Window Scan Count = {0}, Table = {1}", local_hbasescan_count, this.HBaseTableName); } } if (last_emit_count > 0) { return true; } else { return false; } } catch (Exception ex) { last_error_count++; global_error_count++; Context.Logger.Error(ex.ToString()); return false; } }
// Popular a CellSet object to be written into HBase private void CreateTweetByWordsCells(CellSet set, ITweet tweet) { // Split the Tweet into words string[] words = tweet.Text.ToLower().Split(_punctuationChars); // Calculate sentiment score base on the words int sentimentScore = CalcSentimentScore(words); var word_pairs = words.Take(words.Length - 1) .Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1])); var all_words = words.Concat(word_pairs).ToList(); // For each word in the Tweet add a row to the HBase table foreach (string word in all_words) { string time_index = (ulong.MaxValue - (ulong)tweet.CreatedAt.ToBinary()).ToString().PadLeft(20) + tweet.IdStr; string key = word + "_" + time_index; // Create a row var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(key) }; // Add columns to the row, including Tweet identifier, language, coordinator(if available), and sentiment var value = new Cell { column = Encoding.UTF8.GetBytes("d:id_str"), data = Encoding.UTF8.GetBytes(tweet.IdStr) }; row.values.Add(value); value = new Cell { column = Encoding.UTF8.GetBytes("d:lang"), data = Encoding.UTF8.GetBytes(tweet.Language.ToString()) }; row.values.Add(value); if (tweet.Coordinates != null) { var str = tweet.Coordinates.Longitude.ToString() + "," + tweet.Coordinates.Latitude.ToString(); value = new Cell { column = Encoding.UTF8.GetBytes("d:coor"), data = Encoding.UTF8.GetBytes(str) }; row.values.Add(value); } value = new Cell { column = Encoding.UTF8.GetBytes("d:sentiment"), data = Encoding.UTF8.GetBytes(sentimentScore.ToString()) }; row.values.Add(value); set.rows.Add(row); } }
public async Task When_I_CheckAndDeleteCells_With_TimeStamp_And_Cells_To_Delete_I_Can_add_with_higher_timestamp() { var client = GetClient(); client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1A", 10))).Wait(); client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c2", "1A", 10))).Wait(); // Deletes all the ColumnFamily with timestamp less than 10 CellSet.Row rowToDelete = new CellSet.Row() { key = Encoding.UTF8.GetBytes("3") }; //rowToDelete.values.Add(GetCell(rowToDelete.key, column = BuildCellColumn(ColumnFamilyName1, "c1"), data= "1A", timestamp = 10 }); rowToDelete.values.Add(GetCell("3", "c1", "1A", 10)); rowToDelete.values.Add(GetCell("3", "c2", "1A", 10)); bool deleted = await client.CheckAndDeleteAsync(_tableName, GetCell("3", "c1", "1A", 10), rowToDelete); deleted.ShouldEqual(true); CellSet retrievedCells; try { // All cells are deleted so this should fail retrievedCells = client.GetCellsAsync(_tableName, "3").Result; throw new AssertFailedException("expecting Get '3' to fail as all cells are removed"); } catch (Exception ex) { if (ex is AggregateException) { ((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound); } else { throw ex; } } client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1B", 11))).Wait(); try { retrievedCells = client.GetCellsAsync(_tableName, "3").Result; retrievedCells.rows[0].values.Count.ShouldEqual(1); Encoding.UTF8.GetString(retrievedCells.rows[0].values[0].column).ShouldBeEqualOrdinalIgnoreCase("c1"); } catch (Exception ex) { if (ex is AggregateException) { ((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound); } } }
public CellSet.Row ToRowOfCellSet(BsonDocument pv) { CellSet.Row cellSetRow = new CellSet.Row() { key = Encoding.UTF8.GetBytes(pv.GetValue(ROW_KEY).AsString) }; try { cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DMAC.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(DMAC).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + MAC.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(MAC, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + IP.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(IP).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DATETIME_POINT.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(DATETIME_POINT, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + CLIENT_AGENT.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(CLIENT_AGENT, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_METHOD.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_METHOD).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_URI.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_URI).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_VERSION.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_VERSION).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + HTTP_HOST.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(HTTP_HOST, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DAY_ID.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(DAY_ID).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + REFERER.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(REFERER).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + CLIENT_OS.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(CLIENT_OS).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + MOBILE_BRAND.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(MOBILE_BRAND).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + CLIENT_BROWSER.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(CLIENT_BROWSER).AsString)}); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + IN_DB_DATETIME.ToUpper()), data = Encoding.UTF8.GetBytes(pv.GetValue(IN_DB_DATETIME).AsInt64 + "") }); } catch (Exception e) { Trace.TraceError(" pv covert to hbase model failed:" + pv + "\r\n" + e.Message + "\n" + e.StackTrace); CommonUtil.LogException(pv.GetValue(DMAC).AsString, e); return null; } return cellSetRow; }
public CellSet.Row ToRowOfCellSet(BsonDocument uv) { CellSet.Row cellSetRow = new CellSet.Row() { key = Encoding.UTF8.GetBytes(uv.GetValue(ROW_KEY).AsString) }; try { cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DMAC.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(DMAC).AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + MAC.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(MAC, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + IP.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(IP, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + STARTTIME.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(STARTTIME, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + ENDTIME.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(ENDTIME, "").AsString) }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + WIFIUP.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue(WIFIUP).ToString()) + "") }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + WIFIDOWN.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue(WIFIDOWN).ToString()) + "") }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + UP3G.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue("3Gup").ToString()) + "") }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DOWN3G.ToUpper()), data = Encoding.UTF8.GetBytes(Convert.ToInt64(uv.GetValue("3Gdown").ToString()) + "") }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + DAY_ID.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(DAY_ID).AsInt32 + "") }); cellSetRow.values.Add(new Cell { column = Encoding.UTF8.GetBytes(COLUMN_FAMILY + ":" + INDB_DATETIME.ToUpper()), data = Encoding.UTF8.GetBytes(uv.GetValue(INDB_DATETIME).AsInt64 + "") }); } catch (Exception e) { Trace.TraceError("uv covert to hbase model failed:" + uv + "\r\n" + e.Message + "\n" + e.StackTrace); CommonUtil.LogException(uv.GetValue(DMAC).AsString, e); return null; } return cellSetRow; }
public async Task When_I_CheckAndDeleteCells_With_ColumnFamily_Deletes_All_cells() { var client = GetClient(); client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1A", 10))).Wait(); client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c2", "1A", 10))).Wait(); // Deletes all the ColumnFamily with timestamp less than 10 CellSet.Row rowToDelete = new CellSet.Row() { key = Encoding.UTF8.GetBytes("3") }; rowToDelete.values.Add(new Cell() { row = rowToDelete.key, column = Encoding.UTF8.GetBytes(ColumnFamilyName1), timestamp = 10 }); bool deleted = await client.CheckAndDeleteAsync(_tableName, GetCell("3", "c1", "1A", 10), rowToDelete); deleted.ShouldEqual(true); CellSet retrievedCells; try { // All cells are deleted so this should fail retrievedCells = client.GetCellsAsync(_tableName, "3").Result; throw new AssertFailedException("expecting Get '3' to fail as all cells are removed"); } catch (Exception ex) { if (ex is AggregateException) { ((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound); } } client.StoreCellsAsync(_tableName, CreateCellSet(GetCellSet("3", "c1", "1B", 11))).Wait(); try { retrievedCells = client.GetCellsAsync(_tableName, "3").Result; retrievedCells.rows[0].values.Count.ShouldEqual(1); } catch (Exception ex) { if (ex is AggregateException) { ((ex.InnerException as WebException).Response as HttpWebResponse).StatusCode.ShouldEqual(HttpStatusCode.NotFound); } } }
/// <summary> /// Create a write set for HBase based on cached tuples /// We delay all the writes to batch the aggregations and writes for them /// </summary> /// <returns></returns> public void WriteToHBase() { Context.Logger.Info("WriteToHBase - Start - Writing cached rows into HBase. Rows to write: {0}", cachedTuples.Count); if (cachedTuples.Count > 0) { var writeSet = new CellSet(); foreach (var cachedTuple in cachedTuples) { var values = cachedTuple.Value.GetValues(); if(this.HBaseTableColumns.Count < (values.Count - 1)) { throw new Exception(String.Format( "Count of HBaseTableColumns is less than fields received. HBaseTableColumns.Count: {0}, Values.Count (without rowkey): {1}", this.HBaseTableColumns.Count, values.Count - 1) ); } //Use the first value as rowkey and add the remaining as a list var tablerow = new CellSet.Row { key = ToBytes(values[0]) }; //Skip the first value and read the remaining for (int i = 1; i < values.Count; i++) { var rowcell = new Cell { //Based on our assumption that ColumnNames do NOT contain rowkey field column = ToBytes(this.HBaseTableColumnFamily + ":" + this.HBaseTableColumns[i-1]), data = ToBytes(values[i]) }; tablerow.values.Add(rowcell); } writeSet.rows.Add(tablerow); } try { //Use the StoreCells API to write the cellset into HBase Table HBaseClusterClient.StoreCells(this.HBaseTableName, writeSet); } catch { Context.Logger.Error("HBase StoreCells Failed"); foreach(var row in writeSet.rows) { Context.Logger.Info("Failed RowKey: {0}, Values (bytes): {1}", Encoding.UTF8.GetString(row.key), String.Join(", ", row.values.Select(v => Encoding.UTF8.GetString(v.column) + " = " + v.data.LongLength))); } throw; } Context.Logger.Info("WriteToHBase - End - Stored cells into HBase. Rows written: {0}", writeSet.rows.Count); } else { Context.Logger.Info("WriteToHBase - End - No cells to write."); } }
private CellSet.Row GetCellSet(string key, string columnName, string value, long timestamp) { CellSet.Row row = new CellSet.Row() { key = Encoding.UTF8.GetBytes(key) }; Cell c1 = new Cell() { column = BuildCellColumn(ColumnFamilyName1, columnName ), row = row.key }; if (value != null) { c1.data = Encoding.UTF8.GetBytes(value); } if (timestamp > 0) { c1.timestamp = timestamp; } row.values.Add(c1); return row; }
public void TestStoreSingleCell() { const string testKey = "content"; const string testValue = "the force is strong in this column"; var client = new HBaseClient(_credentials); var set = new CellSet(); var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(testKey) }; set.rows.Add(row); var value = new Cell { column = Encoding.UTF8.GetBytes("d:starwars"), data = Encoding.UTF8.GetBytes(testValue) }; row.values.Add(value); client.StoreCells(_testTableName, set); CellSet cells = client.GetCells(_testTableName, testKey); Assert.AreEqual(1, cells.rows.Count); Assert.AreEqual(1, cells.rows[0].values.Count); Assert.AreEqual(testValue, Encoding.UTF8.GetString(cells.rows[0].values[0].data)); }
public static CellSet.Row WordRelationToRow(FSWordRelationship relation) { var row = new CellSet.Row { key = Encoding.UTF8.GetBytes(relation.Id) }; row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:WordOne" ), data = Encoding.UTF8.GetBytes(relation.WordOne) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:WordOneId"), data = Encoding.UTF8.GetBytes(relation.WordOneId.ToString()) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:WordTwo"), data = Encoding.UTF8.GetBytes(relation.WordTwo) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:WordTwoId"), data = Encoding.UTF8.GetBytes(relation.WordTwoId.ToString()) }); row.values.Add( new Cell { column = Encoding.UTF8.GetBytes("d:RScore"), data = Encoding.UTF8.GetBytes(relation.rScore.ToString()) }); return row; }