/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("updating database" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
db.insertValue(CurrentTimeMillis(), partialCount);
partialCount = 0L;
if (enableAck)
{
Context.Logger.Info("tuplesToAck: " + tuplesToAck);
foreach (var tupleToAck in tuplesToAck)
{
this.ctx.Ack(tupleToAck);
}
tuplesToAck.Clear();
}
}
}
else
{
//Merge partialCount from all PartialCountBolt tasks
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
//Do no ack here but add to the acking queue
if (enableAck)
{
tuplesToAck.Enqueue(tuple);
}
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting totalCount" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
//emit with anchors set the tuples in this batch
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(CurrentTimeMillis(), totalCount));
Context.Logger.Info("acking the batch: " + tuplesToAck.Count);
foreach (var t in tuplesToAck)
{
this.ctx.Ack(t);
}
//once all the tuples are acked, clear the batch
tuplesToAck.Clear();
partialCount = 0L;
}
}
else
{
//Merge partialCount from all PartialCountBolt tasks
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
//Do no ack here but add to the acking queue
tuplesToAck.Enqueue(tuple);
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting partialCount: " + partialCount +
", totalCount: " + totalCount);
//emit with anchors set the tuples in this batch
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount));
//ideally in the logs partialCount and the batch count will match
Context.Logger.Info("acking the batch: " + tuplesToAck.Count);
foreach (var t in tuplesToAck)
{
this.ctx.Ack(t);
}
//once all the tuples are acked, clear the batch
tuplesToAck.Clear();
partialCount = 0L;
}
}
else
{
partialCount++;
totalCount++;
//Do no ack here but add to the acking queue
tuplesToAck.Enqueue(tuple);
}
}
public void Execute(SCPTuple tuple)
{
Context.Logger.Info("Execute enter");
if (Constants.SYSTEM_TICK_STREAM_ID.Equals(tuple.GetSourceStreamId()))
{
long data = tuple.GetLong(0);
Context.Logger.Info("tick tuple, value: {0}", data);
}
else
{
byte[] data = tuple.GetBinary(0);
int bytesNum = data.Count();
if (enableAck)
{
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple> {
tuple
}, new Values(bytesNum));
this.ctx.Ack(tuple);
Context.Logger.Info("emit bytesNum: {0}", bytesNum);
Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId());
}
else
{
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(bytesNum));
Context.Logger.Info("emit bytesNum: {0}", bytesNum);
}
}
Context.Logger.Info("Execute exit");
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("updating database" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
db.insertValue(CurrentTimeMillis(), partialCount);
partialCount = 0L;
if (enableAck)
{
Context.Logger.Info("tuplesToAck: " + tuplesToAck);
foreach (var tupleToAck in tuplesToAck)
{
this.ctx.Ack(tupleToAck);
}
tuplesToAck.Clear();
}
}
}
else
{
//Merge partialCount from all PartialCountBolt tasks
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
//Do no ack here but add to the acking queue
if (enableAck)
{
tuplesToAck.Enqueue(tuple);
}
}
}
public void Execute(SCPTuple tuple)
{
Context.Logger.Info("Execute enter");
if (Constants.SYSTEM_TICK_STREAM_ID.Equals(tuple.GetSourceStreamId()))
{
long data = tuple.GetLong(0);
Context.Logger.Info("tick tuple, value: {0}", data);
}
else
{
byte[] data = tuple.GetBinary(0);
int bytesNum = data.Count();
if (enableAck)
{
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List<SCPTuple> { tuple }, new Values(bytesNum));
this.ctx.Ack(tuple);
Context.Logger.Info("emit bytesNum: {0}", bytesNum);
Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId());
}
else
{
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(bytesNum));
Context.Logger.Info("emit bytesNum: {0}", bytesNum);
}
}
Context.Logger.Info("Execute exit");
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
Context.Logger.Info("Execute enter");
string streamId = tuple.GetSourceStreamId();
switch (streamId)
{
case SentenceGenerator.STREAM_ID:
{
string sentence = tuple.GetString(0);
Context.Logger.Info("sentence: {0}", sentence);
}
break;
case PersonGenerator.STREAM_ID:
{
Person person = (Person)tuple.GetValue(0);
Context.Logger.Info("person: {0}", person.ToString());
}
break;
default:
Context.Logger.Info("Get unknown tuple from unknown stream.");
break;
}
Context.Logger.Info("Execute exit");
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
Context.Logger.Info("Execute enter");
string streamId = tuple.GetSourceStreamId();
switch (streamId)
{
case SentenceGenerator.STREAM_ID:
{
string sentence = tuple.GetString(0);
Context.Logger.Info("sentence: {0}", sentence);
}
break;
case PersonGenerator.STREAM_ID:
{
Person person = (Person)tuple.GetValue(0);
Context.Logger.Info("person: {0}", person.ToString());
}
break;
default:
Context.Logger.Info("Get unknown tuple from unknown stream.");
break;
}
Context.Logger.Info("Execute exit");
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting partialCount: " + partialCount +
", totalCount: " + totalCount);
//emit with anchors set the tuples in this batch
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount));
//ideally in the logs partialCount and the batch count will match
Context.Logger.Info("acking the batch: " + tuplesToAck.Count);
foreach (var t in tuplesToAck)
{
this.ctx.Ack(t);
}
//once all the tuples are acked, clear the batch
tuplesToAck.Clear();
partialCount = 0L;
}
}
else
{
partialCount++;
totalCount++;
//Do no ack here but add to the acking queue
tuplesToAck.Enqueue(tuple);
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting totalCount" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
//emit with anchors set the tuples in this batch
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(CurrentTimeMillis(), totalCount));
Context.Logger.Info("acking the batch: " + tuplesToAck.Count);
foreach (var t in tuplesToAck)
{
this.ctx.Ack(t);
}
//once all the tuples are acked, clear the batch
tuplesToAck.Clear();
partialCount = 0L;
}
}
else
{
//Merge partialCount from all PartialCountBolt tasks
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
//Do no ack here but add to the acking queue
tuplesToAck.Enqueue(tuple);
}
}
/// <summary>
/// Executes incoming tuples
/// </summary>
/// <param name="tuple">The first field is treated as rowkey and rest as column values</param>
public void Execute(SCPTuple tuple)
{
try
{
var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID);
//Only add to cache if its not a Tick tuple
if (!isTickTuple)
{
//seqId helps in keeping the incoming tuples in order of their arrival
cachedTuples.Add(seqId, tuple);
seqId++;
}
//TODO: You can choose to write into HBase based on cached tuples count or when the tick tuple arrives
//To use Tick tuples make sure that you configure topology.tick.tuple.freq.secs on the bolt and also add the stream in the input streams
/* Add this section to your SetBolt in TopologyBuilder to trigger Tick tuples
* addConfigurations(new Dictionary<string, string>()
* {
* {"topology.tick.tuple.freq.secs", "5"}
* })
*/
//For this example, just emit on every tuple
//since we will be randomly generating end tuples only every few minutes.
if (cachedTuples.Count >= 1 || isTickTuple)
{
WriteToHBase();
//Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack.
if (enableAck)
{
//Ack all the tuples in the batch
foreach (var cachedTuple in cachedTuples)
{
this.context.Ack(cachedTuple.Value);
}
}
cachedTuples.Clear();
}
}
catch (Exception ex)
{
Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}",
tuple.GetTupleId(), ex.ToString());
//Is ack enabled?
if (enableAck)
{
Context.Logger.Error("Failing the entire current batch");
//Fail all the tuples in the batch
foreach (var cachedTuple in cachedTuples)
{
this.context.Fail(cachedTuple.Value);
}
}
}
}
public void Execute(SCPTuple tuple)
{
var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID);
if (isTickTuple)
{
// Get top 10 higest score tweets from last time window
Context.Logger.Debug($"Total tweets in window: {tweetCache.Count}");
var topNTweets = tweetCache.OrderByDescending(o => o.Score).Take(Math.Min(10, tweetCache.Count)).ToList();
// Emit it to TopNTweet Stream
foreach (var tweet in topNTweets)
{
//this.context.Emit(StormConstants.TOPNTWEETS_STREAM, new Values(tweet.Text, tweet.Id, tweet.RetweetCount, tweet.FavoriteCount, tweet.UserFollowerCount, tweet.Score));
this.context.Emit("TOPNTWEETS_STREAM", new Values(tweet));
}
// Remove all existing data and wait for new one
tweetCache.Clear();
}
else
{
try
{
// Process tuple and then acknowledge it
SerializableTweet tweet = tuple.GetValue(0) as SerializableTweet;
if (!tweetCache.Any(o => o.Id.Equals(tweet.Id)))
{
tweetCache.Add(tweet);
}
Context.Logger.Info(tweet.ToString());
if (enableAck)
{
this.context.Ack(tuple);
Context.Logger.Info("Total Ack: " + ++totalAck);
}
}
catch (Exception ex)
{
Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}",
tuple.GetTupleId(), ex.ToString());
//Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed.
if (enableAck)
{
this.context.Fail(tuple);
}
}
}
}
public void Execute(SCPTuple tuple)
{
var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID);
if (isTickTuple)
{
// Get top 10 higest forwards + retweets count from last time window of 5 seconds
Context.Logger.Debug($"Total tweets in window: {tweetCache.Count}");
var topNTweets = tweetCache.OrderByDescending(o => o.Score).Take(Math.Min(10, tweetCache.Count)).ToList();
foreach (var tweet in topNTweets)
{
this.context.Emit("TWEETRANK_STREAM", new Values(tweet));
}
tweetCache.Clear();
}
else
{
try
{
TwitterFeed tweet = tuple.GetValue(0) as TwitterFeed;
if (!tweetCache.Any(o => o.Id.Equals(tweet.Id)))
{
tweetCache.Add(tweet);
}
Context.Logger.Info(tweet.ToString());
if (enableAck)
{
this.context.Ack(tuple);
Context.Logger.Info("Total Ack: " + ++totalAck);
}
}
catch (Exception ex)
{
Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}",
tuple.GetTupleId(), ex.ToString());
if (enableAck)
{
this.context.Fail(tuple);
}
}
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting partialCount: " + partialCount +
", totalCount: " + totalCount);
this.ctx.Emit(new Values(partialCount));
partialCount = 0L;
}
}
else
{
partialCount++;
totalCount++;
this.ctx.Ack(tuple);
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting partialCount: " + partialCount +
", totalCount: " + totalCount);
this.ctx.Emit(new Values(partialCount));
partialCount = 0L;
}
}
else
{
partialCount++;
totalCount++;
this.ctx.Ack(tuple);
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
Context.Logger.Info("Aggregates tuple values..");
foreach (var key in _data.Keys)
{
var avg = _data[key].Average();
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(key, avg));
}
Context.Logger.Info("acking the batch: " + tuplesToAck.Count);
foreach (var t in tuplesToAck)
{
this.ctx.Ack(t);
}
_data.Clear();
tuplesToAck.Clear();
}
else
{
var sensorName = tuple.GetString(0);
var value = tuple.GetDouble(1);
if (!_data.ContainsKey(tuple.GetString(0)))
{
_data.Add(sensorName, new List <double>()
{
value
});
}
else
{
_data[sensorName].Add(value);
}
tuplesToAck.Enqueue(tuple);
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting totalCount" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
this.ctx.Emit(new Values(CurrentTimeMillis(), totalCount));
partialCount = 0L;
}
}
else
{
//Merge partialCount from all EventCountPartialCountBolt
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("updating database" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
db.insertValue(CurrentTimeMillis(), partialCount);
partialCount = 0L;
}
}
else
{
//Merge partialCount from all PartialCountBolt tasks
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting totalCount" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
this.ctx.Emit(new Values(CurrentTimeMillis(), totalCount));
partialCount = 0L;
}
}
else
{
//Merge partialCount from all EventCountPartialCountBolt
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("updating database" +
", partialCount: " + partialCount +
", totalCount: " + totalCount);
db.insertValue(CurrentTimeMillis(), partialCount);
partialCount = 0L;
}
}
else
{
//Merge partialCount from all PartialCountBolt tasks
var incomingPartialCount = tuple.GetLong(0);
partialCount += incomingPartialCount;
totalCount += incomingPartialCount;
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting partialCount: " + partialCount +
", totalCount: " + totalCount);
if (enableAck)
{
Context.Logger.Info("tuplesToAck: " + tuplesToAck);
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount));
foreach (var tupleToAck in tuplesToAck)
{
this.ctx.Ack(tupleToAck);
}
tuplesToAck.Clear();
}
else
{
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(partialCount));
}
partialCount = 0L;
}
}
else
{
partialCount++;
totalCount++;
//Do no ack here but add to the acking queue
if (enableAck)
{
tuplesToAck.Enqueue(tuple);
}
}
}
/// <summary>
/// The Execute() function will be called, when a new tuple is available.
/// </summary>
/// <param name="tuple"></param>
public void Execute(SCPTuple tuple)
{
if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID))
{
if (partialCount > 0)
{
Context.Logger.Info("emitting partialCount: " + partialCount +
", totalCount: " + totalCount);
if (enableAck)
{
Context.Logger.Info("tuplesToAck: " + tuplesToAck);
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount));
foreach (var tupleToAck in tuplesToAck)
{
this.ctx.Ack(tupleToAck);
}
tuplesToAck.Clear();
}
else
{
this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(partialCount));
}
partialCount = 0L;
}
}
else
{
partialCount++;
totalCount++;
//Do no ack here but add to the acking queue
if (enableAck)
{
tuplesToAck.Enqueue(tuple);
}
}
}
public static bool IsTick(this SCPTuple tuple)
{
return(tuple.GetSourceStreamId()
.Equals(Constants.SYSTEM_TICK_STREAM_ID, StringComparison.InvariantCultureIgnoreCase));
}
/// <summary>
/// Executes incoming tuples
/// </summary>
/// <param name="tuple">The first field is treated as rowkey and rest as column values</param>
public void Execute(SCPTuple tuple)
{
try
{
var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID);
//Only add to cache if its not a Tick tuple
if (!isTickTuple)
{
//seqId helps in keeping the incoming tuples in order of their arrival
cachedTuples.Add(seqId, tuple);
seqId++;
}
//TODO: You can choose to write into HBase based on cached tuples count or when the tick tuple arrives
//To use Tick tuples make sure that you configure topology.tick.tuple.freq.secs on the bolt and also add the stream in the input streams
/* Add this section to your SetBolt in TopologyBuilder to trigger Tick tuples
addConfigurations(new Dictionary<string, string>()
{
{"topology.tick.tuple.freq.secs", "5"}
})
*/
if (cachedTuples.Count >= 1000 || isTickTuple)
{
WriteToHBase();
//Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack.
if (enableAck)
{
//Ack all the tuples in the batch
foreach(var cachedTuple in cachedTuples)
{
this.context.Ack(cachedTuple.Value);
}
}
cachedTuples.Clear();
}
}
catch (Exception ex)
{
Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}",
tuple.GetTupleId(), ex.ToString());
if (enableAck)
{
Context.Logger.Error("Failing the entire current batch");
//Fail all the tuples in the batch
foreach (var cachedTuple in cachedTuples)
{
this.context.Fail(cachedTuple.Value);
}
}
}
}