/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("updating database" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); db.insertValue(CurrentTimeMillis(), partialCount); partialCount = 0L; if (enableAck) { Context.Logger.Info("tuplesToAck: " + tuplesToAck); foreach (var tupleToAck in tuplesToAck) { this.ctx.Ack(tupleToAck); } tuplesToAck.Clear(); } } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; //Do no ack here but add to the acking queue if (enableAck) { tuplesToAck.Enqueue(tuple); } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting totalCount" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); //emit with anchors set the tuples in this batch this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(CurrentTimeMillis(), totalCount)); Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } //once all the tuples are acked, clear the batch tuplesToAck.Clear(); partialCount = 0L; } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; //Do no ack here but add to the acking queue tuplesToAck.Enqueue(tuple); } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); //emit with anchors set the tuples in this batch this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount)); //ideally in the logs partialCount and the batch count will match Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } //once all the tuples are acked, clear the batch tuplesToAck.Clear(); partialCount = 0L; } } else { partialCount++; totalCount++; //Do no ack here but add to the acking queue tuplesToAck.Enqueue(tuple); } }
public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); if (Constants.SYSTEM_TICK_STREAM_ID.Equals(tuple.GetSourceStreamId())) { long data = tuple.GetLong(0); Context.Logger.Info("tick tuple, value: {0}", data); } else { byte[] data = tuple.GetBinary(0); int bytesNum = data.Count(); if (enableAck) { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple> { tuple }, new Values(bytesNum)); this.ctx.Ack(tuple); Context.Logger.Info("emit bytesNum: {0}", bytesNum); Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(bytesNum)); Context.Logger.Info("emit bytesNum: {0}", bytesNum); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("updating database" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); db.insertValue(CurrentTimeMillis(), partialCount); partialCount = 0L; if (enableAck) { Context.Logger.Info("tuplesToAck: " + tuplesToAck); foreach (var tupleToAck in tuplesToAck) { this.ctx.Ack(tupleToAck); } tuplesToAck.Clear(); } } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; //Do no ack here but add to the acking queue if (enableAck) { tuplesToAck.Enqueue(tuple); } } }
public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); if (Constants.SYSTEM_TICK_STREAM_ID.Equals(tuple.GetSourceStreamId())) { long data = tuple.GetLong(0); Context.Logger.Info("tick tuple, value: {0}", data); } else { byte[] data = tuple.GetBinary(0); int bytesNum = data.Count(); if (enableAck) { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List<SCPTuple> { tuple }, new Values(bytesNum)); this.ctx.Ack(tuple); Context.Logger.Info("emit bytesNum: {0}", bytesNum); Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(bytesNum)); Context.Logger.Info("emit bytesNum: {0}", bytesNum); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string streamId = tuple.GetSourceStreamId(); switch (streamId) { case SentenceGenerator.STREAM_ID: { string sentence = tuple.GetString(0); Context.Logger.Info("sentence: {0}", sentence); } break; case PersonGenerator.STREAM_ID: { Person person = (Person)tuple.GetValue(0); Context.Logger.Info("person: {0}", person.ToString()); } break; default: Context.Logger.Info("Get unknown tuple from unknown stream."); break; } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string streamId = tuple.GetSourceStreamId(); switch (streamId) { case SentenceGenerator.STREAM_ID: { string sentence = tuple.GetString(0); Context.Logger.Info("sentence: {0}", sentence); } break; case PersonGenerator.STREAM_ID: { Person person = (Person)tuple.GetValue(0); Context.Logger.Info("person: {0}", person.ToString()); } break; default: Context.Logger.Info("Get unknown tuple from unknown stream."); break; } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); //emit with anchors set the tuples in this batch this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount)); //ideally in the logs partialCount and the batch count will match Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } //once all the tuples are acked, clear the batch tuplesToAck.Clear(); partialCount = 0L; } } else { partialCount++; totalCount++; //Do no ack here but add to the acking queue tuplesToAck.Enqueue(tuple); } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting totalCount" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); //emit with anchors set the tuples in this batch this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(CurrentTimeMillis(), totalCount)); Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } //once all the tuples are acked, clear the batch tuplesToAck.Clear(); partialCount = 0L; } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; //Do no ack here but add to the acking queue tuplesToAck.Enqueue(tuple); } }
/// <summary> /// Executes incoming tuples /// </summary> /// <param name="tuple">The first field is treated as rowkey and rest as column values</param> public void Execute(SCPTuple tuple) { try { var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID); //Only add to cache if its not a Tick tuple if (!isTickTuple) { //seqId helps in keeping the incoming tuples in order of their arrival cachedTuples.Add(seqId, tuple); seqId++; } //TODO: You can choose to write into HBase based on cached tuples count or when the tick tuple arrives //To use Tick tuples make sure that you configure topology.tick.tuple.freq.secs on the bolt and also add the stream in the input streams /* Add this section to your SetBolt in TopologyBuilder to trigger Tick tuples * addConfigurations(new Dictionary<string, string>() * { * {"topology.tick.tuple.freq.secs", "5"} * }) */ //For this example, just emit on every tuple //since we will be randomly generating end tuples only every few minutes. if (cachedTuples.Count >= 1 || isTickTuple) { WriteToHBase(); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { //Ack all the tuples in the batch foreach (var cachedTuple in cachedTuples) { this.context.Ack(cachedTuple.Value); } } cachedTuples.Clear(); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Is ack enabled? if (enableAck) { Context.Logger.Error("Failing the entire current batch"); //Fail all the tuples in the batch foreach (var cachedTuple in cachedTuples) { this.context.Fail(cachedTuple.Value); } } } }
public void Execute(SCPTuple tuple) { var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID); if (isTickTuple) { // Get top 10 higest score tweets from last time window Context.Logger.Debug($"Total tweets in window: {tweetCache.Count}"); var topNTweets = tweetCache.OrderByDescending(o => o.Score).Take(Math.Min(10, tweetCache.Count)).ToList(); // Emit it to TopNTweet Stream foreach (var tweet in topNTweets) { //this.context.Emit(StormConstants.TOPNTWEETS_STREAM, new Values(tweet.Text, tweet.Id, tweet.RetweetCount, tweet.FavoriteCount, tweet.UserFollowerCount, tweet.Score)); this.context.Emit("TOPNTWEETS_STREAM", new Values(tweet)); } // Remove all existing data and wait for new one tweetCache.Clear(); } else { try { // Process tuple and then acknowledge it SerializableTweet tweet = tuple.GetValue(0) as SerializableTweet; if (!tweetCache.Any(o => o.Id.Equals(tweet.Id))) { tweetCache.Add(tweet); } Context.Logger.Info(tweet.ToString()); if (enableAck) { this.context.Ack(tuple); Context.Logger.Info("Total Ack: " + ++totalAck); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } } }
public void Execute(SCPTuple tuple) { var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID); if (isTickTuple) { // Get top 10 higest forwards + retweets count from last time window of 5 seconds Context.Logger.Debug($"Total tweets in window: {tweetCache.Count}"); var topNTweets = tweetCache.OrderByDescending(o => o.Score).Take(Math.Min(10, tweetCache.Count)).ToList(); foreach (var tweet in topNTweets) { this.context.Emit("TWEETRANK_STREAM", new Values(tweet)); } tweetCache.Clear(); } else { try { TwitterFeed tweet = tuple.GetValue(0) as TwitterFeed; if (!tweetCache.Any(o => o.Id.Equals(tweet.Id))) { tweetCache.Add(tweet); } Context.Logger.Info(tweet.ToString()); if (enableAck) { this.context.Ack(tuple); Context.Logger.Info("Total Ack: " + ++totalAck); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); if (enableAck) { this.context.Fail(tuple); } } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); this.ctx.Emit(new Values(partialCount)); partialCount = 0L; } } else { partialCount++; totalCount++; this.ctx.Ack(tuple); } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); this.ctx.Emit(new Values(partialCount)); partialCount = 0L; } } else { partialCount++; totalCount++; this.ctx.Ack(tuple); } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { Context.Logger.Info("Aggregates tuple values.."); foreach (var key in _data.Keys) { var avg = _data[key].Average(); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(key, avg)); } Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } _data.Clear(); tuplesToAck.Clear(); } else { var sensorName = tuple.GetString(0); var value = tuple.GetDouble(1); if (!_data.ContainsKey(tuple.GetString(0))) { _data.Add(sensorName, new List <double>() { value }); } else { _data[sensorName].Add(value); } tuplesToAck.Enqueue(tuple); } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting totalCount" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); this.ctx.Emit(new Values(CurrentTimeMillis(), totalCount)); partialCount = 0L; } } else { //Merge partialCount from all EventCountPartialCountBolt var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("updating database" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); db.insertValue(CurrentTimeMillis(), partialCount); partialCount = 0L; } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting totalCount" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); this.ctx.Emit(new Values(CurrentTimeMillis(), totalCount)); partialCount = 0L; } } else { //Merge partialCount from all EventCountPartialCountBolt var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("updating database" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); db.insertValue(CurrentTimeMillis(), partialCount); partialCount = 0L; } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); if (enableAck) { Context.Logger.Info("tuplesToAck: " + tuplesToAck); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount)); foreach (var tupleToAck in tuplesToAck) { this.ctx.Ack(tupleToAck); } tuplesToAck.Clear(); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(partialCount)); } partialCount = 0L; } } else { partialCount++; totalCount++; //Do no ack here but add to the acking queue if (enableAck) { tuplesToAck.Enqueue(tuple); } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); if (enableAck) { Context.Logger.Info("tuplesToAck: " + tuplesToAck); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount)); foreach (var tupleToAck in tuplesToAck) { this.ctx.Ack(tupleToAck); } tuplesToAck.Clear(); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(partialCount)); } partialCount = 0L; } } else { partialCount++; totalCount++; //Do no ack here but add to the acking queue if (enableAck) { tuplesToAck.Enqueue(tuple); } } }
public static bool IsTick(this SCPTuple tuple) { return(tuple.GetSourceStreamId() .Equals(Constants.SYSTEM_TICK_STREAM_ID, StringComparison.InvariantCultureIgnoreCase)); }
/// <summary> /// Executes incoming tuples /// </summary> /// <param name="tuple">The first field is treated as rowkey and rest as column values</param> public void Execute(SCPTuple tuple) { try { var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID); //Only add to cache if its not a Tick tuple if (!isTickTuple) { //seqId helps in keeping the incoming tuples in order of their arrival cachedTuples.Add(seqId, tuple); seqId++; } //TODO: You can choose to write into HBase based on cached tuples count or when the tick tuple arrives //To use Tick tuples make sure that you configure topology.tick.tuple.freq.secs on the bolt and also add the stream in the input streams /* Add this section to your SetBolt in TopologyBuilder to trigger Tick tuples addConfigurations(new Dictionary<string, string>() { {"topology.tick.tuple.freq.secs", "5"} }) */ if (cachedTuples.Count >= 1000 || isTickTuple) { WriteToHBase(); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { //Ack all the tuples in the batch foreach(var cachedTuple in cachedTuples) { this.context.Ack(cachedTuple.Value); } } cachedTuples.Clear(); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); if (enableAck) { Context.Logger.Error("Failing the entire current batch"); //Fail all the tuples in the batch foreach (var cachedTuple in cachedTuples) { this.context.Fail(cachedTuple.Value); } } } }