public void Execute(SCPTuple tuple) { try { // TwitterFeed tweet = tuple.GetValue(0) as TwitterFeed; TwitterFeed tweet = tuple.GetValue(0) as TwitterFeed; if (tweet != null) { Context.Logger.Info("SQL AZURE: Id:" + tweet.Id.ToString()); Context.Logger.Info("SQL AZURE: Text:" + tweet.Text.ToString()); Context.Logger.Info("SQL AZURE: RetweetCount:" + tweet.RetweetCount.ToString()); Context.Logger.Info("SQL AZURE: FavoriteCount:" + tweet.FavoriteCount.ToString()); Context.Logger.Info("SQL AZURE: Score:" + tweet.Score.ToString()); Context.Logger.Info("SQL AZURE: Created Date:" + tweet.Createddate.ToString()); Context.Logger.Info("SQL AZURE: DateTime.UtcNow:" + DateTime.UtcNow.ToString()); } List <object> rowValue = new List <object>(); rowValue.Add(tweet.Id); rowValue.Add(tweet.Text); rowValue.Add(tweet.RetweetCount); rowValue.Add(tweet.FavoriteCount); rowValue.Add(tweet.Score); rowValue.Add(GetSentimentType(tweet.Text)); rowValue.Add(tweet.Createddate); rowValue.Add(DateTime.UtcNow); //Upsert(new List<int> { 1 }, rowValue); Insert(rowValue); } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); } }
public void Execute(SCPTuple tuple) { int cnt = tuple.GetInteger(0); this.count += cnt; Context.Logger.Info("CountSum, Execute(), cnt: {0}, this.count: {1}", cnt, this.count); }
//Process a tuple from the stream public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); try { // Count indexItem RowCount++; // Skip updates for 10 milliseconds and send regularly around every 100 milliseconds if (timer1.ElapsedMilliseconds >= 10 && timer2.ElapsedMilliseconds >= 100) { timer2.Restart(); SendSingnalRUpdate(RowCount); timer1.Restart(); } } catch (Exception ex) { Context.Logger.Error("SignalRBroadcastBolt Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace); } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { ctx.Ack(tuple); //Log tuple content Context.Logger.Warn(tuple.GetString(0)); }
//Process a tuple from the stream public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); try { // Count indexItem RowCount++; if (timer.ElapsedMilliseconds >= 20) { // Update count in HBase client.UpdateRowCount(RowCount); Context.Logger.Info("===== Total Row Count: {0} =====", RowCount); timer.Restart(); } } catch (Exception ex) { Context.Logger.Error("HBaseCounterBolt Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace); } Context.Logger.Info("Execute exit"); }
/// <summary> /// The execute method for incoming tuples /// </summary> /// <param name="tuple">The incoming tuple</param> public void Execute(SCPTuple tuple) { try { var values = tuple.GetValues(); foreach (var value in values) { Context.Logger.Info("Creating document: {0} with value: {1}", value, JsonConvert.SerializeObject(value)); var task = documentClient.CreateDocumentAsync(documentCollection.DocumentsLink, value); task.Wait(); Context.Logger.Info("Document creation result status: {0}", task.Result.StatusCode); } //Ack the tuple if enableAck is set to true in TopologyBuilder. //This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { this.context.Ack(tuple); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("updating database" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); db.insertValue(CurrentTimeMillis(), partialCount); partialCount = 0L; if (enableAck) { Context.Logger.Info("tuplesToAck: " + tuplesToAck); foreach (var tupleToAck in tuplesToAck) { this.ctx.Ack(tupleToAck); } tuplesToAck.Clear(); } } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; //Do no ack here but add to the acking queue if (enableAck) { tuplesToAck.Enqueue(tuple); } } }
public void Execute(SCPTuple tuple) { string json = tuple.GetString(0); var node = JObject.Parse(json); var temp = node.GetValue("temp"); JToken tempVal; if (node.TryGetValue("temp", out tempVal)) //assume must be a temperature reading { Context.Logger.Info("temp:" + temp.Value <double>()); JToken createDate = node.GetValue("createDate"); JToken deviceId = node.GetValue("deviceId"); _context.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple>() { tuple }, new List <object> { tempVal.Value <double>(), createDate.Value <string>(), deviceId.Value <string>() }); } _context.Ack(tuple); }
':', ';', '<', '=', '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~' }; //ascii 58--64 + misc. //Process a tuple from the stream public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); try { var words = tuple.GetString(0).ToLower().Split(_punctuationChars); int sentimentScore = CalcSentimentScore(words); var word_pairs = words.Take(words.Length - 1) .Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1])); var all_words = words.Concat(word_pairs).ToList(); // Emit all index entries for counting and writing downstream foreach (var word in all_words) { this.ctx.Emit(new Values(word, tuple.GetLong(1), tuple.GetString(2), tuple.GetString(3), tuple.GetString(4), sentimentScore)); } } catch (Exception ex) { Context.Logger.Error("SentimentIndexerBolt Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace); } Context.Logger.Info("Execute exit"); }
public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); if (Constants.SYSTEM_TICK_STREAM_ID.Equals(tuple.GetSourceStreamId())) { long data = tuple.GetLong(0); Context.Logger.Info("tick tuple, value: {0}", data); } else { byte[] data = tuple.GetBinary(0); int bytesNum = data.Count(); if (enableAck) { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List<SCPTuple> { tuple }, new Values(bytesNum)); this.ctx.Ack(tuple); Context.Logger.Info("emit bytesNum: {0}", bytesNum); Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(bytesNum)); Context.Logger.Info("emit bytesNum: {0}", bytesNum); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string word = tuple.GetString(0); int count = counts.ContainsKey(word) ? counts[word] : 0; count++; counts[word] = count; Context.Logger.Info("Emit: {0}, count: {1}", word, count); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List<SCPTuple> { tuple }, new Values(word, count)); if (enableAck) { Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Ack(tuple); } // log some info to out file for bvt test validataion if (taskIndex == 0) // For component with multiple parallism, only one of them need to log info { string fileName = @"..\..\..\..\..\HelloWorldOutput" + Process.GetCurrentProcess().Id + ".txt"; FileStream fs = new FileStream(fileName, FileMode.Append); using (StreamWriter writer = new StreamWriter(fs)) { writer.WriteLine("word: {0}, count: {1}", word, count); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string streamId = tuple.GetSourceStreamId(); switch (streamId) { case SentenceGenerator.STREAM_ID: { string sentence = tuple.GetString(0); Context.Logger.Info("sentence: {0}", sentence); } break; case PersonGenerator.STREAM_ID: { Person person = (Person)tuple.GetValue(0); Context.Logger.Info("person: {0}", person.ToString()); } break; default: Context.Logger.Info("Get unknown tuple from unknown stream."); break; } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string sentence = tuple.GetString(0); foreach (string word in sentence.Split(' ')) { Context.Logger.Info("Emit: {0}", word); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple> { tuple }, new Values(word, word[0])); } if (enableAck) { if (Sample(50)) // this is to demo how to fail tuple. We do it randomly { Context.Logger.Info("fail tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Fail(tuple); } else { if (Sample(50)) // this is to simulate timeout { Context.Logger.Info("sleep {0} seconds", msgTimeoutSecs + 1); Thread.Sleep((msgTimeoutSecs + 1) * 1000); } Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Ack(tuple); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); //emit with anchors set the tuples in this batch this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount)); //ideally in the logs partialCount and the batch count will match Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } //once all the tuples are acked, clear the batch tuplesToAck.Clear(); partialCount = 0L; } } else { partialCount++; totalCount++; //Do no ack here but add to the acking queue tuplesToAck.Enqueue(tuple); } }
public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); if (Constants.SYSTEM_TICK_STREAM_ID.Equals(tuple.GetSourceStreamId())) { long data = tuple.GetLong(0); Context.Logger.Info("tick tuple, value: {0}", data); } else { byte[] data = tuple.GetBinary(0); int bytesNum = data.Count(); if (enableAck) { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple> { tuple }, new Values(bytesNum)); this.ctx.Ack(tuple); Context.Logger.Info("emit bytesNum: {0}", bytesNum); Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(bytesNum)); Context.Logger.Info("emit bytesNum: {0}", bytesNum); } } Context.Logger.Info("Execute exit"); }
}; //ascii 58--64 + misc. //Process a tuple from the stream public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); try { var words = tuple.GetString(0).ToLower().Split(_punctuationChars); int sentimentScore = CalcSentimentScore(words); var word_pairs = words.Take(words.Length - 1) .Select((word, idx) => string.Format("{0} {1}", word, words[idx + 1])); var all_words = words.Concat(word_pairs).ToList(); // Emit all index entries for counting and writing downstream foreach (var word in all_words) { this.ctx.Emit(new Values(word, tuple.GetLong(1), tuple.GetString(2), tuple.GetString(3), tuple.GetString(4), sentimentScore)); } } catch (Exception ex) { Context.Logger.Error("SentimentIndexerBolt Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace); } Context.Logger.Info("Execute exit"); }
/// <summary> /// The execute method for tuple received /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { try { var values = tuple.GetValues(); var data = string.Empty; if (values.Count == 1) { data = JsonConvert.SerializeObject(values[0]); } else { data = JsonConvert.SerializeObject(values); } eventHubSender.Send(new EventData(Encoding.UTF8.GetBytes(data))); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { this.context.Ack(tuple); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting totalCount" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); //emit with anchors set the tuples in this batch this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(CurrentTimeMillis(), totalCount)); Context.Logger.Info("acking the batch: " + tuplesToAck.Count); foreach (var t in tuplesToAck) { this.ctx.Ack(t); } //once all the tuples are acked, clear the batch tuplesToAck.Clear(); partialCount = 0L; } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; //Do no ack here but add to the acking queue tuplesToAck.Enqueue(tuple); } }
public virtual void ProcessTuple(SCPTuple tuple) { last_receive_count++; global_receive_count++; try { var aggregationTimestamp = (DateTime)tuple.GetValue(0); var primarykeyvalue = ((string)tuple.GetValue(1)) ?? Utilities.DEFAULT_VALUE; var secondarykeyvalue = ((string)tuple.GetValue(2)) ?? Utilities.DEFAULT_VALUE; var value = (double)tuple.GetValue(3); if (aggregationTimestamp != null) { Aggregate(aggregationTimestamp, primarykeyvalue, secondarykeyvalue, value); } else { Context.Logger.Warn("Cannot Aggregate: aggregationTimestamp is null. PrimaryKeyValue = {0}, SecondaryKeyValue = {1}, AggregationValue = {2}", primarykeyvalue, secondarykeyvalue, value); } } catch (Exception ex) { global_error_count++; last_error_count++; Context.Logger.Error(ex.ToString()); } }
public void Execute(SCPTuple tuple) { try { SerializableTweet tweet = tuple as SerializableTweet; //tweet.Text = tuple.GetString(0); //tweet.Id = tuple.GetLong(1); //tweet.RetweetCount = tuple.GetInteger(2); //tweet.FavoriteCount = tuple.GetInteger(3); //tweet.UserFollowerCount = tuple.GetInteger(4); Context.Logger.Info("SQL AZURE: " + tweet.ToString()); //TODO: Insert or Upsert or Delete depending on your logic //Delete(new List<int>() { 1, 2 }, tuple.GetValues()); //Upsert(new List<int>() { 1, 2 }, tuple.GetValues()); List <object> rowValue = new List <object>(); rowValue.Add(tweet.Id); rowValue.Add(tweet.Text); rowValue.Add(tweet.RetweetCount); rowValue.Add(tweet.FavoriteCount); rowValue.Add(tweet.Score); rowValue.Add(DateTime.UtcNow); Upsert(new List <int> { 1 }, rowValue); } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); } }
public void Execute(SCPTuple tuple) { try { if (hubConnection.State != ConnectionState.Connected) { hubConnection.Stop(); StartSignalRHubConnection(); } var values = tuple.GetValues(); hubProxy.Invoke(this.SignalRMethod, values); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { this.context.Ack(tuple); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string sentence = tuple.GetString(0); foreach (string word in sentence.Split(' ')) { Context.Logger.Info("Emit: {0}", word); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List<SCPTuple> { tuple }, new Values(word, word[0])); } if (enableAck) { if (Sample(50)) // this is to demo how to fail tuple. We do it randomly { Context.Logger.Info("fail tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Fail(tuple); } else { if (Sample(50)) // this is to simulate timeout { Context.Logger.Info("sleep {0} seconds", msgTimeoutSecs+1); Thread.Sleep((msgTimeoutSecs + 1) * 1000); } Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Ack(tuple); } } Context.Logger.Info("Execute exit"); }
public void Execute(SCPTuple tuple) { try { //TODO: Insert or Upsert or Delete depending on your logic //Delete(new List<int>() { 1, 2 }, tuple.GetValues()); //Upsert(new List<int>() { 1, 2 }, tuple.GetValues()); Insert(tuple.GetValues()); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { this.context.Ack(tuple); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); string word = tuple.GetString(0); int count = counts.ContainsKey(word) ? counts[word] : 0; count++; counts[word] = count; Context.Logger.Info("Emit: {0}, count: {1}", word, count); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List <SCPTuple> { tuple }, new Values(word, count)); if (enableAck) { Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Ack(tuple); } // log some info to out file for bvt test validataion if (taskIndex == 0) // For component with multiple parallism, only one of them need to log info { string fileName = @"..\..\..\..\..\HelloWorldOutput" + Process.GetCurrentProcess().Id + ".txt"; FileStream fs = new FileStream(fileName, FileMode.Append); using (StreamWriter writer = new StreamWriter(fs)) { writer.WriteLine("word: {0}, count: {1}", word, count); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); Person person = (Person)tuple.GetValue(0); Context.Logger.Info("person: {0}", person.ToString()); Context.Logger.Info("Execute exit"); }
/// <summary> /// Executes incoming tuples /// </summary> /// <param name="tuple">The first field is treated as rowkey and rest as column values</param> public void Execute(SCPTuple tuple) { try { var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID); //Only add to cache if its not a Tick tuple if (!isTickTuple) { //seqId helps in keeping the incoming tuples in order of their arrival cachedTuples.Add(seqId, tuple); seqId++; } //TODO: You can choose to write into HBase based on cached tuples count or when the tick tuple arrives //To use Tick tuples make sure that you configure topology.tick.tuple.freq.secs on the bolt and also add the stream in the input streams /* Add this section to your SetBolt in TopologyBuilder to trigger Tick tuples * addConfigurations(new Dictionary<string, string>() * { * {"topology.tick.tuple.freq.secs", "5"} * }) */ //For this example, just emit on every tuple //since we will be randomly generating end tuples only every few minutes. if (cachedTuples.Count >= 1 || isTickTuple) { WriteToHBase(); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { //Ack all the tuples in the batch foreach (var cachedTuple in cachedTuples) { this.context.Ack(cachedTuple.Value); } } cachedTuples.Clear(); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Is ack enabled? if (enableAck) { Context.Logger.Error("Failing the entire current batch"); //Fail all the tuples in the batch foreach (var cachedTuple in cachedTuples) { this.context.Fail(cachedTuple.Value); } } } }
public void Execute(SCPTuple tuple) { PowerBIClient.Do(api => { var isObjectInsert = api.Insert(datasetId, new Data.WordCount { Word = tuple.GetString(0), Count = tuple.GetInteger(1) }); }); }
public void Execute(SCPTuple tuple) { var sensor = tuple.GetSensor(); _ctx.Emit(Constants.DEFAULT_STREAM_ID, new List <object>() { sensor.Name, sensor.Value }); _ctx.Ack(tuple); }
public void Execute(SCPTuple tuple) { var isTickTuple = tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID); if (isTickTuple) { // Get top 10 higest score tweets from last time window Context.Logger.Debug($"Total tweets in window: {tweetCache.Count}"); var topNTweets = tweetCache.OrderByDescending(o => o.Score).Take(Math.Min(10, tweetCache.Count)).ToList(); // Emit it to TopNTweet Stream foreach (var tweet in topNTweets) { //this.context.Emit(StormConstants.TOPNTWEETS_STREAM, new Values(tweet.Text, tweet.Id, tweet.RetweetCount, tweet.FavoriteCount, tweet.UserFollowerCount, tweet.Score)); this.context.Emit("TOPNTWEETS_STREAM", new Values(tweet)); } // Remove all existing data and wait for new one tweetCache.Clear(); } else { try { // Process tuple and then acknowledge it SerializableTweet tweet = tuple.GetValue(0) as SerializableTweet; if (!tweetCache.Any(o => o.Id.Equals(tweet.Id))) { tweetCache.Add(tweet); } Context.Logger.Info(tweet.ToString()); if (enableAck) { this.context.Ack(tuple); Context.Logger.Info("Total Ack: " + ++totalAck); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } } }
// Called when a new tuple is available public void Execute(SCPTuple tuple) { // Get the sentance from the tuple string sentence = tuple.GetString(0); // Split at space characters foreach (string word in sentence.Split(' ')) { //Emit each word this.ctx.Emit(new Values(word)); } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { //Get the string data from the tuple string eventValue = (string)tuple.GetValue(0); if (eventValue != null) { //Log the data Context.Logger.Info("Received data: " + eventValue); //ACK the tuple so the spout knows it was processed //If we don't ACK, the EventHubSpout can stop receiving; it expects ACKs this.ctx.Ack(tuple); } }
public virtual void Execute(SCPTuple tuple) { last_receive_count++; global_receive_count++; var inputeventdata = (string)tuple.GetValue(0); try { if (inputeventdata != null) { JToken token = JObject.Parse(inputeventdata); //This assumes that you wish to respect the timestamp field in your tuple //If you dont care on the order or timestamp of tuple, you can send a DateTime.UtcNow i.e. the receive time //This will allow you to aggregate based on current time than original event time. var timestampvalue = (string)token.SelectToken(this.appConfig.TimestampField); var timestamp = new DateTime(); var result = DateTime.TryParse(timestampvalue, out timestamp); //This computes an additional timestamp which is floored to your aggregation window //This acts as an alternative strategy to TickTuples as this allows you to process multiple windows at same time //and events arriving slightly out of order. For events that are huge apart i.e. //do not even fit in multiple AggregationWindows can potentially overwrite your previous aggregations //if you dont handle it properly later in your topology by doing right merges. //Based on your topology, you can choose which strategy suits you better var aggregationTimestamp = timestamp.Floor(this.appConfig.AggregationWindow); var primarykeyvalue = ((string)token.SelectToken(this.PrimaryKey)) ?? Utilities.DEFAULT_VALUE; var secondarykeyvalue = ((string)token.SelectToken(this.SecondaryKey)) ?? Utilities.DEFAULT_VALUE; //Aggregate the current input. The final argument can actually be a value of any field in your input, //allowing you to use this aggregation as sum than count. //We emit the aggregated tuples as part of aggregation process and expiry of the window. Aggregate(aggregationTimestamp, primarykeyvalue, secondarykeyvalue, 1); //Ack the tuple to the spout so that the spout can move forward and remove the tuple from its cache. //This is mandatory requirement if you use the default constructor for EventHubSpout as it uses the ack based PartitionManager this.context.Ack(tuple); } } catch (Exception ex) { global_error_count++; Context.Logger.Error(ex.ToString()); //Fail the tuple in spout if you were not able to deserialize or emit it. this.context.Fail(tuple); } }
/// <summary> /// The Execute() function is called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { //Get the string data from the tuple string eventValue = tuple.GetString(0); if (eventValue != null) { //Log the data Context.Logger.Info("Received data: " + eventValue); //ACK the tuple so the spout knows it was processed //If we don't ACK, the EventHubSpout can stop receiving; it expects ACKs this.ctx.Ack(tuple); } }
/// <summary> /// The execute method for tuple received /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { try { var task = eventHubSender.SendAsync(new EventData(Encoding.UTF8.GetBytes(tuple.GetString(0)))); if (ackEnabled) { tasks.Add(task); this.context.Ack(tuple); if (tasks.Count >= 100) { Context.Logger.Info("Total tasks in waiting = {0}", tasks.Count); Task.WaitAll(tasks.ToArray()); tasks.Clear(); Context.Logger.Info("All waiting tasks completed successfully!", tasks.Count); } } global_emit_count++; if (global_emit_count % 5000 == 0) { Context.Logger.Info("Total events sent to EventHub = {0} ({1} events/sec)", global_emit_count, global_emit_count / globalStopwatch.Elapsed.TotalSeconds); } } catch (Exception ex) { Context.Logger.Error("Failed to send tuples. Last Id = {0}, Value = {1}. Tasks = {2}", tuple.GetTupleId(), tuple.GetString(0), tasks.Count); Context.Logger.Error("Error Details: {0}", ex.ToString()); if (ackEnabled) { this.context.Fail(tuple); } global_error_count++; if (global_error_count > 10) { Context.Logger.Error("High error count: {0}", global_error_count); throw; } if (eventHubSender.IsClosed) { Context.Logger.Warn("EventHubSender is closed, re-intializing..."); InitializeEventHub(); } } }
public void Execute(SCPTuple tuple) { Context.Logger.Info("SqlAzureBolt: Execute enter."); if (Insert(tuple.GetValues())) { Context.Logger.Info("SqlAzureBolt: Tuple inserted."); } else { Context.Logger.Info("SqlAzureBolt: Tuple insert failed."); } Context.Logger.Info("SqlAzureBolt: Execute exit."); }
public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); int bytesNum = tuple.GetInteger(0); totalNum += bytesNum; Context.Logger.Info("bytesNum: {0}, totalNum: {1}", bytesNum, totalNum); if (enableAck) { Context.Logger.Info("Ack tuple: tupleId: {0}", tuple.GetTupleId()); this.ctx.Ack(tuple); } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); this.ctx.Emit(new Values(partialCount)); partialCount = 0L; } } else { partialCount++; totalCount++; this.ctx.Ack(tuple); } }
//Process a tuple from the stream public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); try { // Buffer batch of tuples buffer.Enqueue(tuple); if (buffer.Count >= 1000 || timer.ElapsedMilliseconds >= 50) { // Write buffer to HBase var list = new List<TweetIndexItem>(); do { var tweetWord = buffer.Dequeue(); list.Add(new TweetIndexItem { Word = tweetWord.GetString(0), CreatedAt = tweetWord.GetLong(1), IdStr = tweetWord.GetString(2), Language = tweetWord.GetString(3), Coordinates = tweetWord.GetString(4), SentimentScore = tweetWord.GetInteger(5) }); } while (buffer.Count > 0); client.WriteIndexItems(list); Context.Logger.Info("===== {0} rows written =====", list.Count); timer.Restart(); } } catch (Exception ex) { Context.Logger.Error("HBaseWriterBolt Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace); } Context.Logger.Info("Execute exit"); }
// Called when a new tuple is available public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); // Get the word from the tuple string word = tuple.GetString(0); // Do we already have an entry for the word in the dictionary? // If no, create one with a count of 0 int count = counts.ContainsKey(word) ? counts[word] : 0; // Increment the count count++; // Update the count in the dictionary counts[word] = count; Context.Logger.Info("Emit: {0}, count: {1}", word, count); // Emit the word and count information this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new List<SCPTuple> { tuple }, new Values(word, count)); Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); Person person = (Person)tuple.GetValue(0); Context.Logger.Info("person: {0}", person.ToString()); // log some info to out file for bvt test validataion if (taskIndex == 0) // For component with multiple parallism, only one of them need to log info { string fileName = @"..\..\..\..\..\HybridTopologyOutput" + Process.GetCurrentProcess().Id + ".txt"; FileStream fs = new FileStream(fileName, FileMode.Append); using (StreamWriter writer = new StreamWriter(fs)) { writer.WriteLine("person: {0}", person.ToString()); } } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("updating database" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); db.insertValue(CurrentTimeMillis(), partialCount); partialCount = 0L; } } else { //Merge partialCount from all PartialCountBolt tasks var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; } }
// Called when a new tuple is available public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); // Get the sentance from the tuple string sentence = tuple.GetString(0); // Split at space characters foreach (string word in sentence.Split(' ')) { Context.Logger.Info("Emit: {0}", word); if(word=="cow") { //Emit a true to the cowbell stream this.ctx.Emit("cowbells", new Values("ding")); } //Emit each word to the default stream this.ctx.Emit(new Values(word)); } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting totalCount" + ", partialCount: " + partialCount + ", totalCount: " + totalCount); this.ctx.Emit(new Values(CurrentTimeMillis(), totalCount)); partialCount = 0L; } } else { //Merge partialCount from all EventCountPartialCountBolt var incomingPartialCount = tuple.GetLong(0); partialCount += incomingPartialCount; totalCount += incomingPartialCount; } }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { int wordCnt = 0; string fileName = tuple.GetString(0); Context.Logger.Info("PartialCount, Execute(), tuple content: {0}", fileName); using (StreamReader reader = new StreamReader(fileName)) { while (!reader.EndOfStream) { string line = reader.ReadLine(); Context.Logger.Info("read line: {0}", line); foreach (string word in line.Split(' ')) { wordCnt++; } } } Context.Logger.Info("Execute(), wordCnt: {0}", wordCnt); this.ctx.Emit(new Values(wordCnt)); }
public void Execute(SCPTuple tuple) { try { count++; var sb = new StringBuilder(); sb.AppendFormat("Received Tuple {0}: ", count); var values = tuple.GetValues(); for (int i = 0; i < values.Count; i++) { if (i > 0) { sb.Append(", "); } sb.AppendFormat("{0} = {1}", i, values[i].ToString()); } Context.Logger.Info(sb.ToString()); Context.Logger.Info("Tuple values as JSON: " + (values.Count == 1 ? JsonConvert.SerializeObject(values[0]) : JsonConvert.SerializeObject(values))); //Ack the tuple if enableAck is set to true in TopologyBuilder. This is mandatory if the downstream bolt or spout expects an ack. if (enableAck) { this.context.Ack(tuple); } } catch (Exception ex) { Context.Logger.Error("An error occured while executing Tuple Id: {0}. Exception Details:\r\n{1}", tuple.GetTupleId(), ex.ToString()); //Fail the tuple if enableAck is set to true in TopologyBuilder so that the tuple is replayed. if (enableAck) { this.context.Fail(tuple); } } }
//Process a tuple from the stream public void Execute(SCPTuple tuple) { Context.Logger.Info("Execute enter"); var tweetCount = (long)tuple.GetValue(0); var tweet = tuple.GetValue(1) as string; try { //Only send updates every 500 milliseconds //Ignore the messages in between so that you don't overload the SignalR website with updates at each tuple //If you have only aggreagates to send that can be spaced, you don't need this timer if (timer.ElapsedMilliseconds >= 100) { SendSingnalRUpdate(tweetCount, tweet); timer.Restart(); } } catch (Exception ex) { Context.Logger.Error("SignalRBroadcastBolt Exception: " + ex.Message + "\nStackTrace: \n" + ex.StackTrace); } Context.Logger.Info("Execute exit"); }
/// <summary> /// The Execute() function will be called, when a new tuple is available. /// </summary> /// <param name="tuple"></param> public void Execute(SCPTuple tuple) { if (tuple.GetSourceStreamId().Equals(Constants.SYSTEM_TICK_STREAM_ID)) { if (partialCount > 0) { Context.Logger.Info("emitting partialCount: " + partialCount + ", totalCount: " + totalCount); if (enableAck) { Context.Logger.Info("tuplesToAck: " + tuplesToAck); this.ctx.Emit(Constants.DEFAULT_STREAM_ID, tuplesToAck, new Values(partialCount)); foreach (var tupleToAck in tuplesToAck) { this.ctx.Ack(tupleToAck); } tuplesToAck.Clear(); } else { this.ctx.Emit(Constants.DEFAULT_STREAM_ID, new Values(partialCount)); } partialCount = 0L; } } else { partialCount++; totalCount++; //Do no ack here but add to the acking queue if (enableAck) { tuplesToAck.Enqueue(tuple); } } }
public void Execute(SCPTuple tuple) { string cowBell = tuple.GetString(0); this.ctx.Emit(new Values(cowBell)); }