private void aggregator_put(UserRecord ur) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.aggregator_put. aggregator_ NULL = " + (this.aggregator_ == null ? "true" : "false")); KinesisRecord kr = this.aggregator_.put(ur); if (kr != null) { this.limiter_put(kr); } }
void finish_user_record(UserRecord ur, Attempt final_attempt) { if (!ur.Finished()) { StdErrorOut.Instance.StdOut(LogLevel.debug, "finish_user_record -> if (!ur.Finished())"); ur.add_attempt(final_attempt); //emit_metrics(ur); StdErrorOut.Instance.StdOut(LogLevel.debug, "finish_cb_(ur)"); finish_cb_(ur); } }
void on_put_record(Message m) { var ur = new UserRecord(m); ur.set_deadline_from_now((long)config_.recordMaxBufferedTime); ur.set_expiration_from_now((long)config_.recordTtl); var stream = ur.Stream(); var pipeline = GetPipeline(stream); pipeline.put(ur); }
void emit_metrics(UserRecord ur) { if (ur.Attempts().Count == 0) { return; } bool successful = ur.Attempts()[ur.Attempts().Count - 1].Success(); string shard_id = ""; if (successful) { shard_id = ur.Attempts()[ur.Attempts().Count - 1].Shard_id(); } else if (ur.Predicted_shard() != -1) { shard_id = ShardMap.shard_id_to_str((ulong)ur.Predicted_shard()); } Action <string, double> put = (name, val) => { var f = metrics_manager_.finder().set_name(name).set_stream(ur.Stream()); if (!string.IsNullOrEmpty(shard_id)) { f.set_shard(shard_id); } f.find().Put(val); }; if (successful) { put(Names.UserRecordsPut, 1); put(Names.UserRecordsDataPut, ur.Data().Length); } else { put(Names.UserRecordsPut, 0); if (ur.Attempts()[ur.Attempts().Count - 1].Error_code() == "Expired") { put(Names.UserRecordExpired, 1); } } var last = ur.Arrival(); foreach (var a in ur.Attempts()) { put(Names.BufferingTime, (a.Start() - last).TotalMilliseconds); last = a.End(); } put(Names.RetriesPerRecord, ur.Attempts().Count - 1); }
public void put(UserRecord ur) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.put called."); try { this.outstanding_user_records_ += 1; this.aggregator_put(ur); } catch (Exception ex) { StdErrorOut.Instance.StdError("pipeline put failed.", ex); } }
public void succeed_if_correct_shard(UserRecord ur, DateTime start, DateTime end, string shard_id, string sequence_number) { StdErrorOut.Instance.StdOut(LogLevel.debug, "succeed_if_correct_shard"); if (ur.Predicted_shard() != -1 && ur.Predicted_shard() != ShardMap.shard_id_from_str(shard_id)) { StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Record went to shard {0} instead of the prediceted shard {1}; this usually means the sharp map has changed.", shard_id, ur.Predicted_shard())); shard_map_invalidate_cb_(start); retry_not_expired(ur, start, end, "Wrong Shard", "Record did not end up in expected shard."); } else { finish_user_record(ur, new Attempt().set_start(start).set_end(end).set_result(shard_id, sequence_number)); } }
void retry_not_expired(UserRecord ur, DateTime start, DateTime end, string err_code, string err_msg) { ur.add_attempt(new Attempt().set_start(start).set_end(end).set_error(err_code, err_msg)); if (ur.expired()) { fail(ur, DateTime.Now, DateTime.Now, "Expired", "Record has reached expiration"); } else { // TimeSensitive automatically sets the deadline to the expiration if the given deadline is later than the expiration. ur.set_deadline_from_now((long)config_.recordMaxBufferedTime / 2); retry_cb_(ur); } }
public void fail(UserRecord ur, DateTime start, DateTime end, string err_code, string err_msg) { finish_user_record(ur, new Attempt().set_start(start).set_end(end).set_error(err_code, err_msg)); }
public void finish_user_record(UserRecord ur) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.finish_user_record"); this.finish_user_record_cb_(ur); --this.outstanding_user_records_; }