private void aggregator_put(UserRecord ur)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.aggregator_put. aggregator_ NULL = " + (this.aggregator_ == null ? "true" : "false"));
KinesisRecord kr = this.aggregator_.put(ur);
if (kr != null)
{
this.limiter_put(kr);
}
}
void finish_user_record(UserRecord ur, Attempt final_attempt)
{
if (!ur.Finished())
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "finish_user_record -> if (!ur.Finished())");
ur.add_attempt(final_attempt);
//emit_metrics(ur);
StdErrorOut.Instance.StdOut(LogLevel.debug, "finish_cb_(ur)");
finish_cb_(ur);
}
}
void on_put_record(Message m)
{
var ur = new UserRecord(m);
ur.set_deadline_from_now((long)config_.recordMaxBufferedTime);
ur.set_expiration_from_now((long)config_.recordTtl);
var stream = ur.Stream();
var pipeline = GetPipeline(stream);
pipeline.put(ur);
}
void emit_metrics(UserRecord ur)
{
if (ur.Attempts().Count == 0)
{
return;
}
bool successful = ur.Attempts()[ur.Attempts().Count - 1].Success();
string shard_id = "";
if (successful)
{
shard_id = ur.Attempts()[ur.Attempts().Count - 1].Shard_id();
}
else if (ur.Predicted_shard() != -1)
{
shard_id = ShardMap.shard_id_to_str((ulong)ur.Predicted_shard());
}
Action <string, double> put = (name, val) =>
{
var f = metrics_manager_.finder().set_name(name).set_stream(ur.Stream());
if (!string.IsNullOrEmpty(shard_id))
{
f.set_shard(shard_id);
}
f.find().Put(val);
};
if (successful)
{
put(Names.UserRecordsPut, 1);
put(Names.UserRecordsDataPut, ur.Data().Length);
}
else
{
put(Names.UserRecordsPut, 0);
if (ur.Attempts()[ur.Attempts().Count - 1].Error_code() == "Expired")
{
put(Names.UserRecordExpired, 1);
}
}
var last = ur.Arrival();
foreach (var a in ur.Attempts())
{
put(Names.BufferingTime, (a.Start() - last).TotalMilliseconds);
last = a.End();
}
put(Names.RetriesPerRecord, ur.Attempts().Count - 1);
}
public void put(UserRecord ur)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.put called.");
try
{
this.outstanding_user_records_ += 1;
this.aggregator_put(ur);
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError("pipeline put failed.", ex);
}
}
public void succeed_if_correct_shard(UserRecord ur, DateTime start, DateTime end, string shard_id, string sequence_number)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "succeed_if_correct_shard");
if (ur.Predicted_shard() != -1 && ur.Predicted_shard() != ShardMap.shard_id_from_str(shard_id))
{
StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Record went to shard {0} instead of the prediceted shard {1}; this usually means the sharp map has changed.", shard_id, ur.Predicted_shard()));
shard_map_invalidate_cb_(start);
retry_not_expired(ur, start, end, "Wrong Shard", "Record did not end up in expected shard.");
}
else
{
finish_user_record(ur, new Attempt().set_start(start).set_end(end).set_result(shard_id, sequence_number));
}
}
void retry_not_expired(UserRecord ur, DateTime start, DateTime end, string err_code, string err_msg)
{
ur.add_attempt(new Attempt().set_start(start).set_end(end).set_error(err_code, err_msg));
if (ur.expired())
{
fail(ur, DateTime.Now, DateTime.Now, "Expired", "Record has reached expiration");
}
else
{
// TimeSensitive automatically sets the deadline to the expiration if the given deadline is later than the expiration.
ur.set_deadline_from_now((long)config_.recordMaxBufferedTime / 2);
retry_cb_(ur);
}
}
public void fail(UserRecord ur, DateTime start, DateTime end, string err_code, string err_msg)
{
finish_user_record(ur, new Attempt().set_start(start).set_end(end).set_error(err_code, err_msg));
}
public void finish_user_record(UserRecord ur)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.finish_user_record");
this.finish_user_record_cb_(ur);
--this.outstanding_user_records_;
}