public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation.");
try
{
this.region_ = region;
this.config_ = config;
this.executor_ = executor;
this.http_client_ = http_client;
this.kinesis_client_ = kinesis_client;
this.metrics_manager_ = metrics_manager;
this.finish_user_record_cb_ = finish_user_record_cb;
this.sig_v4_ctx_ = new SigV4Context(this.region_, "kinesis", creds_provider);
this.shard_map_ = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000);
StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation.");
this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_);
this.limiter_ = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_);
this.collector_ = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_);
this.retrier_ = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_);
this.outstanding_user_records_ = 0;
StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation.");
}
catch (Exception ex)
{
StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString());
}
}
void emit_metrics(UserRecord ur)
{
if (ur.Attempts().Count == 0)
{
return;
}
bool successful = ur.Attempts()[ur.Attempts().Count - 1].Success();
string shard_id = "";
if (successful)
{
shard_id = ur.Attempts()[ur.Attempts().Count - 1].Shard_id();
}
else if (ur.Predicted_shard() != -1)
{
shard_id = ShardMap.shard_id_to_str((ulong)ur.Predicted_shard());
}
Action <string, double> put = (name, val) =>
{
var f = metrics_manager_.finder().set_name(name).set_stream(ur.Stream());
if (!string.IsNullOrEmpty(shard_id))
{
f.set_shard(shard_id);
}
f.find().Put(val);
};
if (successful)
{
put(Names.UserRecordsPut, 1);
put(Names.UserRecordsDataPut, ur.Data().Length);
}
else
{
put(Names.UserRecordsPut, 0);
if (ur.Attempts()[ur.Attempts().Count - 1].Error_code() == "Expired")
{
put(Names.UserRecordExpired, 1);
}
}
var last = ur.Arrival();
foreach (var a in ur.Attempts())
{
put(Names.BufferingTime, (a.Start() - last).TotalMilliseconds);
last = a.End();
}
put(Names.RetriesPerRecord, ur.Attempts().Count - 1);
}
public void succeed_if_correct_shard(UserRecord ur, DateTime start, DateTime end, string shard_id, string sequence_number)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "succeed_if_correct_shard");
if (ur.Predicted_shard() != -1 && ur.Predicted_shard() != ShardMap.shard_id_from_str(shard_id))
{
StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Record went to shard {0} instead of the prediceted shard {1}; this usually means the sharp map has changed.", shard_id, ur.Predicted_shard()));
shard_map_invalidate_cb_(start);
retry_not_expired(ur, start, end, "Wrong Shard", "Record did not end up in expected shard.");
}
else
{
finish_user_record(ur, new Attempt().set_start(start).set_end(end).set_result(shard_id, sequence_number));
}
}
public Aggregator
(
Executor executor,
ShardMap shard_map,
Action <KinesisRecord> deadline_callback,
KPLNETConfiguration config,
MetricsManager metrics_manager = null
)
{
executor_ = executor;
shard_map_ = shard_map;
deadline_callback_ = deadline_callback;
config_ = config;
metrics_manager_ = metrics_manager ?? new NullMetricsManager();
reducers_ = new ReducerMap();
}
public MetricsPutter put(
string name,
double val,
ulong shard_id = 0,
string err_code = null)
{
var f = metrics_manager.finder().set_name(name);
if (!string.IsNullOrEmpty(err_code))
{
f.set_error_code(err_code);
}
f.set_stream(stream);
if (shard_id != 0)
{
f.set_shard(ShardMap.shard_id_to_str(shard_id));
}
f.find().Put(val);
return(this);
}
