public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation.");
try
{
this.region_ = region;
this.config_ = config;
this.executor_ = executor;
this.http_client_ = http_client;
this.kinesis_client_ = kinesis_client;
this.metrics_manager_ = metrics_manager;
this.finish_user_record_cb_ = finish_user_record_cb;
this.sig_v4_ctx_ = new SigV4Context(this.region_, "kinesis", creds_provider);
this.shard_map_ = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000);
StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation.");
this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_);
this.limiter_ = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_);
this.collector_ = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_);
this.retrier_ = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_);
this.outstanding_user_records_ = 0;
StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation.");
}
catch (Exception ex)
{
StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString());
}
}
void upload_with_path(string path)
{
AwsHttpRequest req = new AwsHttpRequest("POST", path);
req.add_header("Host", endpoint);
req.add_header("Content-Length", "0");
SigV4Context ctx = new SigV4Context(region, "monitoring", creds);
try
{
RequestSigner.sign_v4(req, ctx);
http_client.put(req, (r) => { handle_result(r); }, path, DateTime.Now, DateTime.Now);
}
catch (Exception e)
{
handle_result(new AwsHttpResult(e.ToString(), path, DateTime.Now, DateTime.Now));
}
}
private void HttpClientUpdate(string start_shard_id)
{
StringBuilder sb = new StringBuilder();
sb.Append("{\"StreamName\":\"").Append(stream).Append("\"");
if (!string.IsNullOrEmpty(start_shard_id))
{
sb.Append(",\"ExclusiveStartShardId\":\"").Append(start_shard_id).Append("\"");
}
sb.Append("}");
var req = AwsHttp.create_kinesis_request(region, "DescribeStream", sb.ToString());
var ctx = new SigV4Context(region, "kinesis", creds_provider);
try
{
RequestSigner.sign_v4(req, ctx);
http_client.put(req, (r) => { update_callback(r); }, null, DateTime.Now, DateTime.Now);
}
catch (Exception e)
{
update_callback(new AwsKinesisResult(e.ToString(), null, DateTime.Now, DateTime.Now));
}
}
