public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb) { StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation."); try { this.region_ = region; this.config_ = config; this.executor_ = executor; this.http_client_ = http_client; this.kinesis_client_ = kinesis_client; this.metrics_manager_ = metrics_manager; this.finish_user_record_cb_ = finish_user_record_cb; this.sig_v4_ctx_ = new SigV4Context(this.region_, "kinesis", creds_provider); this.shard_map_ = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000); StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation."); this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_); this.limiter_ = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_); this.collector_ = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_); this.retrier_ = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_); this.outstanding_user_records_ = 0; StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation."); } catch (Exception ex) { StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString()); } }
public ShardMap( Executor executor, KPLNETConfiguration config, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds, string region, string stream, MetricsManager metrics_manager, int min_backoff = kMinBackoff, int max_backoff = kMaxBackoff) { this.executor = executor; this.http_client = http_client; this.kinesis_client = kinesis_client; this.creds_provider = creds; this.config = config; this.region = region; this.stream = stream; this.metrics_manager = metrics_manager; this.state = State.INVALID; this.min_backoff = min_backoff; this.max_backoff = max_backoff; this.backoff = min_backoff; update(); }
void create_kinesis_client() { kinesis_client_ = new AwsKinesisClient( executor_, config_.AWSCredentials, config_.region, config_.minConnections, config_.maxConnections, config_.connectTimeout, config_.requestTimeout); }