Ejemplo n.º 1
0
 public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb)
 {
     StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation.");
     try
     {
         this.region_                = region;
         this.config_                = config;
         this.executor_              = executor;
         this.http_client_           = http_client;
         this.kinesis_client_        = kinesis_client;
         this.metrics_manager_       = metrics_manager;
         this.finish_user_record_cb_ = finish_user_record_cb;
         this.sig_v4_ctx_            = new SigV4Context(this.region_, "kinesis", creds_provider);
         this.shard_map_             = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000);
         StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation.");
         this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_);
         this.limiter_    = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_);
         this.collector_  = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_);
         this.retrier_    = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_);
         this.outstanding_user_records_ = 0;
         StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation.");
     }
     catch (Exception ex)
     {
         StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString());
     }
 }
Ejemplo n.º 2
0
 public ShardMap(
     Executor executor,
     KPLNETConfiguration config,
     AwsHttpClient http_client,
     AwsKinesisClient kinesis_client,
     AwsCredentialsProvider creds,
     string region,
     string stream,
     MetricsManager metrics_manager,
     int min_backoff = kMinBackoff,
     int max_backoff = kMaxBackoff)
 {
     this.executor        = executor;
     this.http_client     = http_client;
     this.kinesis_client  = kinesis_client;
     this.creds_provider  = creds;
     this.config          = config;
     this.region          = region;
     this.stream          = stream;
     this.metrics_manager = metrics_manager;
     this.state           = State.INVALID;
     this.min_backoff     = min_backoff;
     this.max_backoff     = max_backoff;
     this.backoff         = min_backoff;
     update();
 }
Ejemplo n.º 3
0
 void create_kinesis_client()
 {
     kinesis_client_ = new AwsKinesisClient(
         executor_,
         config_.AWSCredentials,
         config_.region,
         config_.minConnections,
         config_.maxConnections,
         config_.connectTimeout,
         config_.requestTimeout);
 }