public Pipeline(string region, string stream, KPLNETConfiguration config, Executor executor, AwsHttpClient http_client, AwsKinesisClient kinesis_client, AwsCredentialsProvider creds_provider, MetricsManager metrics_manager, Action <UserRecord> finish_user_record_cb)
{
StdErrorOut.Instance.StdOut(LogLevel.debug, "started pipeline creation.");
try
{
this.region_ = region;
this.config_ = config;
this.executor_ = executor;
this.http_client_ = http_client;
this.kinesis_client_ = kinesis_client;
this.metrics_manager_ = metrics_manager;
this.finish_user_record_cb_ = finish_user_record_cb;
this.sig_v4_ctx_ = new SigV4Context(this.region_, "kinesis", creds_provider);
this.shard_map_ = new ShardMap(executor, config, http_client, kinesis_client, creds_provider, this.region_, stream, this.metrics_manager_, 1000, 30000);
StdErrorOut.Instance.StdOut(LogLevel.debug, "after shard_map_ pipeline creation.");
this.aggregator_ = new Aggregator(this.executor_, this.shard_map_, (Action <KinesisRecord>)(kr => this.limiter_put(kr)), this.config_, this.metrics_manager_);
this.limiter_ = new Limiter(this.executor_, (Action <KinesisRecord>)(kr => this.collector_put(kr)), (Action <KinesisRecord>)(kr => this.retrier_put_kr(kr)), this.config_);
this.collector_ = new Collector(this.executor_, (Action <PutRecordsRequest>)(prr => this.send_put_records_request(prr)), this.config_, this.metrics_manager_);
this.retrier_ = new Retrier(this.config_, (Action <UserRecord>)(ur => this.finish_user_record(ur)), (Action <UserRecord>)(ur => this.aggregator_put(ur)), (Action <DateTime>)(tp => this.shard_map_.invalidate(tp)), (Action <string, string>)((code, msg) => this.limiter_.add_error(code, msg)), this.metrics_manager_);
this.outstanding_user_records_ = 0;
StdErrorOut.Instance.StdOut(LogLevel.debug, "done pipeline creation.");
}
catch (Exception ex)
{
StdErrorOut.Instance.StdOut(LogLevel.error, ex.ToString());
}
}
public ShardMap(
Executor executor,
KPLNETConfiguration config,
AwsHttpClient http_client,
AwsKinesisClient kinesis_client,
AwsCredentialsProvider creds,
string region,
string stream,
MetricsManager metrics_manager,
int min_backoff = kMinBackoff,
int max_backoff = kMaxBackoff)
{
this.executor = executor;
this.http_client = http_client;
this.kinesis_client = kinesis_client;
this.creds_provider = creds;
this.config = config;
this.region = region;
this.stream = stream;
this.metrics_manager = metrics_manager;
this.state = State.INVALID;
this.min_backoff = min_backoff;
this.max_backoff = max_backoff;
this.backoff = min_backoff;
update();
}
void create_http_client()
{
var endpoint = config_.kinesisEndpoint;
var port = (int)config_.kinesisPort;
if (string.IsNullOrEmpty(endpoint))
{
endpoint = "kinesis." + region_ + ".amazonaws.com";
port = 443;
}
http_client_ = new AwsHttpClient(
executor_,
socket_factory_,
endpoint,
port,
true,
config_.verifyCertificate,
config_.minConnections,
config_.maxConnections,
config_.connectTimeout,
config_.requestTimeout);
}
public MetricsManager(Executor executor, ISocketFactory socket_factory, AwsCredentialsProvider creds, KPLNETConfiguration config, string region, string cw_namespace = "KinesisProducerLib",
Level level = Level.Detailed,
Granularity granularity = Granularity.Shard,
ExtraDimensions extra_dimensions = null,
string custom_endpoint = "",
ulong port = 443,
double upload_frequency = 1 * 60 * 1000,
double retry_frequency = 10 * 1000)
{
if (extra_dimensions == null)
{
extra_dimensions = new ExtraDimensions();
}
this.creds = creds;
this.config_ = config;
this.region = region;
this.metrics_index = new MetricsIndex();
region_ = Core.Clients.KinesisClient.AwsKinesisClient.MapRegion(config.region);
this.cw_namespace = cw_namespace;
this.socket_factory_ = socket_factory;
this.level = level;
this.granularity = granularity;
this.upload_frequency = upload_frequency;
this.retry_frequency = retry_frequency;
this.executor = executor;
this.endpoint = string.IsNullOrEmpty(custom_endpoint) ? "monitoring." + region + ".amazonaws.com" : custom_endpoint;
this.http_client = new AwsHttpClient(executor, socket_factory_, endpoint, (int)port, true, string.IsNullOrEmpty(custom_endpoint));
retryable_requests = new List <string>();
if (level != Level.None)
{
StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Uploading metrics to {0}:{1}", endpoint, port));
}
extra_dimensionsMap[Granularity.Global] = new Dimensions();
extra_dimensionsMap[Granularity.Stream] = new Dimensions();
extra_dimensionsMap[Granularity.Shard] = new Dimensions();
foreach (var t in extra_dimensions)
{
extra_dimensionsMap[t.Item3].Add(new KeyValuePair <string, string>(t.Item1, t.Item2));
}
scheduled_upload = executor.Schedule(
() =>
{
try
{
scheduled_upload.reschedule(upload_frequency);
upload();
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError("Scheduled Upload failed.", ex);
}
},
upload_frequency);
scheduled_retry = executor.Schedule(
() =>
{
try
{
scheduled_retry.reschedule(retry_frequency);
retry_uploads();
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError("Scheduled Upload failed.", ex);
}
},
retry_frequency);
}
