public void collector_put(KinesisRecord kr) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.collector_put"); PutRecordsRequest prr = this.collector_.put(kr); if (null != prr) { this.send_put_records_request(prr); } }
private void handle_flush(PutRecordsRequest prr) { if (prr == null) { return; } decrease_buffered_data(prr); flush_callback_(prr); }
private void send_put_records_request(PutRecordsRequest prr) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.send_put_records_request called."); if (config_.clientType == KPLNETInterface.ClientType.SocketClient) { HttpClientSendPutRecordsRequest(prr); } else { KinesisClientSendPutRecordsRequest(prr); } }
private void KinesisClientSendPutRecordsRequest(PutRecordsRequest prr) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.send_put_records_request"); try { StdErrorOut.Instance.StdOut(LogLevel.debug, "before RequestSigner.sign_v4(request, sig_v4_ctx_)"); this.kinesis_client_.PutRecordsRequest(prr, (result) => this.retrier_put(result), (object)prr, prr.Deadline(), prr.Expiration()); StdErrorOut.Instance.StdOut(LogLevel.debug, "http_client_.put"); } catch (Exception ex) { this.retrier_.put(new AwsHttpResult(ex.ToString(), (object)prr, DateTime.Now, DateTime.Now)); } }
private void decrease_buffered_data(PutRecordsRequest prr) { if (prr == null) { return; } foreach (var kr in prr.Items()) { var shard_id = kr.Items()[0].Predicted_shard(); if (shard_id != -1 && buffered_data_.Keys.Contains(shard_id)) { buffered_data_[shard_id] -= kr.accurate_size(); } } }
private void HttpClientSendPutRecordsRequest(PutRecordsRequest prr) { StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.send_put_records_request"); AwsHttpRequest kinesisRequest = AwsHttp.create_kinesis_request(this.region_, "PutRecords", prr.serialize()); try { RequestSigner.sign_v4(kinesisRequest, this.sig_v4_ctx_); StdErrorOut.Instance.StdOut(LogLevel.debug, "after RequestSigner.sign_v4(request, sig_v4_ctx_)"); this.http_client_.put(kinesisRequest, (Action <AwsHttpResult>)(result => this.retrier_put(result)), (object)prr, prr.Deadline(), prr.Expiration()); StdErrorOut.Instance.StdOut(LogLevel.debug, "http_client_.put"); } catch (Exception ex) { this.retrier_.put(new AwsHttpResult(ex.ToString(), (object)prr, DateTime.Now, DateTime.Now)); } }
void emit_metrics(AwsHttpResult result) { MetricsPutter metrics_putter = new MetricsPutter(metrics_manager_, result); PutRecordsRequest prr = result.context <PutRecordsRequest>(); double num_urs = 0; foreach (var kr in prr.Items()) { metrics_putter.put(Names.UserRecordsPerKinesisRecord, kr.Items().Count, (ulong)kr.Items()[kr.Items().Count - 1].Predicted_shard()); num_urs += kr.Items().Count; } metrics_putter.put (Names.RequestTime, result.duration_millis()).put (Names.KinesisRecordsPerPutRecordsRequest, prr.Items().Count).put (Names.UserRecordsPerPutRecordsRequest, num_urs); string err_code = null; if (result.successful()) { var status_code = result.status_code(); if (status_code != 200) { // TODO parse the json (if any) to get the error code err_code = "Http" + status_code; } } else { err_code = result.error().Substring(0, 255); } if (err_code != null) { metrics_putter.put (Names.ErrorsByCode, 1, 0, err_code).put (Names.AllErrors, 1); } }