示例#1
0
 void fail(KinesisRecord kr, DateTime start, DateTime end, string err_code, string err_msg)
 {
     foreach (var ur in kr.Items())
     {
         fail(ur, start, end, err_code, err_msg);
     }
 }
示例#2
0
        public PutRecordsRequest put(KinesisRecord kr)
        {
            var prr = reducer_.add(kr) as PutRecordsRequest;

            decrease_buffered_data(prr);
            return(prr);
        }
示例#3
0
        public KinesisRecord put(UserRecord ur)
        {
            // If shard map is not available, or aggregation is disabled, just send the record by itself, and do not attempt to aggrgegate.
            long       shard_id = -1;
            BigInteger hk       = 0;

            StdErrorOut.Instance.StdOut(LogLevel.debug, "Aggregator.put -> Called");
            if (config_.aggregationEnabled && shard_map_ != null)
            {
                hk       = ur.Hash_key();
                shard_id = shard_map_.Shard_id(hk);

                StdErrorOut.Instance.StdOut(LogLevel.debug, "hk = " + hk + "    shard_id= " + shard_id);
            }
            if (-1 == shard_id)
            {
                StdErrorOut.Instance.StdOut(LogLevel.debug, "if (-1 == shard_id)");
                var kr = new KinesisRecord();
                kr.add(ur);
                return(kr);
            }
            else
            {
                ur.Predicted_shard(shard_id);
                return(GetReducer(shard_id).add(ur) as KinesisRecord);
            }
        }
示例#4
0
 void retry_not_expired(KinesisRecord kr, DateTime start, DateTime end, string err_code, string err_msg)
 {
     foreach (var ur in kr.Items())
     {
         retry_not_expired(ur, start, end, err_code, err_msg);
     }
 }
示例#5
0
        public void collector_put(KinesisRecord kr)
        {
            StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.collector_put");
            PutRecordsRequest prr = this.collector_.put(kr);

            if (null != prr)
            {
                this.send_put_records_request(prr);
            }
        }
示例#6
0
        private void aggregator_put(UserRecord ur)
        {
            StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.aggregator_put. aggregator_ NULL = " + (this.aggregator_ == null ? "true" : "false"));
            KinesisRecord kr = this.aggregator_.put(ur);

            if (kr != null)
            {
                this.limiter_put(kr);
            }
        }
示例#7
0
        // We don't want any individual shard to accumulate too much data
        // because that makes traffic to that shard bursty, and might cause
        // throttling, so we flush whenever a shard reaches a certain limit.
        private bool should_flush(KinesisRecord kr)
        {
            var shard_id = kr.Items()[0].Predicted_shard();

            if (shard_id != -1 && buffered_data_.Keys.Contains(shard_id))
            {
                var d = (buffered_data_[shard_id] += kr.accurate_size());
                if (d >= 256 * 1024)
                {
                    return(true);
                }
            }
            return(false);
        }
示例#8
0
        public void put(KinesisRecord kr)
        {
            var shard_id = kr.Items()[kr.Items().Count - 1].Predicted_shard();

            // Limiter doesn't work if we don't know which shard the record is going to
            if (shard_id == -1)
            {
                callback_(kr);
            }
            else
            {
                GetShardLimiter(shard_id).put(kr, callback_, expired_callback_);
            }
        }
示例#9
0
        public void put(KinesisRecord kr, string err_code, string err_msg)
        {
            var now = DateTime.Now;

            retry_not_expired(kr, now, now, err_code, err_msg);
        }
示例#10
0
 private void limiter_put(KinesisRecord kr)
 {
     StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.limiter_put");
     this.limiter_.put(kr);
 }
示例#11
0
 private void retrier_put_kr(KinesisRecord kr)
 {
     StdErrorOut.Instance.StdOut(LogLevel.debug, "Pipeline.retrier_put_kr");
     this.executor_.Submit((Action)(() => { try { this.retrier_.put(kr, "Expired", "Expiration reached while waiting in limiter"); } catch (Exception e) { StdErrorOut.Instance.StdError("pipeline retrier_put_kr failed", e); } }));
 }
示例#12
0
 public void put(KinesisRecord incoming, Callback callback, Callback expired_callback)
 {
     queue_.Enqueue(incoming);
     drain(callback, expired_callback);
 }