예제 #1
0
        void emit_metrics(UserRecord ur)
        {
            if (ur.Attempts().Count == 0)
            {
                return;
            }

            bool   successful = ur.Attempts()[ur.Attempts().Count - 1].Success();
            string shard_id   = "";

            if (successful)
            {
                shard_id = ur.Attempts()[ur.Attempts().Count - 1].Shard_id();
            }
            else if (ur.Predicted_shard() != -1)
            {
                shard_id = ShardMap.shard_id_to_str((ulong)ur.Predicted_shard());
            }

            Action <string, double> put = (name, val) =>
            {
                var f = metrics_manager_.finder().set_name(name).set_stream(ur.Stream());
                if (!string.IsNullOrEmpty(shard_id))
                {
                    f.set_shard(shard_id);
                }
                f.find().Put(val);
            };

            if (successful)
            {
                put(Names.UserRecordsPut, 1);
                put(Names.UserRecordsDataPut, ur.Data().Length);
            }
            else
            {
                put(Names.UserRecordsPut, 0);
                if (ur.Attempts()[ur.Attempts().Count - 1].Error_code() == "Expired")
                {
                    put(Names.UserRecordExpired, 1);
                }
            }

            var last = ur.Arrival();

            foreach (var a in ur.Attempts())
            {
                put(Names.BufferingTime, (a.Start() - last).TotalMilliseconds);
                last = a.End();
            }

            put(Names.RetriesPerRecord, ur.Attempts().Count - 1);
        }
예제 #2
0
        public void succeed_if_correct_shard(UserRecord ur, DateTime start, DateTime end, string shard_id, string sequence_number)
        {
            StdErrorOut.Instance.StdOut(LogLevel.debug, "succeed_if_correct_shard");
            if (ur.Predicted_shard() != -1 && ur.Predicted_shard() != ShardMap.shard_id_from_str(shard_id))
            {
                StdErrorOut.Instance.StdOut(LogLevel.warn, string.Format("Record went to shard {0} instead of the prediceted shard {1}; this  usually means the sharp map has changed.", shard_id, ur.Predicted_shard()));
                shard_map_invalidate_cb_(start);

                retry_not_expired(ur, start, end, "Wrong Shard", "Record did not end up in expected shard.");
            }
            else
            {
                finish_user_record(ur, new Attempt().set_start(start).set_end(end).set_result(shard_id, sequence_number));
            }
        }
예제 #3
0
        public KinesisRecord put(UserRecord ur)
        {
            // If shard map is not available, or aggregation is disabled, just send the record by itself, and do not attempt to aggrgegate.
            long       shard_id = -1;
            BigInteger hk       = 0;

            StdErrorOut.Instance.StdOut(LogLevel.debug, "Aggregator.put -> Called");
            if (config_.aggregationEnabled && shard_map_ != null)
            {
                hk       = ur.Hash_key();
                shard_id = shard_map_.Shard_id(hk);

                StdErrorOut.Instance.StdOut(LogLevel.debug, "hk = " + hk + "    shard_id= " + shard_id);
            }
            if (-1 == shard_id)
            {
                StdErrorOut.Instance.StdOut(LogLevel.debug, "if (-1 == shard_id)");
                var kr = new KinesisRecord();
                kr.add(ur);
                return(kr);
            }
            else
            {
                ur.Predicted_shard(shard_id);
                return(GetReducer(shard_id).add(ur) as KinesisRecord);
            }
        }