//http://docs.aws.amazon.com/firehose/latest/dev/limits.html public KinesisFirehoseSink(IPlugInContext context, IAmazonKinesisFirehose firehoseClient ) : base(context, 1, 500, 4 * 1024 * 1024) { if (_count > 500) { throw new ArgumentException("The maximum buffer size for firehose is 500"); } //Set Defaults if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond)) { _maxRecordsPerSecond = 5000; } if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond)) { _maxBytesPerSecond = 5 * 1024 * 1024; } _firehoseClient = firehoseClient; _deliveryStreamName = ResolveVariables(_config["StreamName"]); _throttle = new AdaptiveThrottle( new TokenBucket[] { new TokenBucket(1, _maxRecordsPerSecond / 200), //Number of API calls per second. For simplicity, we tie to _maxRecordsPerSecond new TokenBucket(_count, _maxRecordsPerSecond), new TokenBucket(_maxBatchSize, _maxBytesPerSecond) }, _backoffFactor, _recoveryFactor, _minRateAdjustmentFactor); }
public KinesisStreamSink( IPlugInContext context, IAmazonKinesis kineisClient ) : base(context, 1, 500, 5 * 1024 * 1024) { if (_count > 500) { throw new ArgumentException("The maximum buffer size for kinesis stream is 500"); } //Set Defaults for 1 shard if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond)) { _maxRecordsPerSecond = 1000; } if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond)) { _maxBytesPerSecond = 1024 * 1024; } _kinesisClient = kineisClient; _streamName = ResolveVariables(_config["StreamName"]); _throttle = new AdaptiveThrottle( new TokenBucket[] { new TokenBucket(_count, _maxRecordsPerSecond), new TokenBucket(_maxBatchSize, _maxBytesPerSecond) }, _backoffFactor, _recoveryFactor, _minRateAdjustmentFactor); }
public FileSystemEventSink(IPlugInContext context, int requestRate, int defaultInterval, int defaultRecordCount, long maxBatchSize) : base(context, defaultInterval, defaultRecordCount, maxBatchSize) { // If the user hasn't specified a FilePath, generate one based on the Id of the sink, or the name if it doesn't have a value. FilePath = _context.Configuration["FilePath"] ?? Path.Combine(Path.GetTempPath(), (Id ?? nameof(FileSystemEventSink)) + ".txt"); Throttle = new AdaptiveThrottle( new TokenBucket(1, requestRate), _backoffFactor, _recoveryFactor, _minRateAdjustmentFactor); }
public void TestAdaptiveThrottle() { AdaptiveThrottle throttle = new AdaptiveThrottle(new TokenBucket(1000, 2000), 1.0d / 2, 1.0d / 2, 1.0d / 8); throttle.SetError(); Assert.Equal(1.0d / 2, throttle.RateAdjustmentFactor); throttle.SetError(); throttle.SetError(); throttle.SetError(); double rateAdjustmentFactor = throttle.RateAdjustmentFactor; //Should backoff Assert.Equal(1.0d / 8, rateAdjustmentFactor); throttle.SetSuccess(); //Should stay Assert.Equal(rateAdjustmentFactor, throttle.RateAdjustmentFactor); long millisecondsDelay = throttle.GetDelayMilliseconds(1200); //Should be throttled Assert.True(millisecondsDelay > 0); //Should recover Assert.True(throttle.RateAdjustmentFactor > rateAdjustmentFactor); }