//http://docs.aws.amazon.com/firehose/latest/dev/limits.html
public KinesisFirehoseSink(IPlugInContext context,
IAmazonKinesisFirehose firehoseClient
) : base(context, 1, 500, 4 * 1024 * 1024)
{
if (_count > 500)
{
throw new ArgumentException("The maximum buffer size for firehose is 500");
}
//Set Defaults
if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond))
{
_maxRecordsPerSecond = 5000;
}
if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond))
{
_maxBytesPerSecond = 5 * 1024 * 1024;
}
_firehoseClient = firehoseClient;
_deliveryStreamName = ResolveVariables(_config["StreamName"]);
_throttle = new AdaptiveThrottle(
new TokenBucket[]
{
new TokenBucket(1, _maxRecordsPerSecond / 200), //Number of API calls per second. For simplicity, we tie to _maxRecordsPerSecond
new TokenBucket(_count, _maxRecordsPerSecond),
new TokenBucket(_maxBatchSize, _maxBytesPerSecond)
},
_backoffFactor,
_recoveryFactor,
_minRateAdjustmentFactor);
}
public KinesisStreamSink(
IPlugInContext context,
IAmazonKinesis kineisClient
) : base(context, 1, 500, 5 * 1024 * 1024)
{
if (_count > 500)
{
throw new ArgumentException("The maximum buffer size for kinesis stream is 500");
}
//Set Defaults for 1 shard
if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond))
{
_maxRecordsPerSecond = 1000;
}
if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond))
{
_maxBytesPerSecond = 1024 * 1024;
}
_kinesisClient = kineisClient;
_streamName = ResolveVariables(_config["StreamName"]);
_throttle = new AdaptiveThrottle(
new TokenBucket[]
{
new TokenBucket(_count, _maxRecordsPerSecond),
new TokenBucket(_maxBatchSize, _maxBytesPerSecond)
},
_backoffFactor,
_recoveryFactor,
_minRateAdjustmentFactor);
}
public FileSystemEventSink(IPlugInContext context, int requestRate, int defaultInterval, int defaultRecordCount, long maxBatchSize)
: base(context, defaultInterval, defaultRecordCount, maxBatchSize)
{
// If the user hasn't specified a FilePath, generate one based on the Id of the sink, or the name if it doesn't have a value.
FilePath = _context.Configuration["FilePath"] ?? Path.Combine(Path.GetTempPath(), (Id ?? nameof(FileSystemEventSink)) + ".txt");
Throttle = new AdaptiveThrottle(
new TokenBucket(1, requestRate),
_backoffFactor,
_recoveryFactor,
_minRateAdjustmentFactor);
}
public void TestAdaptiveThrottle()
{
AdaptiveThrottle throttle = new AdaptiveThrottle(new TokenBucket(1000, 2000), 1.0d / 2, 1.0d / 2, 1.0d / 8);
throttle.SetError();
Assert.Equal(1.0d / 2, throttle.RateAdjustmentFactor);
throttle.SetError();
throttle.SetError();
throttle.SetError();
double rateAdjustmentFactor = throttle.RateAdjustmentFactor;
//Should backoff
Assert.Equal(1.0d / 8, rateAdjustmentFactor);
throttle.SetSuccess();
//Should stay
Assert.Equal(rateAdjustmentFactor, throttle.RateAdjustmentFactor);
long millisecondsDelay = throttle.GetDelayMilliseconds(1200);
//Should be throttled
Assert.True(millisecondsDelay > 0);
//Should recover
Assert.True(throttle.RateAdjustmentFactor > rateAdjustmentFactor);
}
