public KinesisStreamManager(
IAmazonKinesis amazonKinesisClient,
string streamName)
{
_AmazonKinesisClient = amazonKinesisClient;
_StreamName = streamName;
}
public KinesisStreamCreator(IAwsClientFactory clientFactory, string rawStreamName, string measurementsStreamName, ILogger logger)
{
_client = clientFactory.CreateKinesisClient();
_rawStreamName = rawStreamName;
_measurementsStreamName = measurementsStreamName;
_logger = logger;
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="options"></param>
/// <param name="kinesisClient"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException">A required parameter is null.</exception>
public static LoggerConfiguration AmazonKinesis(
this LoggerSinkConfiguration loggerConfiguration,
KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient)
{
if (loggerConfiguration == null)
{
throw new ArgumentNullException("loggerConfiguration");
}
if (options == null)
{
throw new ArgumentNullException("options");
}
if (kinesisClient == null)
{
throw new ArgumentNullException("kinesisClient");
}
ILogEventSink sink;
if (options.BufferBaseFilename == null)
{
sink = new KinesisSink(options, kinesisClient);
}
else
{
sink = new DurableKinesisSink(options, kinesisClient);
}
return(loggerConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum));
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="kinesisClient"></param>
/// <param name="streamName"></param>
/// <param name="shardCount"></param>
/// <param name="bufferBaseFilename"></param>
/// <param name="bufferFileSizeLimitBytes"></param>
/// <param name="batchPostingLimit"></param>
/// <param name="period"></param>
/// <param name="minimumLogEventLevel"></param>
/// <param name="onLogSendError"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException"></exception>
public static LoggerConfiguration AmazonKinesis(
this LoggerSinkConfiguration loggerConfiguration,
IAmazonKinesis kinesisClient,
string streamName,
int? shardCount = null,
string bufferBaseFilename = null,
int? bufferFileSizeLimitBytes = null,
int? batchPostingLimit = null,
TimeSpan? period = null,
LogEventLevel? minimumLogEventLevel = null,
EventHandler<LogSendErrorEventArgs> onLogSendError = null)
{
if (streamName == null) throw new ArgumentNullException("streamName");
var options = new KinesisStreamSinkOptions(streamName: streamName, shardCount: shardCount)
{
BufferFileSizeLimitBytes = bufferFileSizeLimitBytes,
BufferBaseFilename = bufferBaseFilename,
Period = period ?? KinesisStreamSinkOptions.DefaultPeriod,
BatchPostingLimit = batchPostingLimit ?? KinesisStreamSinkOptions.DefaultBatchPostingLimit,
MinimumLogEventLevel = minimumLogEventLevel ?? LevelAlias.Minimum,
OnLogSendError = onLogSendError
};
return AmazonKinesis(loggerConfiguration, options, kinesisClient);
}
/// <summary>
/// Construct a sink posting to the specified database.
/// </summary>
/// <param name="options">Options for configuring how the sink behaves, may NOT be null.</param>
/// <param name="kinesisClient"></param>
public KinesisSink(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) :
base(options.BatchPostingLimit, options.Period)
{
_state = new KinesisSinkState(options, kinesisClient);
_minimumAcceptedLevel = _state.Options.MinimumLogEventLevel;
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="kinesisClient"></param>
/// <param name="streamName"></param>
/// <param name="bufferBaseFilename"></param>
/// <param name="bufferFileSizeLimitBytes"></param>
/// <param name="batchPostingLimit"></param>
/// <param name="period"></param>
/// <param name="minimumLogEventLevel"></param>
/// <param name="onLogSendError"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException"></exception>
public static LoggerConfiguration AmazonKinesis(
this LoggerSinkConfiguration loggerConfiguration,
IAmazonKinesis kinesisClient,
string streamName,
string bufferBaseFilename = null,
int?bufferFileSizeLimitBytes = null,
int?batchPostingLimit = null,
TimeSpan?period = null,
LogEventLevel?minimumLogEventLevel = null,
EventHandler <LogSendErrorEventArgs> onLogSendError = null)
{
if (streamName == null)
{
throw new ArgumentNullException("streamName");
}
var options = new KinesisStreamSinkOptions(streamName: streamName)
{
BufferFileSizeLimitBytes = bufferFileSizeLimitBytes,
BufferBaseFilename = bufferBaseFilename == null ? null : bufferBaseFilename + ".stream",
Period = period ?? KinesisSinkOptionsBase.DefaultPeriod,
BatchPostingLimit = batchPostingLimit ?? KinesisSinkOptionsBase.DefaultBatchPostingLimit,
MinimumLogEventLevel = minimumLogEventLevel ?? LevelAlias.Minimum,
OnLogSendError = onLogSendError
};
return(AmazonKinesis(loggerConfiguration, options, kinesisClient));
}
internal KinesisSinkState(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) : base(options)
{
if (options == null) throw new ArgumentNullException("options");
if (kinesisClient == null) throw new ArgumentNullException("kinesisClient");
KinesisClient = kinesisClient;
Options = options;
}
/// <summary>
/// Construct a sink posting to the specified database.
/// </summary>
/// <param name="options">Options for configuring how the sink behaves, may NOT be null.</param>
/// <param name="kinesisClient"></param>
public KinesisSink(KinesisStreamSinkOptions options,IAmazonKinesis kinesisClient) :
base(options.BatchPostingLimit, options.Period)
{
_state = new KinesisSinkState(options,kinesisClient);
_minimumAcceptedLevel = _state.Options.MinimumLogEventLevel;
}
public KinesisStreamSink(
IPlugInContext context,
IAmazonKinesis kineisClient
) : base(context, 1, 500, 5 * 1024 * 1024)
{
if (_count > 500)
{
throw new ArgumentException("The maximum buffer size for kinesis stream is 500");
}
//Set Defaults for 1 shard
if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond))
{
_maxRecordsPerSecond = 1000;
}
if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond))
{
_maxBytesPerSecond = 1024 * 1024;
}
_kinesisClient = kineisClient;
_streamName = ResolveVariables(_config["StreamName"]);
_throttle = new AdaptiveThrottle(
new TokenBucket[]
{
new TokenBucket(_count, _maxRecordsPerSecond),
new TokenBucket(_maxBatchSize, _maxBytesPerSecond)
},
_backoffFactor,
_recoveryFactor,
_minRateAdjustmentFactor);
}
public KinesisShardReaderBase(IAmazonKinesis client,
ICheckpointRepository checkpointRepository,
IHeartbeatService heartbeatService)
{
_client = client;
CheckpointRepository = checkpointRepository;
HeartbeatService = heartbeatService;
}
public IShardReader Create(IAmazonKinesis client)
{
return(new ProcessedMeasurementsKinesisShardReader(client,
_checkpointRepository,
_heartbeatService,
_bus,
_logger));
}
public KinesisSinkState(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) : base(options)
{
if (kinesisClient == null)
{
throw new ArgumentNullException("kinesisClient");
}
KinesisClient = kinesisClient;
}
public Logic(KinesisSourceStage stage) : base(stage.Shape)
{
_settings = stage.Settings;
_out = stage.Outlet;
_kinesisClient = stage._clientFactory();
SetHandler(_out, onPull: () => _self.Tell(Pump.Instance));
}
/// <summary>
/// Writer for processed measurements stream
/// </summary>
/// <returns></returns>
public IProcessedMeasurementStreamWriter CreateWriter()
{
// Processed measurements go to this stream.
var measurementsStreamName = _awsSettings.ProcessedMeasurementsStreamName;
IAmazonKinesis client = _clientFactory.CreateKinesisClient();
return(new ProcessedMeasurementStreamWriter(client, measurementsStreamName, _logger));
}
public KinesisClient(IAmazonKinesis kinesis, string streamName)
{
if (string.IsNullOrEmpty(streamName))
{
throw new ArgumentNullException(nameof(streamName));
}
_kinesis = kinesis;
_streamName = streamName;
}
/// <summary>
/// Default constructor that Lambda will invoke.
/// </summary>
public Entrypoint()
{
this._KinesisClient = new AmazonKinesisClient();
if (!int.TryParse(Environment.GetEnvironmentVariable("WAIT_TIME"), out _WaitTimeInMillis))
{
_WaitTimeInMillis = _DefaultWaitTime;
}
}
public HttpLogShipper(KinesisSinkState state) : base(state.Options,
new LogReaderFactory(),
new PersistedBookmarkFactory(),
new LogShipperFileManager()
)
{
_throttle = new Throttle(ShipLogs, state.Options.Period);
_kinesisClient = state.KinesisClient;
}
public KinesisSourceSpec(ITestOutputHelper output) : base("akka.loglevel=DEBUG", output)
{
_materializer = Sys.Materializer();
_kinesisClient = NSubstitute.Substitute.For <IAmazonKinesis>();
_settings = ShardSettings.Create("test-stream", "shard-id")
.WithShardIteratorType(ShardIteratorType.TRIM_HORIZON)
.WithRefreshInterval(1.Seconds())
.WithLimit(500);
}
public ProcessedMeasurementsKinesisShardReader(IAmazonKinesis client,
ICheckpointRepository checkpointRepository,
IHeartbeatService heartbeatService,
IBus bus,
ILogger logger)
: base(client, checkpointRepository, heartbeatService)
{
_bus = bus;
_logger = logger;
}
public ReplicatingConsumer(IAmazonKinesis kinesis, KinesisSettings kinesisSettings, DittoSettings dittoSettings, ILogger logger, string streamName, string groupName)
{
_kinesis = kinesis ?? throw new ArgumentNullException(nameof(kinesis));
_kinesisSettings = kinesisSettings ?? throw new ArgumentNullException(nameof(kinesisSettings));
_dittoSettings = dittoSettings ?? throw new ArgumentNullException(nameof(dittoSettings));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
StreamName = streamName ?? throw new ArgumentNullException(nameof(streamName));
GroupName = groupName ?? throw new ArgumentNullException(nameof(groupName));
}
public KinesisService(IAWSConfiguration awsConfiguration)
{
if (awsConfiguration == null)
{
throw new ArgumentNullException(nameof(awsConfiguration));
}
Enum.TryParse(typeof(RegionEndpoint), awsConfiguration.DefaultRegion, true, out object result);
KinesisClient = new AmazonKinesisClient(awsConfiguration.AccessKey, awsConfiguration.SecretKey, (RegionEndpoint)result);
}
public IShardReader Create(IAmazonKinesis client)
{
var recordProcessor = _recordProcessorFactory.Create();
return(new RawMeasurementsKinesisShardReader(client,
_checkpointRepository,
_heartbeatService,
_processedMeasurementStreamWriter,
recordProcessor,
_logger));
}
public Consumer(IAmazonKinesis client, IUtilities utilities, IDynamoDB dynamoDb)
{
_client = client;
_utilities = utilities;
_dynamoDb = dynamoDb;
_isRunning = false;
_cancellationTokenSource = null;
_saveProgressToDynamo = true;
_currentRecordsProcessing = 0;
}
/// <summary>
/// Creates a new Kinesis stream.
/// </summary>
/// <param name="client">An initialized Kinesis client.</param>
/// <param name="streamName">The name for the new stream.</param>
/// <param name="shardCount">The number of shards the new stream will
/// use. The throughput of the stream is a function of the number of
/// shards; more shards are required for greater provisioned
/// throughput.</param>
/// <returns>A Boolean value indicating whether the stream was created.</returns>
public static async Task <bool> CreateNewStreamAsync(IAmazonKinesis client, string streamName, int shardCount)
{
var request = new CreateStreamRequest
{
StreamName = streamName,
ShardCount = shardCount,
};
var response = await client.CreateStreamAsync(request);
return(response.HttpStatusCode == System.Net.HttpStatusCode.OK);
}
/// <summary>
/// Registers the consumer to a Kinesis stream.
/// </summary>
/// <param name="client">The initialized Kinesis client object.</param>
/// <param name="consumerName">A string representing the consumer.</param>
/// <param name="streamARN">The ARN of the stream.</param>
/// <returns>A Consumer object that contains information about the consumer.</returns>
public static async Task <Consumer> RegisterConsumerAsync(IAmazonKinesis client, string consumerName, string streamARN)
{
var request = new RegisterStreamConsumerRequest
{
ConsumerName = consumerName,
StreamARN = streamARN,
};
var response = await client.RegisterStreamConsumerAsync(request);
return(response.Consumer);
}
/// <summary>
/// Retrieve a list of the consumers for a Kinesis stream.
/// </summary>
/// <param name="client">An initialized Kinesis client object.</param>
/// <param name="streamARN">The ARN of the stream for which we want to
/// retrieve a list of clients.</param>
/// <param name="maxResults">The maximum number of results to return.</param>
/// <returns>A list of Consumer objects.</returns>
public static async Task <List <Consumer> > ListConsumersAsync(IAmazonKinesis client, string streamARN, int maxResults)
{
var request = new ListStreamConsumersRequest
{
StreamARN = streamARN,
MaxResults = maxResults,
};
var response = await client.ListStreamConsumersAsync(request);
return(response.Consumers);
}
public RawMeasurementsKinesisShardReader(IAmazonKinesis client,
ICheckpointRepository checkpointRepository,
IHeartbeatService heartbeatService,
IProcessedMeasurementStreamWriter processedMeasurementStreamWriter,
IRecordProcessor recordProcessor,
ILogger logger)
: base(client, checkpointRepository, heartbeatService)
{
_processedMeasurementStreamWriter = processedMeasurementStreamWriter;
_recordProcessor = recordProcessor;
_logger = logger;
}
public KinesisStreamReader(IAmazonKinesis client,
string streamName,
string readerName,
IShardReaderFactory shardReaderFactory,
ICheckpointRepository checkpointRepository,
ILogger logger)
{
_client = client;
_streamName = streamName;
_readerName = readerName;
_shardReaderFactory = shardReaderFactory;
_checkpointRepository = checkpointRepository;
_logger = logger;
}
public Utilities(IAmazonKinesis client, string workerId)
{
_client = client;
_readCapacityUnits = 1;
_writeCapacityUnits = 1;
_workerId = workerId;
if (_workingConsumer == null)
{
_workingConsumer = new ConcurrentDictionary <string, string>();
}
}
protected static async Task SendRecord(IAmazonKinesis kinesisClient)
{
byte[] data = Encoding.UTF8.GetBytes(DateTime.Now.ToString());
using MemoryStream stream = new MemoryStream(data);
var response = await kinesisClient.PutRecordAsync(new PutRecordRequest()
{
StreamName = "vcsjones-test-stream",
PartitionKey = "" + rgen.NextDouble() * 100000,
Data = stream
});
Console.WriteLine(response.HttpStatusCode);
}
/// <summary>
/// Applies the set of tags to the named Kinesis stream.
/// </summary>
/// <param name="client">The initialized Kinesis client.</param>
/// <param name="streamName">The name of the Kinesis stream to which
/// the tags will be attached.</param>
/// <param name="tags">A sictionary containing key/value pairs which
/// will be used to create the Kinesis tags.</param>
/// <returns>A Boolean value which represents the success or failure
/// of AddTagsToStreamAsync.</returns>
public static async Task <bool> ApplyTagsToStreamAsync(IAmazonKinesis client,
string streamName,
Dictionary <string, string> tags)
{
var request = new AddTagsToStreamRequest
{
StreamName = streamName,
Tags = tags,
};
var response = await client.AddTagsToStreamAsync(request);
return(response.HttpStatusCode == System.Net.HttpStatusCode.OK);
}
/// <summary>
/// Deletes a Kinesis stream.
/// </summary>
/// <param name="client">An initialized Kinesis client object.</param>
/// <param name="streamName">The name of the string to delete.</param>
/// <returns>A Boolean value representing the success of the operation.</returns>
public static async Task <bool> DeleteStreamAsync(IAmazonKinesis client, string streamName)
{
// If EnforceConsumerDeletion is true, any consumers
// of this stream will also be deleted. If it is set
// to false and this stream has any consumers, the
// call will fail with a ResourceInUseException.
var request = new DeleteStreamRequest
{
StreamName = streamName,
EnforceConsumerDeletion = true,
};
var response = await client.DeleteStreamAsync(request);
return(response.HttpStatusCode == System.Net.HttpStatusCode.OK);
}
/// <summary>
/// Configures the Amazon Kinesis sink.
/// </summary>
/// <param name="kinesisClient">The Amazon Kinesis client.</param>
/// <param name="streamName">The name of the Kinesis stream.</param>
/// <param name="shardCount"></param>
public KinesisSinkOptions(IAmazonKinesis kinesisClient, string streamName, int?shardCount = null)
: this()
{
if (kinesisClient == null)
{
throw new ArgumentNullException("kinesisClient");
}
if (streamName == null)
{
throw new ArgumentNullException("streamName");
}
KinesisClient = kinesisClient;
StreamName = streamName;
ShardCount = shardCount ?? 1;
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="options"></param>
/// <param name="kinesisClient"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException">A required parameter is null.</exception>
public static LoggerConfiguration AmazonKinesis(
this LoggerSinkConfiguration loggerConfiguration,
KinesisStreamSinkOptions options,IAmazonKinesis kinesisClient)
{
if (loggerConfiguration == null) throw new ArgumentNullException("loggerConfiguration");
if (options == null) throw new ArgumentNullException("options");
if (kinesisClient == null) throw new ArgumentNullException("kinesisClient");
ILogEventSink sink;
if (options.BufferBaseFilename == null)
{
sink = new KinesisSink(options, kinesisClient);
}
else
{
sink = new DurableKinesisSink(options, kinesisClient);
}
return loggerConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum);
}
public DurableKinesisSink(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient)
{
var state = new KinesisSinkState(options, kinesisClient);
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable Amazon Kinesis sink without a buffer base file name.");
}
_sink = new RollingFileSink(
options.BufferBaseFilename + "-{Date}.json",
state.DurableFormatter,
options.BufferFileSizeLimitBytes,
null);
_shipper = new HttpLogShipper(state);
if (options.OnLogSendError != null)
{
_shipper.LogSendError += options.OnLogSendError;
}
}
static bool StreamExists(IAmazonKinesis kinesisClient, string streamName)
{
var stream = DescribeStream(kinesisClient, streamName);
return stream != null;
}
/// <summary>
/// Creates the Amazon Kinesis stream specified and waits for it to become active.
/// </summary>
/// <param name="kinesisClient">The Amazon Kinesis client.</param>
/// <param name="streamName">The name of the steam to be created.</param>
/// <param name="shardCount">The number of shards the stream should be created with.</param>
public static bool CreateAndWaitForStreamToBecomeAvailable(IAmazonKinesis kinesisClient, string streamName, int shardCount)
{
SelfLog.WriteLine(string.Format("Checking stream '{0}' status.", streamName));
var stream = DescribeStream(kinesisClient, streamName);
if (stream != null)
{
string state = stream.StreamDescription.StreamStatus;
switch (state)
{
case "DELETING":
{
SelfLog.WriteLine(string.Format("Stream '{0}' is {1}", streamName, state));
var startTime = DateTime.UtcNow;
var endTime = startTime + TimeSpan.FromSeconds(120);
while (DateTime.UtcNow < endTime && StreamExists(kinesisClient, streamName))
{
SelfLog.WriteLine(string.Format("... waiting for stream '{0}' to delete ...", streamName));
Thread.Sleep(1000 * 5);
}
if (StreamExists(kinesisClient, streamName))
{
var error = string.Format("Timed out waiting for stream '{0}' to delete", streamName);
SelfLog.WriteLine(error);
return false;
}
SelfLog.WriteLine(string.Format("Creating stream '{0}'.", streamName));
var response = CreateStream(kinesisClient, streamName, shardCount);
if (response == null)
{
return false;
}
break;
}
case "CREATING":
{
break;
}
case "ACTIVE":
case "UPDATING":
{
SelfLog.WriteLine(string.Format("Stream '{0}' is {1}", streamName, state));
return true;
}
default:
{
SelfLog.WriteLine(string.Format("Unknown stream state: {0}", state));
return false;
}
}
}
else
{
SelfLog.WriteLine(string.Format("Creating stream '{0}'.", streamName));
var response = CreateStream(kinesisClient, streamName, shardCount);
if (response == null)
{
return false;
}
}
{
// Wait for the stream status to become ACTIVE, timeout after 2 minutes
var startTime = DateTime.UtcNow;
var endTime = startTime + TimeSpan.FromSeconds(120);
while (DateTime.UtcNow < endTime)
{
Thread.Sleep(1000 * 5);
var response = DescribeStream(kinesisClient, streamName);
if (response != null)
{
string state = response.StreamDescription.StreamStatus;
if (state == "ACTIVE")
{
SelfLog.WriteLine(string.Format("Stream '{0}' is {1}", streamName, state));
return true;
}
}
SelfLog.WriteLine(string.Format("... waiting for stream {0} to become active ....", streamName));
}
SelfLog.WriteLine(string.Format("Stream '{0}' never went active.", streamName));
return false;
}
}
public KinesisSinkState(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient)
: base(options)
{
if (kinesisClient == null) throw new ArgumentNullException("kinesisClient");
KinesisClient = kinesisClient;
}
static CreateStreamResponse CreateStream(IAmazonKinesis kinesisClient, string streamName, int shardCount)
{
var request = new CreateStreamRequest { StreamName = streamName, ShardCount = shardCount };
try
{
CreateStreamResponse response = kinesisClient.CreateStream(request);
return response;
}
catch (AmazonServiceException e)
{
var error = string.Format("Failed to create stream '{0}'. Reason: {1}", streamName, e.Message);
SelfLog.WriteLine(error);
return null;
}
}
static DescribeStreamResponse DescribeStream(IAmazonKinesis kinesisClient, string streamName)
{
var request = new DescribeStreamRequest { StreamName = streamName };
try
{
var response = kinesisClient.DescribeStream(request);
return response;
}
catch (ResourceNotFoundException)
{
return null;
}
}
