public KinesisStreamManager( IAmazonKinesis amazonKinesisClient, string streamName) { _AmazonKinesisClient = amazonKinesisClient; _StreamName = streamName; }
public KinesisStreamCreator(IAwsClientFactory clientFactory, string rawStreamName, string measurementsStreamName, ILogger logger) { _client = clientFactory.CreateKinesisClient(); _rawStreamName = rawStreamName; _measurementsStreamName = measurementsStreamName; _logger = logger; }
/// <summary> /// Adds a sink that writes log events as documents to Amazon Kinesis. /// </summary> /// <param name="loggerConfiguration">The logger configuration.</param> /// <param name="options"></param> /// <param name="kinesisClient"></param> /// <returns>Logger configuration, allowing configuration to continue.</returns> /// <exception cref="ArgumentNullException">A required parameter is null.</exception> public static LoggerConfiguration AmazonKinesis( this LoggerSinkConfiguration loggerConfiguration, KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) { if (loggerConfiguration == null) { throw new ArgumentNullException("loggerConfiguration"); } if (options == null) { throw new ArgumentNullException("options"); } if (kinesisClient == null) { throw new ArgumentNullException("kinesisClient"); } ILogEventSink sink; if (options.BufferBaseFilename == null) { sink = new KinesisSink(options, kinesisClient); } else { sink = new DurableKinesisSink(options, kinesisClient); } return(loggerConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum)); }
/// <summary> /// Adds a sink that writes log events as documents to Amazon Kinesis. /// </summary> /// <param name="loggerConfiguration">The logger configuration.</param> /// <param name="kinesisClient"></param> /// <param name="streamName"></param> /// <param name="shardCount"></param> /// <param name="bufferBaseFilename"></param> /// <param name="bufferFileSizeLimitBytes"></param> /// <param name="batchPostingLimit"></param> /// <param name="period"></param> /// <param name="minimumLogEventLevel"></param> /// <param name="onLogSendError"></param> /// <returns>Logger configuration, allowing configuration to continue.</returns> /// <exception cref="ArgumentNullException"></exception> public static LoggerConfiguration AmazonKinesis( this LoggerSinkConfiguration loggerConfiguration, IAmazonKinesis kinesisClient, string streamName, int? shardCount = null, string bufferBaseFilename = null, int? bufferFileSizeLimitBytes = null, int? batchPostingLimit = null, TimeSpan? period = null, LogEventLevel? minimumLogEventLevel = null, EventHandler<LogSendErrorEventArgs> onLogSendError = null) { if (streamName == null) throw new ArgumentNullException("streamName"); var options = new KinesisStreamSinkOptions(streamName: streamName, shardCount: shardCount) { BufferFileSizeLimitBytes = bufferFileSizeLimitBytes, BufferBaseFilename = bufferBaseFilename, Period = period ?? KinesisStreamSinkOptions.DefaultPeriod, BatchPostingLimit = batchPostingLimit ?? KinesisStreamSinkOptions.DefaultBatchPostingLimit, MinimumLogEventLevel = minimumLogEventLevel ?? LevelAlias.Minimum, OnLogSendError = onLogSendError }; return AmazonKinesis(loggerConfiguration, options, kinesisClient); }
/// <summary> /// Construct a sink posting to the specified database. /// </summary> /// <param name="options">Options for configuring how the sink behaves, may NOT be null.</param> /// <param name="kinesisClient"></param> public KinesisSink(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) : base(options.BatchPostingLimit, options.Period) { _state = new KinesisSinkState(options, kinesisClient); _minimumAcceptedLevel = _state.Options.MinimumLogEventLevel; }
/// <summary> /// Adds a sink that writes log events as documents to Amazon Kinesis. /// </summary> /// <param name="loggerConfiguration">The logger configuration.</param> /// <param name="kinesisClient"></param> /// <param name="streamName"></param> /// <param name="bufferBaseFilename"></param> /// <param name="bufferFileSizeLimitBytes"></param> /// <param name="batchPostingLimit"></param> /// <param name="period"></param> /// <param name="minimumLogEventLevel"></param> /// <param name="onLogSendError"></param> /// <returns>Logger configuration, allowing configuration to continue.</returns> /// <exception cref="ArgumentNullException"></exception> public static LoggerConfiguration AmazonKinesis( this LoggerSinkConfiguration loggerConfiguration, IAmazonKinesis kinesisClient, string streamName, string bufferBaseFilename = null, int?bufferFileSizeLimitBytes = null, int?batchPostingLimit = null, TimeSpan?period = null, LogEventLevel?minimumLogEventLevel = null, EventHandler <LogSendErrorEventArgs> onLogSendError = null) { if (streamName == null) { throw new ArgumentNullException("streamName"); } var options = new KinesisStreamSinkOptions(streamName: streamName) { BufferFileSizeLimitBytes = bufferFileSizeLimitBytes, BufferBaseFilename = bufferBaseFilename == null ? null : bufferBaseFilename + ".stream", Period = period ?? KinesisSinkOptionsBase.DefaultPeriod, BatchPostingLimit = batchPostingLimit ?? KinesisSinkOptionsBase.DefaultBatchPostingLimit, MinimumLogEventLevel = minimumLogEventLevel ?? LevelAlias.Minimum, OnLogSendError = onLogSendError }; return(AmazonKinesis(loggerConfiguration, options, kinesisClient)); }
internal KinesisSinkState(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) : base(options) { if (options == null) throw new ArgumentNullException("options"); if (kinesisClient == null) throw new ArgumentNullException("kinesisClient"); KinesisClient = kinesisClient; Options = options; }
/// <summary> /// Construct a sink posting to the specified database. /// </summary> /// <param name="options">Options for configuring how the sink behaves, may NOT be null.</param> /// <param name="kinesisClient"></param> public KinesisSink(KinesisStreamSinkOptions options,IAmazonKinesis kinesisClient) : base(options.BatchPostingLimit, options.Period) { _state = new KinesisSinkState(options,kinesisClient); _minimumAcceptedLevel = _state.Options.MinimumLogEventLevel; }
public KinesisStreamSink( IPlugInContext context, IAmazonKinesis kineisClient ) : base(context, 1, 500, 5 * 1024 * 1024) { if (_count > 500) { throw new ArgumentException("The maximum buffer size for kinesis stream is 500"); } //Set Defaults for 1 shard if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond)) { _maxRecordsPerSecond = 1000; } if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond)) { _maxBytesPerSecond = 1024 * 1024; } _kinesisClient = kineisClient; _streamName = ResolveVariables(_config["StreamName"]); _throttle = new AdaptiveThrottle( new TokenBucket[] { new TokenBucket(_count, _maxRecordsPerSecond), new TokenBucket(_maxBatchSize, _maxBytesPerSecond) }, _backoffFactor, _recoveryFactor, _minRateAdjustmentFactor); }
public KinesisShardReaderBase(IAmazonKinesis client, ICheckpointRepository checkpointRepository, IHeartbeatService heartbeatService) { _client = client; CheckpointRepository = checkpointRepository; HeartbeatService = heartbeatService; }
public IShardReader Create(IAmazonKinesis client) { return(new ProcessedMeasurementsKinesisShardReader(client, _checkpointRepository, _heartbeatService, _bus, _logger)); }
public KinesisSinkState(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) : base(options) { if (kinesisClient == null) { throw new ArgumentNullException("kinesisClient"); } KinesisClient = kinesisClient; }
public Logic(KinesisSourceStage stage) : base(stage.Shape) { _settings = stage.Settings; _out = stage.Outlet; _kinesisClient = stage._clientFactory(); SetHandler(_out, onPull: () => _self.Tell(Pump.Instance)); }
/// <summary> /// Writer for processed measurements stream /// </summary> /// <returns></returns> public IProcessedMeasurementStreamWriter CreateWriter() { // Processed measurements go to this stream. var measurementsStreamName = _awsSettings.ProcessedMeasurementsStreamName; IAmazonKinesis client = _clientFactory.CreateKinesisClient(); return(new ProcessedMeasurementStreamWriter(client, measurementsStreamName, _logger)); }
public KinesisClient(IAmazonKinesis kinesis, string streamName) { if (string.IsNullOrEmpty(streamName)) { throw new ArgumentNullException(nameof(streamName)); } _kinesis = kinesis; _streamName = streamName; }
/// <summary> /// Default constructor that Lambda will invoke. /// </summary> public Entrypoint() { this._KinesisClient = new AmazonKinesisClient(); if (!int.TryParse(Environment.GetEnvironmentVariable("WAIT_TIME"), out _WaitTimeInMillis)) { _WaitTimeInMillis = _DefaultWaitTime; } }
public HttpLogShipper(KinesisSinkState state) : base(state.Options, new LogReaderFactory(), new PersistedBookmarkFactory(), new LogShipperFileManager() ) { _throttle = new Throttle(ShipLogs, state.Options.Period); _kinesisClient = state.KinesisClient; }
public KinesisSourceSpec(ITestOutputHelper output) : base("akka.loglevel=DEBUG", output) { _materializer = Sys.Materializer(); _kinesisClient = NSubstitute.Substitute.For <IAmazonKinesis>(); _settings = ShardSettings.Create("test-stream", "shard-id") .WithShardIteratorType(ShardIteratorType.TRIM_HORIZON) .WithRefreshInterval(1.Seconds()) .WithLimit(500); }
public ProcessedMeasurementsKinesisShardReader(IAmazonKinesis client, ICheckpointRepository checkpointRepository, IHeartbeatService heartbeatService, IBus bus, ILogger logger) : base(client, checkpointRepository, heartbeatService) { _bus = bus; _logger = logger; }
public ReplicatingConsumer(IAmazonKinesis kinesis, KinesisSettings kinesisSettings, DittoSettings dittoSettings, ILogger logger, string streamName, string groupName) { _kinesis = kinesis ?? throw new ArgumentNullException(nameof(kinesis)); _kinesisSettings = kinesisSettings ?? throw new ArgumentNullException(nameof(kinesisSettings)); _dittoSettings = dittoSettings ?? throw new ArgumentNullException(nameof(dittoSettings)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); StreamName = streamName ?? throw new ArgumentNullException(nameof(streamName)); GroupName = groupName ?? throw new ArgumentNullException(nameof(groupName)); }
public KinesisService(IAWSConfiguration awsConfiguration) { if (awsConfiguration == null) { throw new ArgumentNullException(nameof(awsConfiguration)); } Enum.TryParse(typeof(RegionEndpoint), awsConfiguration.DefaultRegion, true, out object result); KinesisClient = new AmazonKinesisClient(awsConfiguration.AccessKey, awsConfiguration.SecretKey, (RegionEndpoint)result); }
public IShardReader Create(IAmazonKinesis client) { var recordProcessor = _recordProcessorFactory.Create(); return(new RawMeasurementsKinesisShardReader(client, _checkpointRepository, _heartbeatService, _processedMeasurementStreamWriter, recordProcessor, _logger)); }
public Consumer(IAmazonKinesis client, IUtilities utilities, IDynamoDB dynamoDb) { _client = client; _utilities = utilities; _dynamoDb = dynamoDb; _isRunning = false; _cancellationTokenSource = null; _saveProgressToDynamo = true; _currentRecordsProcessing = 0; }
/// <summary> /// Creates a new Kinesis stream. /// </summary> /// <param name="client">An initialized Kinesis client.</param> /// <param name="streamName">The name for the new stream.</param> /// <param name="shardCount">The number of shards the new stream will /// use. The throughput of the stream is a function of the number of /// shards; more shards are required for greater provisioned /// throughput.</param> /// <returns>A Boolean value indicating whether the stream was created.</returns> public static async Task <bool> CreateNewStreamAsync(IAmazonKinesis client, string streamName, int shardCount) { var request = new CreateStreamRequest { StreamName = streamName, ShardCount = shardCount, }; var response = await client.CreateStreamAsync(request); return(response.HttpStatusCode == System.Net.HttpStatusCode.OK); }
/// <summary> /// Registers the consumer to a Kinesis stream. /// </summary> /// <param name="client">The initialized Kinesis client object.</param> /// <param name="consumerName">A string representing the consumer.</param> /// <param name="streamARN">The ARN of the stream.</param> /// <returns>A Consumer object that contains information about the consumer.</returns> public static async Task <Consumer> RegisterConsumerAsync(IAmazonKinesis client, string consumerName, string streamARN) { var request = new RegisterStreamConsumerRequest { ConsumerName = consumerName, StreamARN = streamARN, }; var response = await client.RegisterStreamConsumerAsync(request); return(response.Consumer); }
/// <summary> /// Retrieve a list of the consumers for a Kinesis stream. /// </summary> /// <param name="client">An initialized Kinesis client object.</param> /// <param name="streamARN">The ARN of the stream for which we want to /// retrieve a list of clients.</param> /// <param name="maxResults">The maximum number of results to return.</param> /// <returns>A list of Consumer objects.</returns> public static async Task <List <Consumer> > ListConsumersAsync(IAmazonKinesis client, string streamARN, int maxResults) { var request = new ListStreamConsumersRequest { StreamARN = streamARN, MaxResults = maxResults, }; var response = await client.ListStreamConsumersAsync(request); return(response.Consumers); }
public RawMeasurementsKinesisShardReader(IAmazonKinesis client, ICheckpointRepository checkpointRepository, IHeartbeatService heartbeatService, IProcessedMeasurementStreamWriter processedMeasurementStreamWriter, IRecordProcessor recordProcessor, ILogger logger) : base(client, checkpointRepository, heartbeatService) { _processedMeasurementStreamWriter = processedMeasurementStreamWriter; _recordProcessor = recordProcessor; _logger = logger; }
public KinesisStreamReader(IAmazonKinesis client, string streamName, string readerName, IShardReaderFactory shardReaderFactory, ICheckpointRepository checkpointRepository, ILogger logger) { _client = client; _streamName = streamName; _readerName = readerName; _shardReaderFactory = shardReaderFactory; _checkpointRepository = checkpointRepository; _logger = logger; }
public Utilities(IAmazonKinesis client, string workerId) { _client = client; _readCapacityUnits = 1; _writeCapacityUnits = 1; _workerId = workerId; if (_workingConsumer == null) { _workingConsumer = new ConcurrentDictionary <string, string>(); } }
protected static async Task SendRecord(IAmazonKinesis kinesisClient) { byte[] data = Encoding.UTF8.GetBytes(DateTime.Now.ToString()); using MemoryStream stream = new MemoryStream(data); var response = await kinesisClient.PutRecordAsync(new PutRecordRequest() { StreamName = "vcsjones-test-stream", PartitionKey = "" + rgen.NextDouble() * 100000, Data = stream }); Console.WriteLine(response.HttpStatusCode); }
/// <summary> /// Applies the set of tags to the named Kinesis stream. /// </summary> /// <param name="client">The initialized Kinesis client.</param> /// <param name="streamName">The name of the Kinesis stream to which /// the tags will be attached.</param> /// <param name="tags">A sictionary containing key/value pairs which /// will be used to create the Kinesis tags.</param> /// <returns>A Boolean value which represents the success or failure /// of AddTagsToStreamAsync.</returns> public static async Task <bool> ApplyTagsToStreamAsync(IAmazonKinesis client, string streamName, Dictionary <string, string> tags) { var request = new AddTagsToStreamRequest { StreamName = streamName, Tags = tags, }; var response = await client.AddTagsToStreamAsync(request); return(response.HttpStatusCode == System.Net.HttpStatusCode.OK); }
/// <summary> /// Deletes a Kinesis stream. /// </summary> /// <param name="client">An initialized Kinesis client object.</param> /// <param name="streamName">The name of the string to delete.</param> /// <returns>A Boolean value representing the success of the operation.</returns> public static async Task <bool> DeleteStreamAsync(IAmazonKinesis client, string streamName) { // If EnforceConsumerDeletion is true, any consumers // of this stream will also be deleted. If it is set // to false and this stream has any consumers, the // call will fail with a ResourceInUseException. var request = new DeleteStreamRequest { StreamName = streamName, EnforceConsumerDeletion = true, }; var response = await client.DeleteStreamAsync(request); return(response.HttpStatusCode == System.Net.HttpStatusCode.OK); }
/// <summary> /// Configures the Amazon Kinesis sink. /// </summary> /// <param name="kinesisClient">The Amazon Kinesis client.</param> /// <param name="streamName">The name of the Kinesis stream.</param> /// <param name="shardCount"></param> public KinesisSinkOptions(IAmazonKinesis kinesisClient, string streamName, int?shardCount = null) : this() { if (kinesisClient == null) { throw new ArgumentNullException("kinesisClient"); } if (streamName == null) { throw new ArgumentNullException("streamName"); } KinesisClient = kinesisClient; StreamName = streamName; ShardCount = shardCount ?? 1; }
/// <summary> /// Adds a sink that writes log events as documents to Amazon Kinesis. /// </summary> /// <param name="loggerConfiguration">The logger configuration.</param> /// <param name="options"></param> /// <param name="kinesisClient"></param> /// <returns>Logger configuration, allowing configuration to continue.</returns> /// <exception cref="ArgumentNullException">A required parameter is null.</exception> public static LoggerConfiguration AmazonKinesis( this LoggerSinkConfiguration loggerConfiguration, KinesisStreamSinkOptions options,IAmazonKinesis kinesisClient) { if (loggerConfiguration == null) throw new ArgumentNullException("loggerConfiguration"); if (options == null) throw new ArgumentNullException("options"); if (kinesisClient == null) throw new ArgumentNullException("kinesisClient"); ILogEventSink sink; if (options.BufferBaseFilename == null) { sink = new KinesisSink(options, kinesisClient); } else { sink = new DurableKinesisSink(options, kinesisClient); } return loggerConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum); }
public DurableKinesisSink(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) { var state = new KinesisSinkState(options, kinesisClient); if (string.IsNullOrWhiteSpace(options.BufferBaseFilename)) { throw new ArgumentException("Cannot create the durable Amazon Kinesis sink without a buffer base file name."); } _sink = new RollingFileSink( options.BufferBaseFilename + "-{Date}.json", state.DurableFormatter, options.BufferFileSizeLimitBytes, null); _shipper = new HttpLogShipper(state); if (options.OnLogSendError != null) { _shipper.LogSendError += options.OnLogSendError; } }
static bool StreamExists(IAmazonKinesis kinesisClient, string streamName) { var stream = DescribeStream(kinesisClient, streamName); return stream != null; }
/// <summary> /// Creates the Amazon Kinesis stream specified and waits for it to become active. /// </summary> /// <param name="kinesisClient">The Amazon Kinesis client.</param> /// <param name="streamName">The name of the steam to be created.</param> /// <param name="shardCount">The number of shards the stream should be created with.</param> public static bool CreateAndWaitForStreamToBecomeAvailable(IAmazonKinesis kinesisClient, string streamName, int shardCount) { SelfLog.WriteLine(string.Format("Checking stream '{0}' status.", streamName)); var stream = DescribeStream(kinesisClient, streamName); if (stream != null) { string state = stream.StreamDescription.StreamStatus; switch (state) { case "DELETING": { SelfLog.WriteLine(string.Format("Stream '{0}' is {1}", streamName, state)); var startTime = DateTime.UtcNow; var endTime = startTime + TimeSpan.FromSeconds(120); while (DateTime.UtcNow < endTime && StreamExists(kinesisClient, streamName)) { SelfLog.WriteLine(string.Format("... waiting for stream '{0}' to delete ...", streamName)); Thread.Sleep(1000 * 5); } if (StreamExists(kinesisClient, streamName)) { var error = string.Format("Timed out waiting for stream '{0}' to delete", streamName); SelfLog.WriteLine(error); return false; } SelfLog.WriteLine(string.Format("Creating stream '{0}'.", streamName)); var response = CreateStream(kinesisClient, streamName, shardCount); if (response == null) { return false; } break; } case "CREATING": { break; } case "ACTIVE": case "UPDATING": { SelfLog.WriteLine(string.Format("Stream '{0}' is {1}", streamName, state)); return true; } default: { SelfLog.WriteLine(string.Format("Unknown stream state: {0}", state)); return false; } } } else { SelfLog.WriteLine(string.Format("Creating stream '{0}'.", streamName)); var response = CreateStream(kinesisClient, streamName, shardCount); if (response == null) { return false; } } { // Wait for the stream status to become ACTIVE, timeout after 2 minutes var startTime = DateTime.UtcNow; var endTime = startTime + TimeSpan.FromSeconds(120); while (DateTime.UtcNow < endTime) { Thread.Sleep(1000 * 5); var response = DescribeStream(kinesisClient, streamName); if (response != null) { string state = response.StreamDescription.StreamStatus; if (state == "ACTIVE") { SelfLog.WriteLine(string.Format("Stream '{0}' is {1}", streamName, state)); return true; } } SelfLog.WriteLine(string.Format("... waiting for stream {0} to become active ....", streamName)); } SelfLog.WriteLine(string.Format("Stream '{0}' never went active.", streamName)); return false; } }
public KinesisSinkState(KinesisStreamSinkOptions options, IAmazonKinesis kinesisClient) : base(options) { if (kinesisClient == null) throw new ArgumentNullException("kinesisClient"); KinesisClient = kinesisClient; }
static CreateStreamResponse CreateStream(IAmazonKinesis kinesisClient, string streamName, int shardCount) { var request = new CreateStreamRequest { StreamName = streamName, ShardCount = shardCount }; try { CreateStreamResponse response = kinesisClient.CreateStream(request); return response; } catch (AmazonServiceException e) { var error = string.Format("Failed to create stream '{0}'. Reason: {1}", streamName, e.Message); SelfLog.WriteLine(error); return null; } }
static DescribeStreamResponse DescribeStream(IAmazonKinesis kinesisClient, string streamName) { var request = new DescribeStreamRequest { StreamName = streamName }; try { var response = kinesisClient.DescribeStream(request); return response; } catch (ResourceNotFoundException) { return null; } }