/// <summary>
/// Construct a sink posting to the specified database.
/// </summary>
/// <param name="kinesisFirehoseClient">The Amazon Kinesis Firehose client.</param>
/// <param name="options">Options for configuring how the sink behaves, may NOT be null.</param>
public KinesisFirehoseSink(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient) :
base(options.BatchPostingLimit, options.Period)
{
_state = new KinesisSinkState(options, kinesisFirehoseClient);
_minimumAcceptedLevel = _state.Options.MinimumLogEventLevel;
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="kinesisFirehoseClient"></param>
/// <param name="streamName"></param>
/// <param name="bufferBaseFilename"></param>
/// <param name="bufferFileSizeLimitBytes"></param>
/// <param name="batchPostingLimit"></param>
/// <param name="period"></param>
/// <param name="minimumLogEventLevel"></param>
/// <param name="onLogSendError"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException"></exception>
public static LoggerConfiguration AmazonKinesisFirehose(
this LoggerSinkConfiguration loggerConfiguration,
IAmazonKinesisFirehose kinesisFirehoseClient,
string streamName,
string bufferBaseFilename = null,
int?bufferFileSizeLimitBytes = null,
int?batchPostingLimit = null,
TimeSpan?period = null,
ITextFormatter customFormatter = null,
LogEventLevel?minimumLogEventLevel = null,
EventHandler <LogSendErrorEventArgs> onLogSendError = null)
{
if (kinesisFirehoseClient == null)
{
throw new ArgumentNullException("kinesisFirehoseClient");
}
if (streamName == null)
{
throw new ArgumentNullException("streamName");
}
var options = new KinesisFirehoseSinkOptions(streamName)
{
BufferFileSizeLimitBytes = bufferFileSizeLimitBytes,
BufferBaseFilename = bufferBaseFilename == null ? null : bufferBaseFilename + ".firehose",
Period = period ?? KinesisSinkOptionsBase.DefaultPeriod,
BatchPostingLimit = batchPostingLimit ?? KinesisSinkOptionsBase.DefaultBatchPostingLimit,
MinimumLogEventLevel = minimumLogEventLevel ?? LevelAlias.Minimum,
OnLogSendError = onLogSendError,
CustomDurableFormatter = customFormatter
};
return(AmazonKinesisFirehose(loggerConfiguration, options, kinesisFirehoseClient));
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="kinesisFirehoseClient"></param>
/// <param name="streamName"></param>
/// <param name="bufferBaseFilename"></param>
/// <param name="bufferFileSizeLimitBytes"></param>
/// <param name="batchPostingLimit"></param>
/// <param name="period"></param>
/// <param name="minimumLogEventLevel"></param>
/// <param name="onLogSendError"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException"></exception>
public static LoggerConfiguration AmazonKinesisFirehose(
this LoggerSinkConfiguration loggerConfiguration,
IAmazonKinesisFirehose kinesisFirehoseClient,
string streamName,
string bufferBaseFilename = null,
int? bufferFileSizeLimitBytes = null,
int? batchPostingLimit = null,
TimeSpan? period = null,
LogEventLevel? minimumLogEventLevel = null,
EventHandler<LogSendErrorEventArgs> onLogSendError = null)
{
if (kinesisFirehoseClient == null) throw new ArgumentNullException("kinesisFirehoseClient");
if (streamName == null) throw new ArgumentNullException("streamName");
var options = new KinesisFirehoseSinkOptions(streamName)
{
BufferFileSizeLimitBytes = bufferFileSizeLimitBytes,
BufferBaseFilename = bufferBaseFilename == null ? null : bufferBaseFilename + ".firehose",
Period = period ?? KinesisSinkOptionsBase.DefaultPeriod,
BatchPostingLimit = batchPostingLimit ?? KinesisSinkOptionsBase.DefaultBatchPostingLimit,
MinimumLogEventLevel = minimumLogEventLevel ?? LevelAlias.Minimum,
OnLogSendError = onLogSendError
};
return AmazonKinesisFirehose(loggerConfiguration, options, kinesisFirehoseClient);
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis Firehose.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="options"></param>
/// <param name="kinesisFirehoseClient"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException">A required parameter is null.</exception>
public static LoggerConfiguration AmazonKinesisFirehose(
this LoggerSinkConfiguration loggerConfiguration,
KinesisFirehoseSinkOptions options,
IAmazonKinesisFirehose kinesisFirehoseClient)
{
if (loggerConfiguration == null)
{
throw new ArgumentNullException("loggerConfiguration");
}
if (options == null)
{
throw new ArgumentNullException("options");
}
ILogEventSink sink;
if (options.BufferBaseFilename == null)
{
sink = new KinesisFirehoseSink(options, kinesisFirehoseClient);
}
else
{
sink = new DurableKinesisFirehoseSink(options, kinesisFirehoseClient);
}
return(loggerConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum));
}
//http://docs.aws.amazon.com/firehose/latest/dev/limits.html
public KinesisFirehoseSink(IPlugInContext context,
IAmazonKinesisFirehose firehoseClient
) : this(context)
{
// Setup Client
FirehoseClient = firehoseClient;
}
/// <summary>
/// Construct a sink posting to the specified database.
/// </summary>
/// <param name="kinesisFirehoseClient">The Amazon Kinesis Firehose client.</param>
/// <param name="options">Options for configuring how the sink behaves, may NOT be null.</param>
public KinesisFirehoseSink(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient) :
base(options.BatchPostingLimit, options.Period)
{
_state = new KinesisSinkState(options, kinesisFirehoseClient);
_minimumAcceptedLevel = _state.Options.MinimumLogEventLevel;
}
//http://docs.aws.amazon.com/firehose/latest/dev/limits.html
public KinesisFirehoseSink(IPlugInContext context,
IAmazonKinesisFirehose firehoseClient
) : base(context, 1, 500, 4 * 1024 * 1024)
{
if (_count > 500)
{
throw new ArgumentException("The maximum buffer size for firehose is 500");
}
//Set Defaults
if (!int.TryParse(_config[ConfigConstants.RECORDS_PER_SECOND], out _maxRecordsPerSecond))
{
_maxRecordsPerSecond = 5000;
}
if (!long.TryParse(_config[ConfigConstants.BYTES_PER_SECOND], out _maxBytesPerSecond))
{
_maxBytesPerSecond = 5 * 1024 * 1024;
}
_firehoseClient = firehoseClient;
_deliveryStreamName = ResolveVariables(_config["StreamName"]);
_throttle = new AdaptiveThrottle(
new TokenBucket[]
{
new TokenBucket(1, _maxRecordsPerSecond / 200), //Number of API calls per second. For simplicity, we tie to _maxRecordsPerSecond
new TokenBucket(_count, _maxRecordsPerSecond),
new TokenBucket(_maxBatchSize, _maxBytesPerSecond)
},
_backoffFactor,
_recoveryFactor,
_minRateAdjustmentFactor);
}
internal KinesisSinkState(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient) : base(options)
{
if (options == null) throw new ArgumentNullException("options");
if (kinesisFirehoseClient == null) throw new ArgumentNullException("kinesisFirehoseClient");
KinesisFirehoseClient = kinesisFirehoseClient;
Options = options;
}
public InputsController(LogContext logContext, AccountContext accountContext, IAmazonS3 S3Client, IAmazonKinesisFirehose FirehoseClient, IAmazonLambda LambdaClient)
{
_logContext = logContext;
_accountContext = accountContext;
_S3Client = S3Client;
_FirehoseClient = FirehoseClient;
_LambdaClient = LambdaClient;
}
public KinesisSinkState(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient) : base(options)
{
if (kinesisFirehoseClient == null)
{
throw new ArgumentNullException("kinesisFirehoseClient");
}
KinesisFirehoseClient = kinesisFirehoseClient;
}
private async Task <bool> TryPublishRecordsAsync(
IAmazonKinesisFirehose firehose,
List <Record> records
)
{
m_metrics.IncrementPutRecordBatchRequests();
try {
int recordCount = records.Count;
var batchRequest = new PutRecordBatchRequest {
DeliveryStreamName = m_args.DeliveryStreamName,
Records = records
};
PutRecordBatchResponse response = await firehose
.PutRecordBatchAsync(batchRequest);
LogEventInfo log = new LogEventInfo {
Properties =
{
{ "Response", response }
}
};
int failedPutCount = response.FailedPutCount;
if (failedPutCount == recordCount)
{
log.Level = LogLevel.Error;
log.Message = "Failed to put all records in batch.";
m_metrics.IncrementRecordsFailed(failedPutCount);
}
else if (failedPutCount > 0)
{
log.Level = LogLevel.Warn;
log.Message = "Failed to put some records in batch.";
m_metrics.IncrementRecordsPut(recordCount - failedPutCount);
m_metrics.IncrementRecordsFailed(failedPutCount);
}
else
{
log.Level = LogLevel.Debug;
log.Message = "Successfully put all records.";
m_metrics.IncrementRecordsPut(recordCount);
}
m_log.Log(log);
return(true);
} catch (Exception err) {
m_log.Error(err, "Failed to put record batch.");
m_metrics.IncrementPutRecordBatchErrors();
return(false);
}
}
public HttpLogShipper(KinesisSinkState state) : base(state.Options,
new LogReaderFactory(),
new PersistedBookmarkFactory(),
new LogShipperFileManager()
)
{
_throttle = new Throttle(ShipLogs, state.Options.Period);
_kinesisFirehoseClient = state.KinesisFirehoseClient;
}
public AmazonKinesisClient(AmazonKinesisOptions options, IAlohaSerializer serializer)
{
var region = RegionEndpoint.GetBySystemName(options.Region);
_client = new AmazonKinesisFirehoseClient(
new BasicAWSCredentials(options.AccessKey, options.SecretKey),
new AmazonKinesisFirehoseConfig {
RegionEndpoint = region
});
_serializer = serializer;
}
public KinesisProducer(IConfiguration config, ILogger logger)
{
_config = config;
_logger = logger;
DeliveryStreamName = _config.GetValue <string>("aws:kinesisStream");
_region = RegionEndpoint.GetBySystemName(_config.GetValue <string>("aws:region"));
if (string.IsNullOrEmpty(DeliveryStreamName))
{
throw new ArgumentNullException("deliveryStreamName");
}
_kinesisFirehoseClient = CreateKinesisFirehouseClient(_region);
}
static async Task Main(string[] args)
{
Console.WriteLine("Loading AWS credentials from environment variables");
var config = new ConfigurationBuilder()
.Build();
// to easily support docker, i'm reading aws credentials from environment variables
// https://stackoverflow.com/questions/43053495/how-to-set-credentials-on-aws-sdk-on-net-core
var options = config.GetAWSOptions();
options.Credentials = new EnvironmentVariablesAWSCredentials();
_firehoseClient = options.CreateServiceClient <IAmazonKinesisFirehose>();
Console.WriteLine("AWS credentials successfully loaded");
await PublishGenericEventsAsync(100);
}
public KinesisMessageArchiver(
IAmazonKinesisFirehose firehose,
IAmazonComprehend comprehend,
IChatMessageStorage storage,
IUserIdsStorageManager userIdsStorage,
ISentimentSnapshotsStorage sentimentSnapshotsStorage,
IAlertBroadcaster broadcaster,
ILogger <KinesisMessageArchiver> logger,
string deliveryStreamName)
{
this.firehose = firehose;
this.comprehend = comprehend;
this.userIdsStorage = userIdsStorage;
this.sentimentSnapshotsStorage = sentimentSnapshotsStorage;
this.chatMessageStorage = storage;
this.broadcaster = broadcaster;
this.logger = logger;
this.deliveryStreamName = deliveryStreamName;
}
/// <summary>
/// Adds a sink that writes log events as documents to Amazon Kinesis Firehose.
/// </summary>
/// <param name="loggerConfiguration">The logger configuration.</param>
/// <param name="options"></param>
/// <param name="kinesisFirehoseClient"></param>
/// <returns>Logger configuration, allowing configuration to continue.</returns>
/// <exception cref="ArgumentNullException">A required parameter is null.</exception>
public static LoggerConfiguration AmazonKinesisFirehose(
this LoggerSinkConfiguration loggerConfiguration,
KinesisFirehoseSinkOptions options,
IAmazonKinesisFirehose kinesisFirehoseClient)
{
if (loggerConfiguration == null) throw new ArgumentNullException("loggerConfiguration");
if (options == null) throw new ArgumentNullException("options");
ILogEventSink sink;
if (options.BufferBaseFilename == null)
{
sink = new KinesisFirehoseSink(options, kinesisFirehoseClient);
}
else
{
sink = new DurableKinesisFirehoseSink(options, kinesisFirehoseClient);
}
return loggerConfiguration.Sink(sink, options.MinimumLogEventLevel ?? LevelAlias.Minimum);
}
public DurableKinesisFirehoseSink(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient)
{
var state = new KinesisSinkState(options, kinesisFirehoseClient);
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable Amazon Kinesis Firehose sink without a buffer base file name.");
}
_sink = new RollingFileSink(
options.BufferBaseFilename + "-{Date}.json",
state.DurableFormatter,
options.BufferFileSizeLimitBytes,
null);
_shipper = new HttpLogShipper(state);
if (options.OnLogSendError != null) {
_shipper.LogSendError += options.OnLogSendError;
}
}
public async Task <HttpResponseMessage> PostAsync(HttpRequestMessage request)
{
Stopwatch watch = new Stopwatch();
bool hadErrors = false;
using (IAmazonKinesisFirehose firehose = m_firehoseProvider.Create())
using (Stream content = await request.Content.ReadAsStreamAsync())
using (StreamReader reader = new StreamReader(content)) {
for (; ;)
{
List <Record> records = await ReadRecordsAsync(reader);
if (records.Count == 0)
{
break;
}
bool published = await TryPublishRecordsAsync(firehose, records);
if (!published)
{
hadErrors = true;
}
}
}
watch.Stop();
var result = new {
took = watch.ElapsedMilliseconds,
errors = hadErrors,
items = Enumerable.Empty <object>()
};
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = JsonHttpContent.Create(result)
});
}
public DurableKinesisFirehoseSink(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient)
{
var state = new KinesisSinkState(options, kinesisFirehoseClient);
if (string.IsNullOrWhiteSpace(options.BufferBaseFilename))
{
throw new ArgumentException("Cannot create the durable Amazon Kinesis Firehose sink without a buffer base file name.");
}
_sink = new RollingFileSink(
options.BufferBaseFilename + "-{Date}.json",
state.DurableFormatter,
options.BufferFileSizeLimitBytes,
null);
_shipper = new HttpLogShipper(state);
_logSendErrorHandler = options.OnLogSendError;
if (_logSendErrorHandler != null)
{
_shipper.LogSendError += _logSendErrorHandler;
}
}
/// <summary>
/// Initializes the connection to AWS Kinesis Firehose.
/// </summary>
private void InitializeAWSKinesisFirehose()
{
string accessKey = "foo";
string secretKey = "bar";
string filename = "BTHumTempReader.awskeys.txt"; // Need to add current namespace in front of the real filename
// Read the needed keys from assembly
// Keys reside in a separate .txt file for better security, not added as literals to source code!
// As you cannot interact with host filesystem in Windows 10 UWP apps (other than few specific locations),
// the file is included in the assembly unit
// File format:
// 1st line: accesskey
// 2nd line: secretkey
// Trick from: http://www.c-sharpcorner.com/UploadFile/4088a7/reading-winrt-component-embedded-resource-file-in-javascript/
try
{
using (var stream = typeof(MainPage).GetTypeInfo().Assembly.GetManifestResourceStream(filename))
{
using (var reader = new StreamReader(stream))
{
accessKey = reader.ReadLine();
secretKey = reader.ReadLine();
}
}
}
catch (Exception e)
{
// Notify the user
statusText.Text = "Error in loading secret keys for cloud connection: " + e.ToString();
return;
}
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
// Create the Kinesis Firehose Client
_client = new AmazonKinesisFirehoseClient(credentials, RegionEndpoint.EUWest1);
}
protected override void ProcessRecord()
{
base.ProcessRecord();
Client = CreateClient(_CurrentCredentials, _RegionEndpoint);
}
private Amazon.KinesisFirehose.Model.StopDeliveryStreamEncryptionResponse CallAWSServiceOperation(IAmazonKinesisFirehose client, Amazon.KinesisFirehose.Model.StopDeliveryStreamEncryptionRequest request)
{
Utils.Common.WriteVerboseEndpointMessage(this, client.Config, "Amazon Kinesis Firehose", "StopDeliveryStreamEncryption");
try
{
#if DESKTOP
return(client.StopDeliveryStreamEncryption(request));
#elif CORECLR
return(client.StopDeliveryStreamEncryptionAsync(request).GetAwaiter().GetResult());
#else
#error "Unknown build edition"
#endif
}
catch (AmazonServiceException exc)
{
var webException = exc.InnerException as System.Net.WebException;
if (webException != null)
{
throw new Exception(Utils.Common.FormatNameResolutionFailureMessage(client.Config, webException.Message), webException);
}
throw;
}
}
public KinesisSinkState(KinesisFirehoseSinkOptions options, IAmazonKinesisFirehose kinesisFirehoseClient)
: base(options)
{
if (kinesisFirehoseClient == null) throw new ArgumentNullException("kinesisFirehoseClient");
KinesisFirehoseClient = kinesisFirehoseClient;
}
public FirehoseSiteTracker()
{
this._client = new AmazonKinesisFirehoseClient(RegionEndpoint.USWest2);
}
